Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone. Two source URLs serving the same bytes store them twice. Reconciliation still drops a source row whose file is missing, as TestReconciliationReadsAPageAtATime counts those reads. The tests of the removed sharing, lock and hook go with them. Model: opus-5-5
This commit is contained in:
@@ -359,9 +359,9 @@ func withoutCancelKeepingDeadline(
|
||||
// returns it with its size; nil if the cached data is unavailable, empty or
|
||||
// exceeds maxResponseSize.
|
||||
func (s *Service) loadCachedSource(
|
||||
contentHash ContentHash,
|
||||
sourceKey imagerequest.SourceKey,
|
||||
) (io.ReadCloser, int64) {
|
||||
reader, size, err := s.cache.GetSourceContent(contentHash)
|
||||
reader, size, err := s.cache.GetSourceContent(sourceKey)
|
||||
if err != nil {
|
||||
s.log.Warn("failed to load cached source, fetching", "error", err)
|
||||
|
||||
@@ -372,7 +372,7 @@ func (s *Service) loadCachedSource(
|
||||
_ = reader.Close()
|
||||
|
||||
s.log.Warn("cached source exceeds max response size, discarding",
|
||||
"hash", contentHash,
|
||||
"source_key", sourceKey,
|
||||
"max_bytes", s.maxResponseSize,
|
||||
)
|
||||
|
||||
@@ -396,7 +396,7 @@ func (s *Service) processFromSourceOrFetch(
|
||||
cacheKey imagerequest.VariantKey,
|
||||
) (*processedVariant, error) {
|
||||
// Check if we have cached source content
|
||||
contentHash, _, err := s.cache.LookupSource(ctx, req)
|
||||
sourceKey, _, err := s.cache.LookupSource(ctx, req)
|
||||
if err != nil {
|
||||
s.log.Warn("source lookup failed", "error", err)
|
||||
}
|
||||
@@ -406,9 +406,9 @@ func (s *Service) processFromSourceOrFetch(
|
||||
sourceSize int64
|
||||
)
|
||||
|
||||
if contentHash != "" {
|
||||
s.log.Debug("using cached source", "hash", contentHash)
|
||||
source, sourceSize = s.loadCachedSource(contentHash)
|
||||
if sourceKey != "" {
|
||||
s.log.Debug("using cached source", "source_key", sourceKey)
|
||||
source, sourceSize = s.loadCachedSource(sourceKey)
|
||||
}
|
||||
|
||||
// Fetch from upstream if we don't have source data or it's empty
|
||||
|
||||
Reference in New Issue
Block a user