Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone. Two source URLs serving the same
bytes store them twice. Reconciliation still drops a source row whose
file is missing, as TestReconciliationReadsAPageAtATime counts those
reads. The tests of the removed sharing, lock and hook go with them.

Model: opus-5-5
This commit is contained in:
2026-10-21 13:05:11 +00:00
parent 00d8ba26c5
commit d571045d01
22 changed files with 435 additions and 1597 deletions
+7 -7
View File
@@ -359,9 +359,9 @@ func withoutCancelKeepingDeadline(
// returns it with its size; nil if the cached data is unavailable, empty or
// exceeds maxResponseSize.
func (s *Service) loadCachedSource(
contentHash ContentHash,
sourceKey imagerequest.SourceKey,
) (io.ReadCloser, int64) {
reader, size, err := s.cache.GetSourceContent(contentHash)
reader, size, err := s.cache.GetSourceContent(sourceKey)
if err != nil {
s.log.Warn("failed to load cached source, fetching", "error", err)
@@ -372,7 +372,7 @@ func (s *Service) loadCachedSource(
_ = reader.Close()
s.log.Warn("cached source exceeds max response size, discarding",
"hash", contentHash,
"source_key", sourceKey,
"max_bytes", s.maxResponseSize,
)
@@ -396,7 +396,7 @@ func (s *Service) processFromSourceOrFetch(
cacheKey imagerequest.VariantKey,
) (*processedVariant, error) {
// Check if we have cached source content
contentHash, _, err := s.cache.LookupSource(ctx, req)
sourceKey, _, err := s.cache.LookupSource(ctx, req)
if err != nil {
s.log.Warn("source lookup failed", "error", err)
}
@@ -406,9 +406,9 @@ func (s *Service) processFromSourceOrFetch(
sourceSize int64
)
if contentHash != "" {
s.log.Debug("using cached source", "hash", contentHash)
source, sourceSize = s.loadCachedSource(contentHash)
if sourceKey != "" {
s.log.Debug("using cached source", "source_key", sourceKey)
source, sourceSize = s.loadCachedSource(sourceKey)
}
// Fetch from upstream if we don't have source data or it's empty