Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone. Two source URLs serving the same
bytes store them twice. Reconciliation still drops a source row whose
file is missing, as TestReconciliationReadsAPageAtATime counts those
reads. The tests of the removed sharing, lock and hook go with them.

Model: opus-5-5
This commit is contained in:
2026-10-21 13:05:11 +00:00
parent 00d8ba26c5
commit d571045d01
22 changed files with 435 additions and 1597 deletions
+2 -40
View File
@@ -3,9 +3,7 @@ package server
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"image"
"image/color"
"image/jpeg"
@@ -43,8 +41,8 @@ const upstreamHost = "192.0.2.10"
// upstream fetcher, image processor, disk cache and database, with only the
// upstream origin replaced by a local test server. The first request for a URL
// is fetched and converted; the second is served from the cache without
// another upstream request. The source and the converted image are then on
// disk, with their rows in the database.
// another upstream request. The converted image is then on disk, with its row
// in the database.
func TestImageProxyFlow(t *testing.T) {
t.Parallel()
@@ -113,7 +111,6 @@ func TestImageProxyFlow(t *testing.T) {
t.Errorf("upstream received %d requests, want 1", got)
}
checkSourceCached(t, db, stateDir, source)
checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType)
})
}
@@ -254,41 +251,6 @@ func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder
return rec
}
// checkSourceCached checks that source is stored under its SHA-256 in
// cache/sources, recorded in source_content, and that the source URL's row in
// source_metadata points at it.
func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
t.Helper()
sum := sha256.Sum256(source)
hash := hex.EncodeToString(sum[:])
checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
source)
var rows int
err := db.QueryRowContext(t.Context(),
"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
).Scan(&rows)
if err != nil || rows != 1 {
t.Errorf("source_content rows for the source = %d (error %v), want 1",
rows, err)
}
var metadataHash string
err = db.QueryRowContext(t.Context(),
`SELECT content_hash FROM source_metadata
WHERE source_host = ? AND source_path = ?`,
upstreamHost, "/photo.png",
).Scan(&metadataHash)
if err != nil || metadataHash != hash {
t.Errorf("source_metadata content_hash = %q (error %v), want %q",
metadataHash, err, hash)
}
}
// checkVariantCached checks that the converted image served is recorded in
// variant_content with contentType, and stored under its cache key in
// cache/variants.