Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing, so
TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20.
Two source URLs serving the same bytes store them twice. The tests of
the removed sharing and lock go with them; the two eviction
cancellation tests no longer use the hook.

Model: opus-5-5
This commit is contained in:
2026-10-21 13:50:43 +00:00
parent 00d8ba26c5
commit e06192b7a5
24 changed files with 599 additions and 1561 deletions
+7 -21
View File
@@ -3,9 +3,7 @@ package server
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"image"
"image/color"
"image/jpeg"
@@ -30,6 +28,7 @@ import (
"sneak.berlin/go/pixa/internal/handlers"
"sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/logger"
"sneak.berlin/go/pixa/internal/middleware"
)
@@ -254,39 +253,26 @@ func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder
return rec
}
// checkSourceCached checks that source is stored under its SHA-256 in
// cache/sources, recorded in source_content, and that the source URL's row in
// source_metadata points at it.
// checkSourceCached checks that source is stored in cache/sources under the
// source key of upstreamHost and /photo.png, and recorded under that key in
// source_content.
func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
t.Helper()
sum := sha256.Sum256(source)
hash := hex.EncodeToString(sum[:])
key := string(imagerequest.Source{Host: upstreamHost, Path: "/photo.png"}.Key())
checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
checkFile(t, filepath.Join(stateDir, "cache", "sources", key[0:2], key[2:4], key),
source)
var rows int
err := db.QueryRowContext(t.Context(),
"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
"SELECT COUNT(*) FROM source_content WHERE source_key = ?", key,
).Scan(&rows)
if err != nil || rows != 1 {
t.Errorf("source_content rows for the source = %d (error %v), want 1",
rows, err)
}
var metadataHash string
err = db.QueryRowContext(t.Context(),
`SELECT content_hash FROM source_metadata
WHERE source_host = ? AND source_path = ?`,
upstreamHost, "/photo.png",
).Scan(&metadataHash)
if err != nil || metadataHash != hash {
t.Errorf("source_metadata content_hash = %q (error %v), want %q",
metadataHash, err, hash)
}
}
// checkVariantCached checks that the converted image served is recorded in