Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5
This commit was merged in pull request #324.
This commit is contained in:
2026-10-21 16:31:33 +02:00
parent 00d8ba26c5
commit a19cca2f34
24 changed files with 622 additions and 1468 deletions
+7 -21
View File
@@ -3,9 +3,7 @@ package server
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"image"
"image/color"
"image/jpeg"
@@ -30,6 +28,7 @@ import (
"sneak.berlin/go/pixa/internal/handlers"
"sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/logger"
"sneak.berlin/go/pixa/internal/middleware"
)
@@ -254,39 +253,26 @@ func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder
return rec
}
// checkSourceCached checks that source is stored under its SHA-256 in
// cache/sources, recorded in source_content, and that the source URL's row in
// source_metadata points at it.
// checkSourceCached checks that source is stored in cache/sources under the
// source key of upstreamHost and /photo.png, and recorded under that key in
// source_content.
func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
t.Helper()
sum := sha256.Sum256(source)
hash := hex.EncodeToString(sum[:])
key := string(imagerequest.Source{Host: upstreamHost, Path: "/photo.png"}.Key())
checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
checkFile(t, filepath.Join(stateDir, "cache", "sources", key[0:2], key[2:4], key),
source)
var rows int
err := db.QueryRowContext(t.Context(),
"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
"SELECT COUNT(*) FROM source_content WHERE source_key = ?", key,
).Scan(&rows)
if err != nil || rows != 1 {
t.Errorf("source_content rows for the source = %d (error %v), want 1",
rows, err)
}
var metadataHash string
err = db.QueryRowContext(t.Context(),
`SELECT content_hash FROM source_metadata
WHERE source_host = ? AND source_path = ?`,
upstreamHost, "/photo.png",
).Scan(&metadataHash)
if err != nil || metadataHash != hash {
t.Errorf("source_metadata content_hash = %q (error %v), want %q",
metadataHash, err, hash)
}
}
// checkVariantCached checks that the converted image served is recorded in