Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5
This commit was merged in pull request #324.
This commit is contained in:
@@ -3,9 +3,7 @@ package server
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
@@ -30,6 +28,7 @@ import (
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
)
|
||||
@@ -254,39 +253,26 @@ func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder
|
||||
return rec
|
||||
}
|
||||
|
||||
// checkSourceCached checks that source is stored under its SHA-256 in
|
||||
// cache/sources, recorded in source_content, and that the source URL's row in
|
||||
// source_metadata points at it.
|
||||
// checkSourceCached checks that source is stored in cache/sources under the
|
||||
// source key of upstreamHost and /photo.png, and recorded under that key in
|
||||
// source_content.
|
||||
func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
|
||||
t.Helper()
|
||||
|
||||
sum := sha256.Sum256(source)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
key := string(imagerequest.Source{Host: upstreamHost, Path: "/photo.png"}.Key())
|
||||
|
||||
checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
|
||||
checkFile(t, filepath.Join(stateDir, "cache", "sources", key[0:2], key[2:4], key),
|
||||
source)
|
||||
|
||||
var rows int
|
||||
|
||||
err := db.QueryRowContext(t.Context(),
|
||||
"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
|
||||
"SELECT COUNT(*) FROM source_content WHERE source_key = ?", key,
|
||||
).Scan(&rows)
|
||||
if err != nil || rows != 1 {
|
||||
t.Errorf("source_content rows for the source = %d (error %v), want 1",
|
||||
rows, err)
|
||||
}
|
||||
|
||||
var metadataHash string
|
||||
|
||||
err = db.QueryRowContext(t.Context(),
|
||||
`SELECT content_hash FROM source_metadata
|
||||
WHERE source_host = ? AND source_path = ?`,
|
||||
upstreamHost, "/photo.png",
|
||||
).Scan(&metadataHash)
|
||||
if err != nil || metadataHash != hash {
|
||||
t.Errorf("source_metadata content_hash = %q (error %v), want %q",
|
||||
metadataHash, err, hash)
|
||||
}
|
||||
}
|
||||
|
||||
// checkVariantCached checks that the converted image served is recorded in
|
||||
|
||||
Reference in New Issue
Block a user