sneak/pixa
1
0
Fork 0
Code Issues 19 Pull Requests 3 Actions 6 Packages Projects Releases Wiki Activity

Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
Details

Browse Source 
A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5
This commit was merged in pull request #324.
This commit is contained in:
clawbot
2026-10-21 16:31:33 +02:00
parent 00d8ba26c5
commit a19cca2f34
24 changed files with 622 additions and 1468 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imgcache/eviction_internal_test.go
+161 -319
View File
@@ -10,10 +10,11 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
_ "modernc.org/sqlite"
"modernc.org/sqlite"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imagerequest"
@@ -83,10 +84,10 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) {
}
// storeEvictionTestSource stores content as a fetched source for
// host/path and returns the resulting content hash.
// host/path and returns the resulting source key.
func storeEvictionTestSource(
t *testing.T, cache *Cache, host, path string, content []byte,
) ContentHash {
) imagerequest.SourceKey {
t.Helper()
req := &imagerequest.Request{
@@ -110,14 +111,14 @@ func storeEvictionTestSource(
},
}
hash, err := cache.StoreSource(
key, err := cache.StoreSource(
context.Background(), req, bytes.NewReader(content), result,
)
if err != nil {
t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err)
}
return hash
return key
}
// storeEvictionTestVariant stores content as a processed variant under
@@ -162,13 +163,13 @@ func setVariantLastAccessed(
// setSourceLastAccessed backdates the last access time of a tracked
// source content blob.
func setSourceLastAccessed(
t *testing.T, cache *Cache, hash ContentHash, when time.Time,
t *testing.T, cache *Cache, key imagerequest.SourceKey, when time.Time,
) {
t.Helper()
res, err := cache.db.ExecContext(t.Context(),
`UPDATE source_content SET last_accessed_at = ? WHERE content_hash = ?`,
when.UTC().Format(sqliteTimestampFormat), string(hash),
`UPDATE source_content SET last_accessed_at = ? WHERE source_key = ?`,
when.UTC().Format(sqliteTimestampFormat), string(key),
)
if err != nil {
t.Fatalf("failed to set source last_accessed_at: %v", err)
@@ -180,7 +181,7 @@ func setSourceLastAccessed(
}
if affected != 1 {
t.Fatalf("source %s has no accounting row (affected=%d)", hash, affected)
t.Fatalf("source %s has no accounting row (affected=%d)", key, affected)
}
}
@@ -205,32 +206,31 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) {
t.Helper()
rows, err := cache.db.QueryContext(t.Context(),
`SELECT content_hash FROM source_metadata
WHERE content_hash IS NOT NULL AND content_hash != ''`,
`SELECT source_key FROM source_content`,
)
if err != nil {
t.Fatalf("failed to query source_metadata: %v", err)
t.Fatalf("failed to query source_content: %v", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var hash string
var key string
err := rows.Scan(&hash)
err := rows.Scan(&key)
if err != nil {
t.Fatalf("failed to scan content_hash: %v", err)
t.Fatalf("failed to scan source_key: %v", err)
}
if !cache.srcContent.Exists(ContentHash(hash)) {
t.Errorf("source_metadata references content %s but the file is missing",
hash)
if !cache.srcContent.Exists(imagerequest.SourceKey(key)) {
t.Errorf("source_content references content %s but the file is missing",
key)
}
}
err = rows.Err()
if err != nil {
t.Fatalf("source_metadata iteration failed: %v", err)
t.Fatalf("source_content iteration failed: %v", err)
}
variantRows, err := cache.db.QueryContext(t.Context(),
@@ -313,33 +313,6 @@ func TestUsageBytesAccountsSourceAndVariantBytes(t *testing.T) {
}
}
func TestUsageBytesCountsMultiReferencedBlobOnce(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
content := bytes.Repeat([]byte{0xCC}, 1200)
hashOne := storeEvictionTestSource(t, cache,
"src.example.com", "/one.jpg", content)
hashTwo := storeEvictionTestSource(t, cache,
"src.example.com", "/two.jpg", content)
if hashOne != hashTwo {
t.Fatalf("identical content produced different hashes: %s vs %s",
hashOne, hashTwo)
}
usage, err := cache.UsageBytes(context.Background())
if err != nil {
t.Fatalf("UsageBytes failed: %v", err)
}
if usage != 1200 {
t.Errorf("UsageBytes = %d, want 1200 (deduplicated blob counted once)", usage)
}
}
func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) {
t.Parallel()
@@ -393,102 +366,15 @@ func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) {
assertNoDanglingReferences(t, cache)
}
func TestEvictionRemovesMultiReferencedBlobTogetherWithAllReferences(t *testing.T) {
t.Parallel()
const limit = 1000
cache, _ := newEvictionTestCache(t, limit)
now := time.Now()
// One 800-byte blob referenced by two source paths.
sharedContent := bytes.Repeat([]byte{0xDD}, 800)
sharedHash := storeEvictionTestSource(t, cache,
"src.example.com", "/a.jpg", sharedContent)
h := storeEvictionTestSource(t, cache,
"src.example.com", "/b.jpg", sharedContent)
if h != sharedHash {
t.Fatalf("identical content produced different hashes: %s vs %s",
h, sharedHash)
}
// A newer 600-byte blob referenced by one source path.
recentHash := storeEvictionTestSource(t, cache, "src.example.com", "/c.jpg",
bytes.Repeat([]byte{0xEE}, 600))
setSourceLastAccessed(t, cache, sharedHash, now.Add(-2*time.Hour))
setSourceLastAccessed(t, cache, recentHash, now.Add(-time.Minute))
err := cache.EvictToLimit(context.Background())
if err != nil {
t.Fatalf("EvictToLimit failed: %v", err)
}
usage, err := cache.UsageBytes(context.Background())
if err != nil {
t.Fatalf("UsageBytes failed: %v", err)
}
if usage > limit {
t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
}
// The multi-referenced blob must be gone from disk, from
// source_content, and from BOTH source_metadata rows: references
// are removed together with the blob, never left dangling.
if cache.srcContent.Exists(sharedHash) {
t.Errorf("evicted blob %s still exists on disk", sharedHash)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(sharedHash),
); n != 0 {
t.Errorf("evicted blob %s still has %d source_content rows", sharedHash, n)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(sharedHash),
); n != 0 {
t.Errorf("evicted blob %s still has %d source_metadata references", sharedHash, n)
}
// The JSON metadata sidecars for both referencing paths must be
// removed along with the rows.
for _, path := range []string{"/a.jpg", "/b.jpg"} {
pathHash := HashPath(path + "?")
if cache.srcMetadata.Exists("src.example.com", pathHash) {
t.Errorf("metadata sidecar for %s must be removed with its row", path)
}
}
// The more recently used blob survives fully intact.
if !cache.srcContent.Exists(recentHash) {
t.Errorf("recently used blob %s must survive eviction", recentHash)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(recentHash),
); n != 1 {
t.Errorf("recently used blob %s has %d source_metadata rows, want 1",
recentHash, n)
}
assertNoDanglingReferences(t, cache)
}
func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
content := bytes.Repeat([]byte{0xDF}, 800)
hash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content)
h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content)
if h != hash {
t.Fatalf("identical content produced different hashes: %s vs %s", h, hash)
keys := []imagerequest.SourceKey{
storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content),
storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content),
}
storeEvictionTestVariant(t, cache, testVariantKeyOne,
@@ -499,14 +385,16 @@ func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) {
t.Fatalf("EvictToLimit failed: %v", err)
}
if !cache.srcContent.Exists(hash) {
t.Errorf("blob %s must not be evicted while usage is under the limit", hash)
}
for _, key := range keys {
if !cache.srcContent.Exists(key) {
t.Errorf("source %s must not be evicted while usage is under the limit", key)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(hash),
); n != 2 {
t.Errorf("blob %s has %d source_metadata rows, want 2", hash, n)
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_content WHERE source_key = ?`, string(key),
); n != 1 {
t.Errorf("source %s has %d source_content rows, want 1", key, n)
}
}
if !cache.variants.Exists(testVariantKeyOne) {
@@ -562,13 +450,13 @@ func assertDisabledCacheWritesAreNoOps(
Headers: map[string][]string{},
}
hash, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result)
key, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result)
if err != nil {
t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err)
}
if hash != "" {
t.Errorf("StoreSource on disabled cache returned hash %q, want empty", hash)
if key != "" {
t.Errorf("StoreSource on disabled cache returned key %q, want empty", key)
}
}
@@ -590,14 +478,14 @@ func assertDisabledCacheReadsAlwaysMiss(
t.Error("Lookup on disabled cache must always miss")
}
srcHash, srcType, err := cache.LookupSource(ctx, req)
key, srcType, err := cache.LookupSource(ctx, req)
if err != nil {
t.Fatalf("LookupSource on disabled cache failed: %v", err)
}
if srcHash != "" || srcType != "" {
if key != "" || srcType != "" {
t.Errorf("LookupSource on disabled cache = (%q, %q), want empty",
srcHash, srcType)
key, srcType)
}
}
@@ -618,10 +506,6 @@ func assertDisabledCacheTracksNothing(t *testing.T, cache *Cache) {
if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 {
t.Errorf("disabled cache wrote %d source_content rows, want 0", n)
}
if n := countRows(t, cache, `SELECT COUNT(*) FROM source_metadata`); n != 0 {
t.Errorf("disabled cache wrote %d source_metadata rows, want 0", n)
}
}
// assertDisabledCacheWroteNothingToDisk verifies that a disabled cache
@@ -1064,9 +948,15 @@ func TestStopEvictionInterruptsPassInProgress(t *testing.T) {
}
// TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context
// while the oldest of three source blobs is being evicted, and checks that
// EvictToLimit then returns context.Canceled without evicting the other
// two or logging a warning for either of them.
// once the oldest of three sources is evicted, while the next one waits
// for the database, and checks that EvictToLimit then returns
// context.Canceled, with that source gone and the other two kept, and
// logs no warning. The test database has one connection. When the
// deletion of the oldest source's row commits, a commit hook holds the
// deletion until the test waits for the connection, so the test takes it
// as soon as the deletion is done. The eviction of the next source then
// waits for the connection, and the test cancels the context while it
// waits.
func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
t.Parallel()
@@ -1076,7 +966,7 @@ func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))
hashes := []ContentHash{
keys := []imagerequest.SourceKey{
storeEvictionTestSource(t, cache, "cancel.example.com", "/a.jpg",
bytes.Repeat([]byte{0x61}, 1000)),
storeEvictionTestSource(t, cache, "cancel.example.com", "/b.jpg",
@@ -1087,69 +977,155 @@ func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
base := time.Now().Add(-time.Hour)
for i, hash := range hashes {
setSourceLastAccessed(t, cache, hash, base.Add(time.Duration(i)*time.Minute))
for i, key := range keys {
setSourceLastAccessed(t, cache, key, base.Add(time.Duration(i)*time.Minute))
}
waitsBefore := cache.db.Stats().WaitCount
oldestRowDeleted := make(chan struct{})
var oldestRowDeletedOnce sync.Once
setCommitHook(t, cache, func() int32 {
oldestRowDeletedOnce.Do(func() {
close(oldestRowDeleted)
deadline := time.Now().Add(5 * time.Second)
for cache.db.Stats().WaitCount == waitsBefore && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
})
return 0
})
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
cache.evictSourceBlobTestHook = func(ContentHash) { cancel() }
evicted := make(chan error, 1)
err := cache.EvictToLimit(ctx)
go func() { evicted <- cache.EvictToLimit(ctx) }()
select {
case <-oldestRowDeleted:
case err := <-evicted:
t.Fatalf("EvictToLimit() returned %v without deleting a row", err)
case <-time.After(5 * time.Second):
t.Fatal("EvictToLimit never deleted a row")
}
releaseDatabase := holdDatabase(t, cache)
defer releaseDatabase()
waitForConnectionWaits(t, cache.db, waitsBefore+2)
cancel()
err := <-evicted
t.Logf("EvictToLimit() error = %v", err)
t.Logf("log output: %s", logBuf.String())
// The checks below read the database, so the connection goes back.
releaseDatabase()
setCommitHook(t, cache, nil)
if !errors.Is(err, context.Canceled) {
t.Errorf("EvictToLimit() error = %v, want context.Canceled", err)
}
if cache.srcContent.Exists(hashes[0]) {
t.Errorf("source blob %s, evicted when the context was cancelled, "+
"is still on disk", hashes[0])
}
for _, hash := range hashes[1:] {
if !cache.srcContent.Exists(hash) {
t.Errorf("source blob %s was evicted after the context was cancelled", hash)
}
}
assertOnlyOldestSourceEvicted(t, cache, keys)
if strings.Contains(logBuf.String(), `"level":"WARN"`) {
t.Errorf("EvictToLimit logged a warning after the context was cancelled")
}
}
// setCommitHook makes the SQLite driver call hook each time a write on
// the test database's one connection commits, or stops it when hook is
// nil.
func setCommitHook(t *testing.T, cache *Cache, hook sqlite.CommitHookFn) {
t.Helper()
conn, err := cache.db.Conn(t.Context())
if err != nil {
t.Fatalf("failed to take the database connection: %v", err)
}
defer func() { _ = conn.Close() }()
err = conn.Raw(func(driverConn any) error {
hooks, ok := driverConn.(sqlite.HookRegisterer)
if !ok {
t.Fatalf("the database connection is a %T, which takes no hooks",
driverConn)
}
hooks.RegisterCommitHook(hook)
return nil
})
if err != nil {
t.Fatalf("failed to set the commit hook: %v", err)
}
}
// assertOnlyOldestSourceEvicted checks that the first of keys has neither
// its file nor its row, and that every other one has both.
func assertOnlyOldestSourceEvicted(
t *testing.T, cache *Cache, keys []imagerequest.SourceKey,
) {
t.Helper()
if cache.srcContent.Exists(keys[0]) {
t.Errorf("source %s, evicted when the context was cancelled, "+
"is still on disk", keys[0])
}
for _, key := range keys[1:] {
if !cache.srcContent.Exists(key) {
t.Errorf("source %s was evicted after the context was cancelled", key)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_content WHERE source_key = ?`, string(key),
); n != 1 {
t.Errorf("source %s, kept, has %d source_content rows, want 1", key, n)
}
}
assertNoDanglingReferences(t, cache)
}
// TestStopEvictionReturnsWhenItsContextEnds pauses an eviction pass where
// cancellation cannot reach it, after a source blob's rows are deleted and
// before its file is removed, and checks that StopEviction returns its
// context's error when that context ends instead of waiting for the pass.
// Once the pass goes on, the goroutine exits and no row points at a
// missing file.
// TestStopEvictionReturnsWhenItsContextEnds pauses the startup
// reconciliation pass in the hook called after each of its reads, where
// cancellation cannot reach it, and checks that StopEviction returns an
// error wrapping its context's error when that context ends, instead of
// waiting for the pass. Once the pass goes on, the goroutine exits, and a
// second StopEviction returns nil.
func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1)
cache, _ := newEvictionTestCache(t, 1<<30)
paused := make(chan struct{})
resume := make(chan struct{})
cache.evictSourceBlobTestHook = func(ContentHash) {
close(paused)
<-resume
}
var pauseOnce sync.Once
hash := storeEvictionTestSource(t, cache, "stop.example.com", "/a.jpg",
bytes.Repeat([]byte{0x61}, 1000))
cache.reconciliationReadTestHook = func(int) {
pauseOnce.Do(func() {
close(paused)
<-resume
})
}
cache.StartEviction(time.Hour)
select {
case <-paused:
case <-time.After(5 * time.Second):
t.Fatal("the eviction pass never reached the source blob")
t.Fatal("the reconciliation pass never made a read")
}
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
@@ -1168,12 +1144,6 @@ func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) {
if err != nil {
t.Fatalf("second StopEviction() error = %v, want nil", err)
}
assertNoDanglingReferences(t, cache)
if cache.srcContent.Exists(hash) {
t.Errorf("source blob %s is still on disk after its rows were deleted", hash)
}
}
// TestReconciliationWalksStopOnceCancelled checks that both directory
@@ -1245,131 +1215,3 @@ func TestReconciliationPassLogsNoWarningOnceCancelled(t *testing.T) {
t.Errorf("runReconciliationPass logged a warning with a cancelled context")
}
}
// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
// the exact TOCTOU window between evictSourceBlob's row-deletion
// transaction commit and its content file unlink: a concurrent
// StoreSource for a different source path whose content hashes to the
// same value (real SHA-256 dedup, not a contrived case) must not be
// able to insert a fresh row referencing the file while eviction is
// mid-unlink, and must not lose its own store once eviction has fully
// released the content hash.
func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
ctx := context.Background()
content := bytes.Repeat([]byte{0x55}, 400)
hash := storeEvictionTestSource(t, cache,
"race.example.com", "/first.jpg", content)
proceed := make(chan struct{})
storeAttempted := make(chan struct{})
cache.evictSourceBlobTestHook = func(gotHash ContentHash) {
if gotHash != hash {
t.Errorf("test hook invoked for hash %s, want %s", gotHash, hash)
}
close(storeAttempted)
<-proceed
}
evictDone := make(chan error, 1)
go func() {
evictDone <- cache.evictSourceBlob(ctx, hash)
}()
// Wait until eviction has committed its delete transaction and is
// paused (inside the test hook) immediately before unlinking the
// content file: exactly the window the review flagged.
<-storeAttempted
storeDone := storeIdenticalContentConcurrently(ctx, cache, content)
// The concurrent store must not be able to complete while eviction
// still holds the content hash (i.e. before the file is unlinked):
// if it could, it would insert a row referencing a file about to be
// removed out from under it.
select {
case err := <-storeDone:
t.Fatalf("StoreSource for identical content completed (err=%v) while eviction "+
"still held the content hash open between commit and unlink; the store and "+
"the evict of identical content are not mutually exclusive", err)
case <-time.After(200 * time.Millisecond):
// Expected: the store is blocked behind eviction's exclusion.
}
close(proceed)
err := <-evictDone
if err != nil {
t.Fatalf("evictSourceBlob failed: %v", err)
}
err = <-storeDone
if err != nil {
t.Fatalf("StoreSource failed: %v", err)
}
assertNoDanglingReferences(t, cache)
dupHash, _, err := cache.LookupSource(ctx, &imagerequest.Request{
Source: imagerequest.Source{
Host: "race.example.com",
Path: "/dup.jpg",
},
})
if err != nil {
t.Fatalf("LookupSource failed: %v", err)
}
if dupHash == "" {
t.Fatal("re-stored blob was lost: the store legitimately ran after eviction " +
"released the content hash and must have recreated the file and row")
}
if !cache.srcContent.Exists(dupHash) {
t.Errorf("source_content/source_metadata references %s but its file is missing",
dupHash)
}
}
// storeIdenticalContentConcurrently starts a StoreSource for a second
// source path whose body hashes to the same content hash, returning the
// channel its error is delivered on.
func storeIdenticalContentConcurrently(
ctx context.Context, cache *Cache, content []byte,
) chan error {
storeDone := make(chan error, 1)
go func() {
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: "race.example.com",
Path: "/dup.jpg",
},
Transform: imagerequest.Transform{
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
result := &httpfetcher.FetchResult{
StatusCode: 200,
ContentType: testContentTypeJPEG,
ContentLength: int64(len(content)),
Headers: map[string][]string{
testHeaderContentType: {testContentTypeJPEG},
},
}
_, err := cache.StoreSource(ctx, req, bytes.NewReader(content), result)
storeDone <- err
}()
return storeDone
}