sneak/pixa
1
0
Fork 0
Code Issues 19 Pull Requests 3 Actions 4 Packages Projects Releases Wiki Activity

Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
Details

Browse Source 
A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone. Two source URLs serving the same
bytes store them twice. Reconciliation still drops a source row whose
file is missing, as TestReconciliationReadsAPageAtATime counts those
reads. The tests of the removed sharing, lock and hook go with them.

Model: opus-5-5
This commit is contained in:
sneak
2026-10-21 13:05:11 +00:00
parent 00d8ba26c5
commit d571045d01
22 changed files with 435 additions and 1597 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imgcache/eviction_internal_test.go
+10 -406
View File
@@ -83,10 +83,10 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) {
}
// storeEvictionTestSource stores content as a fetched source for
// host/path and returns the resulting content hash.
// host/path and returns the resulting source key.
func storeEvictionTestSource(
t *testing.T, cache *Cache, host, path string, content []byte,
) ContentHash {
) imagerequest.SourceKey {
t.Helper()
req := &imagerequest.Request{
@@ -162,12 +162,12 @@ func setVariantLastAccessed(
// setSourceLastAccessed backdates the last access time of a tracked
// source content blob.
func setSourceLastAccessed(
t *testing.T, cache *Cache, hash ContentHash, when time.Time,
t *testing.T, cache *Cache, hash imagerequest.SourceKey, when time.Time,
) {
t.Helper()
res, err := cache.db.ExecContext(t.Context(),
`UPDATE source_content SET last_accessed_at = ? WHERE content_hash = ?`,
`UPDATE source_content SET last_accessed_at = ? WHERE source_key = ?`,
when.UTC().Format(sqliteTimestampFormat), string(hash),
)
if err != nil {
@@ -205,11 +205,10 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) {
t.Helper()
rows, err := cache.db.QueryContext(t.Context(),
`SELECT content_hash FROM source_metadata
WHERE content_hash IS NOT NULL AND content_hash != ''`,
`SELECT source_key FROM source_content`,
)
if err != nil {
t.Fatalf("failed to query source_metadata: %v", err)
t.Fatalf("failed to query source_content: %v", err)
}
defer func() { _ = rows.Close() }()
@@ -219,18 +218,18 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) {
err := rows.Scan(&hash)
if err != nil {
t.Fatalf("failed to scan content_hash: %v", err)
t.Fatalf("failed to scan source_key: %v", err)
}
if !cache.srcContent.Exists(ContentHash(hash)) {
t.Errorf("source_metadata references content %s but the file is missing",
if !cache.srcContent.Exists(imagerequest.SourceKey(hash)) {
t.Errorf("source_content references content %s but the file is missing",
hash)
}
}
err = rows.Err()
if err != nil {
t.Fatalf("source_metadata iteration failed: %v", err)
t.Fatalf("source_content iteration failed: %v", err)
}
variantRows, err := cache.db.QueryContext(t.Context(),
@@ -313,33 +312,6 @@ func TestUsageBytesAccountsSourceAndVariantBytes(t *testing.T) {
}
}
func TestUsageBytesCountsMultiReferencedBlobOnce(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
content := bytes.Repeat([]byte{0xCC}, 1200)
hashOne := storeEvictionTestSource(t, cache,
"src.example.com", "/one.jpg", content)
hashTwo := storeEvictionTestSource(t, cache,
"src.example.com", "/two.jpg", content)
if hashOne != hashTwo {
t.Fatalf("identical content produced different hashes: %s vs %s",
hashOne, hashTwo)
}
usage, err := cache.UsageBytes(context.Background())
if err != nil {
t.Fatalf("UsageBytes failed: %v", err)
}
if usage != 1200 {
t.Errorf("UsageBytes = %d, want 1200 (deduplicated blob counted once)", usage)
}
}
func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) {
t.Parallel()
@@ -393,129 +365,6 @@ func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) {
assertNoDanglingReferences(t, cache)
}
func TestEvictionRemovesMultiReferencedBlobTogetherWithAllReferences(t *testing.T) {
t.Parallel()
const limit = 1000
cache, _ := newEvictionTestCache(t, limit)
now := time.Now()
// One 800-byte blob referenced by two source paths.
sharedContent := bytes.Repeat([]byte{0xDD}, 800)
sharedHash := storeEvictionTestSource(t, cache,
"src.example.com", "/a.jpg", sharedContent)
h := storeEvictionTestSource(t, cache,
"src.example.com", "/b.jpg", sharedContent)
if h != sharedHash {
t.Fatalf("identical content produced different hashes: %s vs %s",
h, sharedHash)
}
// A newer 600-byte blob referenced by one source path.
recentHash := storeEvictionTestSource(t, cache, "src.example.com", "/c.jpg",
bytes.Repeat([]byte{0xEE}, 600))
setSourceLastAccessed(t, cache, sharedHash, now.Add(-2*time.Hour))
setSourceLastAccessed(t, cache, recentHash, now.Add(-time.Minute))
err := cache.EvictToLimit(context.Background())
if err != nil {
t.Fatalf("EvictToLimit failed: %v", err)
}
usage, err := cache.UsageBytes(context.Background())
if err != nil {
t.Fatalf("UsageBytes failed: %v", err)
}
if usage > limit {
t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
}
// The multi-referenced blob must be gone from disk, from
// source_content, and from BOTH source_metadata rows: references
// are removed together with the blob, never left dangling.
if cache.srcContent.Exists(sharedHash) {
t.Errorf("evicted blob %s still exists on disk", sharedHash)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(sharedHash),
); n != 0 {
t.Errorf("evicted blob %s still has %d source_content rows", sharedHash, n)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(sharedHash),
); n != 0 {
t.Errorf("evicted blob %s still has %d source_metadata references", sharedHash, n)
}
// The JSON metadata sidecars for both referencing paths must be
// removed along with the rows.
for _, path := range []string{"/a.jpg", "/b.jpg"} {
pathHash := HashPath(path + "?")
if cache.srcMetadata.Exists("src.example.com", pathHash) {
t.Errorf("metadata sidecar for %s must be removed with its row", path)
}
}
// The more recently used blob survives fully intact.
if !cache.srcContent.Exists(recentHash) {
t.Errorf("recently used blob %s must survive eviction", recentHash)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(recentHash),
); n != 1 {
t.Errorf("recently used blob %s has %d source_metadata rows, want 1",
recentHash, n)
}
assertNoDanglingReferences(t, cache)
}
func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
content := bytes.Repeat([]byte{0xDF}, 800)
hash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content)
h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content)
if h != hash {
t.Fatalf("identical content produced different hashes: %s vs %s", h, hash)
}
storeEvictionTestVariant(t, cache, testVariantKeyOne,
bytes.Repeat([]byte{0xE0}, 500))
err := cache.EvictToLimit(context.Background())
if err != nil {
t.Fatalf("EvictToLimit failed: %v", err)
}
if !cache.srcContent.Exists(hash) {
t.Errorf("blob %s must not be evicted while usage is under the limit", hash)
}
if n := countRows(t, cache,
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(hash),
); n != 2 {
t.Errorf("blob %s has %d source_metadata rows, want 2", hash, n)
}
if !cache.variants.Exists(testVariantKeyOne) {
t.Error("variant must not be evicted while usage is under the limit")
}
assertNoDanglingReferences(t, cache)
}
func TestZeroMaxBytesDisablesDiskCache(t *testing.T) {
t.Parallel()
@@ -618,10 +467,6 @@ func assertDisabledCacheTracksNothing(t *testing.T, cache *Cache) {
if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 {
t.Errorf("disabled cache wrote %d source_content rows, want 0", n)
}
if n := countRows(t, cache, `SELECT COUNT(*) FROM source_metadata`); n != 0 {
t.Errorf("disabled cache wrote %d source_metadata rows, want 0", n)
}
}
// assertDisabledCacheWroteNothingToDisk verifies that a disabled cache
@@ -1063,119 +908,6 @@ func TestStopEvictionInterruptsPassInProgress(t *testing.T) {
}
}
// TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context
// while the oldest of three source blobs is being evicted, and checks that
// EvictToLimit then returns context.Canceled without evicting the other
// two or logging a warning for either of them.
func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1)
var logBuf bytes.Buffer
cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))
hashes := []ContentHash{
storeEvictionTestSource(t, cache, "cancel.example.com", "/a.jpg",
bytes.Repeat([]byte{0x61}, 1000)),
storeEvictionTestSource(t, cache, "cancel.example.com", "/b.jpg",
bytes.Repeat([]byte{0x62}, 1000)),
storeEvictionTestSource(t, cache, "cancel.example.com", "/c.jpg",
bytes.Repeat([]byte{0x63}, 1000)),
}
base := time.Now().Add(-time.Hour)
for i, hash := range hashes {
setSourceLastAccessed(t, cache, hash, base.Add(time.Duration(i)*time.Minute))
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
cache.evictSourceBlobTestHook = func(ContentHash) { cancel() }
err := cache.EvictToLimit(ctx)
t.Logf("EvictToLimit() error = %v", err)
t.Logf("log output: %s", logBuf.String())
if !errors.Is(err, context.Canceled) {
t.Errorf("EvictToLimit() error = %v, want context.Canceled", err)
}
if cache.srcContent.Exists(hashes[0]) {
t.Errorf("source blob %s, evicted when the context was cancelled, "+
"is still on disk", hashes[0])
}
for _, hash := range hashes[1:] {
if !cache.srcContent.Exists(hash) {
t.Errorf("source blob %s was evicted after the context was cancelled", hash)
}
}
if strings.Contains(logBuf.String(), `"level":"WARN"`) {
t.Errorf("EvictToLimit logged a warning after the context was cancelled")
}
assertNoDanglingReferences(t, cache)
}
// TestStopEvictionReturnsWhenItsContextEnds pauses an eviction pass where
// cancellation cannot reach it, after a source blob's rows are deleted and
// before its file is removed, and checks that StopEviction returns its
// context's error when that context ends instead of waiting for the pass.
// Once the pass goes on, the goroutine exits and no row points at a
// missing file.
func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1)
paused := make(chan struct{})
resume := make(chan struct{})
cache.evictSourceBlobTestHook = func(ContentHash) {
close(paused)
<-resume
}
hash := storeEvictionTestSource(t, cache, "stop.example.com", "/a.jpg",
bytes.Repeat([]byte{0x61}, 1000))
cache.StartEviction(time.Hour)
select {
case <-paused:
case <-time.After(5 * time.Second):
t.Fatal("the eviction pass never reached the source blob")
}
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
defer cancel()
err := cache.StopEviction(ctx)
t.Logf("StopEviction() error = %v", err)
if !errors.Is(err, context.DeadlineExceeded) {
t.Errorf("StopEviction() error = %v, want context.DeadlineExceeded", err)
}
close(resume)
err = cache.StopEviction(t.Context())
if err != nil {
t.Fatalf("second StopEviction() error = %v, want nil", err)
}
assertNoDanglingReferences(t, cache)
if cache.srcContent.Exists(hash) {
t.Errorf("source blob %s is still on disk after its rows were deleted", hash)
}
}
// TestReconciliationWalksStopOnceCancelled checks that both directory
// walks of a reconciliation pass return the context's error once it is
// cancelled, leaving in place a stale temp file they would otherwise
@@ -1245,131 +977,3 @@ func TestReconciliationPassLogsNoWarningOnceCancelled(t *testing.T) {
t.Errorf("runReconciliationPass logged a warning with a cancelled context")
}
}
// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
// the exact TOCTOU window between evictSourceBlob's row-deletion
// transaction commit and its content file unlink: a concurrent
// StoreSource for a different source path whose content hashes to the
// same value (real SHA-256 dedup, not a contrived case) must not be
// able to insert a fresh row referencing the file while eviction is
// mid-unlink, and must not lose its own store once eviction has fully
// released the content hash.
func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) {
t.Parallel()
cache, _ := newEvictionTestCache(t, 1<<30)
ctx := context.Background()
content := bytes.Repeat([]byte{0x55}, 400)
hash := storeEvictionTestSource(t, cache,
"race.example.com", "/first.jpg", content)
proceed := make(chan struct{})
storeAttempted := make(chan struct{})
cache.evictSourceBlobTestHook = func(gotHash ContentHash) {
if gotHash != hash {
t.Errorf("test hook invoked for hash %s, want %s", gotHash, hash)
}
close(storeAttempted)
<-proceed
}
evictDone := make(chan error, 1)
go func() {
evictDone <- cache.evictSourceBlob(ctx, hash)
}()
// Wait until eviction has committed its delete transaction and is
// paused (inside the test hook) immediately before unlinking the
// content file: exactly the window the review flagged.
<-storeAttempted
storeDone := storeIdenticalContentConcurrently(ctx, cache, content)
// The concurrent store must not be able to complete while eviction
// still holds the content hash (i.e. before the file is unlinked):
// if it could, it would insert a row referencing a file about to be
// removed out from under it.
select {
case err := <-storeDone:
t.Fatalf("StoreSource for identical content completed (err=%v) while eviction "+
"still held the content hash open between commit and unlink; the store and "+
"the evict of identical content are not mutually exclusive", err)
case <-time.After(200 * time.Millisecond):
// Expected: the store is blocked behind eviction's exclusion.
}
close(proceed)
err := <-evictDone
if err != nil {
t.Fatalf("evictSourceBlob failed: %v", err)
}
err = <-storeDone
if err != nil {
t.Fatalf("StoreSource failed: %v", err)
}
assertNoDanglingReferences(t, cache)
dupHash, _, err := cache.LookupSource(ctx, &imagerequest.Request{
Source: imagerequest.Source{
Host: "race.example.com",
Path: "/dup.jpg",
},
})
if err != nil {
t.Fatalf("LookupSource failed: %v", err)
}
if dupHash == "" {
t.Fatal("re-stored blob was lost: the store legitimately ran after eviction " +
"released the content hash and must have recreated the file and row")
}
if !cache.srcContent.Exists(dupHash) {
t.Errorf("source_content/source_metadata references %s but its file is missing",
dupHash)
}
}
// storeIdenticalContentConcurrently starts a StoreSource for a second
// source path whose body hashes to the same content hash, returning the
// channel its error is delivered on.
func storeIdenticalContentConcurrently(
ctx context.Context, cache *Cache, content []byte,
) chan error {
storeDone := make(chan error, 1)
go func() {
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: "race.example.com",
Path: "/dup.jpg",
},
Transform: imagerequest.Transform{
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
result := &httpfetcher.FetchResult{
StatusCode: 200,
ContentType: testContentTypeJPEG,
ContentLength: int64(len(content)),
Headers: map[string][]string{
testHeaderContentType: {testContentTypeJPEG},
},
}
_, err := cache.StoreSource(ctx, req, bytes.NewReader(content), result)
storeDone <- err
}()
return storeDone
}