test: add failing test for evictSourceBlob unlink-vs-store TOCTOU
Adds an instrumentation seam (evictSourceBlobTestHook, fired after the row-deletion transaction commits and before the content file is unlinked) and a test that pauses eviction there while a concurrent StoreSource for identical content bytes races it. Currently red: the store completes immediately instead of being excluded, which is exactly the window the review flagged between evictSourceBlob's commit and its unlink.
This commit is contained in:
@@ -667,3 +667,106 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) {
|
||||
t.Errorf("stale accounting row without a file was not dropped (rows=%d)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
|
||||
// the exact TOCTOU window between evictSourceBlob's row-deletion
|
||||
// transaction commit and its content file unlink: a concurrent
|
||||
// StoreSource for a different source path whose content hashes to the
|
||||
// same value (real SHA-256 dedup, not a contrived case) must not be
|
||||
// able to insert a fresh row referencing the file while eviction is
|
||||
// mid-unlink, and must not lose its own store once eviction has fully
|
||||
// released the content hash.
|
||||
func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) {
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := context.Background()
|
||||
|
||||
content := bytes.Repeat([]byte{0x55}, 400)
|
||||
|
||||
hash := storeEvictionTestSource(t, cache, "race.example.com", "/first.jpg", content)
|
||||
|
||||
proceed := make(chan struct{})
|
||||
storeAttempted := make(chan struct{})
|
||||
|
||||
cache.evictSourceBlobTestHook = func(gotHash ContentHash) {
|
||||
if gotHash != hash {
|
||||
t.Errorf("test hook invoked for hash %s, want %s", gotHash, hash)
|
||||
}
|
||||
|
||||
close(storeAttempted)
|
||||
<-proceed
|
||||
}
|
||||
|
||||
evictDone := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
evictDone <- cache.evictSourceBlob(ctx, hash)
|
||||
}()
|
||||
|
||||
// Wait until eviction has committed its delete transaction and is
|
||||
// paused (inside the test hook) immediately before unlinking the
|
||||
// content file: exactly the window the review flagged.
|
||||
<-storeAttempted
|
||||
|
||||
storeDone := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
req := &ImageRequest{
|
||||
SourceHost: "race.example.com",
|
||||
SourcePath: "/dup.jpg",
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
result := &httpfetcher.FetchResult{
|
||||
StatusCode: 200,
|
||||
ContentType: "image/jpeg",
|
||||
ContentLength: int64(len(content)),
|
||||
Headers: map[string][]string{"Content-Type": {"image/jpeg"}},
|
||||
}
|
||||
|
||||
_, err := cache.StoreSource(ctx, req, bytes.NewReader(content), result)
|
||||
storeDone <- err
|
||||
}()
|
||||
|
||||
// The concurrent store must not be able to complete while eviction
|
||||
// still holds the content hash (i.e. before the file is unlinked):
|
||||
// if it could, it would insert a row referencing a file about to be
|
||||
// removed out from under it.
|
||||
select {
|
||||
case err := <-storeDone:
|
||||
t.Fatalf("StoreSource for identical content completed (err=%v) while eviction "+
|
||||
"still held the content hash open between commit and unlink; the store and "+
|
||||
"the evict of identical content are not mutually exclusive", err)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
// Expected: the store is blocked behind eviction's exclusion.
|
||||
}
|
||||
|
||||
close(proceed)
|
||||
|
||||
if err := <-evictDone; err != nil {
|
||||
t.Fatalf("evictSourceBlob failed: %v", err)
|
||||
}
|
||||
|
||||
if err := <-storeDone; err != nil {
|
||||
t.Fatalf("StoreSource failed: %v", err)
|
||||
}
|
||||
|
||||
assertNoDanglingReferences(t, cache)
|
||||
|
||||
dupHash, _, err := cache.LookupSource(ctx, &ImageRequest{
|
||||
SourceHost: "race.example.com",
|
||||
SourcePath: "/dup.jpg",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("LookupSource failed: %v", err)
|
||||
}
|
||||
|
||||
if dupHash == "" {
|
||||
t.Fatal("re-stored blob was lost: the store legitimately ran after eviction " +
|
||||
"released the content hash and must have recreated the file and row")
|
||||
}
|
||||
|
||||
if !cache.srcContent.Exists(dupHash) {
|
||||
t.Errorf("source_content/source_metadata references %s but its file is missing", dupHash)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user