pixa/internal/imgcache/eviction_test.go

package imgcache

import (
	"bytes"
	"context"
	"database/sql"
	"io/fs"
	"os"
	"path/filepath"
	"testing"
	"time"

	_ "modernc.org/sqlite"
	"sneak.berlin/go/pixa/internal/database"
	"sneak.berlin/go/pixa/internal/httpfetcher"
)

// sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP
// produces, so injected timestamps compare correctly against ones the
// implementation writes.
const sqliteTimestampFormat = "2006-01-02 15:04:05"

// evictionTestDB creates an in-memory SQLite database with the real
// production schema, limited to a single connection so the background
// eviction goroutine shares the same in-memory database as the test.
func evictionTestDB(t *testing.T) *sql.DB {
	t.Helper()

	db, err := sql.Open("sqlite", ":memory:")
	if err != nil {
		t.Fatalf("failed to open test db: %v", err)
	}

	db.SetMaxOpenConns(1)

	if err := database.ApplyMigrations(context.Background(), db, nil); err != nil {
		t.Fatalf("failed to apply migrations: %v", err)
	}

	t.Cleanup(func() { _ = db.Close() })

	return db
}

// newEvictionTestCache creates a Cache backed by a temp directory and
// an in-memory database, with the given size limit.
func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) {
	t.Helper()

	tmpDir := t.TempDir()
	db := evictionTestDB(t)

	// maxBytes zero mirrors the production mapping of
	// cache_max_bytes: 0 (handlers sets DisableDiskCache); at the
	// CacheConfig layer itself a zero MaxBytes means "no limit" for
	// backwards compatibility with existing fixtures.
	cache, err := NewCache(db, CacheConfig{
		StateDir:         tmpDir,
		CacheTTL:         time.Hour,
		NegativeTTL:      5 * time.Minute,
		MaxBytes:         maxBytes,
		DisableDiskCache: maxBytes == 0,
	})
	if err != nil {
		t.Fatalf("failed to create cache: %v", err)
	}

	return cache, tmpDir
}

// storeEvictionTestSource stores content as a fetched source for
// host/path and returns the resulting content hash.
func storeEvictionTestSource(
	t *testing.T, cache *Cache, host, path string, content []byte,
) ContentHash {
	t.Helper()

	req := &ImageRequest{
		SourceHost: host,
		SourcePath: path,
		Format:     FormatJPEG,
		Quality:    85,
		FitMode:    FitCover,
	}

	result := &httpfetcher.FetchResult{
		StatusCode:    200,
		ContentType:   "image/jpeg",
		ContentLength: int64(len(content)),
		Headers:       map[string][]string{"Content-Type": {"image/jpeg"}},
	}

	hash, err := cache.StoreSource(context.Background(), req, bytes.NewReader(content), result)
	if err != nil {
		t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err)
	}

	return hash
}

// storeEvictionTestVariant stores content as a processed variant under
// the given cache key.
func storeEvictionTestVariant(t *testing.T, cache *Cache, key VariantKey, content []byte) {
	t.Helper()

	if err := cache.StoreVariant(key, bytes.NewReader(content), "image/webp"); err != nil {
		t.Fatalf("StoreVariant(%s) failed: %v", key, err)
	}
}

// setVariantLastAccessed backdates the last access time of a tracked
// variant, to make LRU ordering deterministic in tests.
func setVariantLastAccessed(t *testing.T, cache *Cache, key VariantKey, when time.Time) {
	t.Helper()

	res, err := cache.db.Exec(
		`UPDATE variant_content SET last_accessed_at = ? WHERE cache_key = ?`,
		when.UTC().Format(sqliteTimestampFormat), string(key),
	)
	if err != nil {
		t.Fatalf("failed to set variant last_accessed_at: %v", err)
	}

	affected, err := res.RowsAffected()
	if err != nil {
		t.Fatalf("failed to read affected rows: %v", err)
	}

	if affected != 1 {
		t.Fatalf("variant %s has no accounting row (affected=%d); "+
			"stores must track variants in the database", key, affected)
	}
}

// setSourceLastAccessed backdates the last access time of a tracked
// source content blob.
func setSourceLastAccessed(t *testing.T, cache *Cache, hash ContentHash, when time.Time) {
	t.Helper()

	res, err := cache.db.Exec(
		`UPDATE source_content SET last_accessed_at = ? WHERE content_hash = ?`,
		when.UTC().Format(sqliteTimestampFormat), string(hash),
	)
	if err != nil {
		t.Fatalf("failed to set source last_accessed_at: %v", err)
	}

	affected, err := res.RowsAffected()
	if err != nil {
		t.Fatalf("failed to read affected rows: %v", err)
	}

	if affected != 1 {
		t.Fatalf("source %s has no accounting row (affected=%d)", hash, affected)
	}
}

// countRows returns the number of rows the given query yields.
func countRows(t *testing.T, cache *Cache, query string, args ...interface{}) int {
	t.Helper()

	var n int
	if err := cache.db.QueryRow(query, args...).Scan(&n); err != nil {
		t.Fatalf("count query %q failed: %v", query, err)
	}

	return n
}

// assertNoDanglingReferences verifies the core eviction invariant:
// every database row that references cache content on disk points at a
// file that actually exists.
func assertNoDanglingReferences(t *testing.T, cache *Cache) {
	t.Helper()

	rows, err := cache.db.Query(
		`SELECT content_hash FROM source_metadata
		 WHERE content_hash IS NOT NULL AND content_hash != ''`,
	)
	if err != nil {
		t.Fatalf("failed to query source_metadata: %v", err)
	}

	defer func() { _ = rows.Close() }()

	for rows.Next() {
		var hash string
		if err := rows.Scan(&hash); err != nil {
			t.Fatalf("failed to scan content_hash: %v", err)
		}

		if !cache.srcContent.Exists(ContentHash(hash)) {
			t.Errorf("source_metadata references content %s but the file is missing", hash)
		}
	}

	if err := rows.Err(); err != nil {
		t.Fatalf("source_metadata iteration failed: %v", err)
	}

	variantRows, err := cache.db.Query(`SELECT cache_key FROM variant_content`)
	if err != nil {
		t.Fatalf("failed to query variant_content: %v", err)
	}

	defer func() { _ = variantRows.Close() }()

	for variantRows.Next() {
		var key string
		if err := variantRows.Scan(&key); err != nil {
			t.Fatalf("failed to scan cache_key: %v", err)
		}

		if !cache.variants.Exists(VariantKey(key)) {
			t.Errorf("variant_content references key %s but the file is missing", key)
		}
	}

	if err := variantRows.Err(); err != nil {
		t.Fatalf("variant_content iteration failed: %v", err)
	}
}

// waitForUsageAtOrBelow polls UsageBytes until it reaches limit or the
// timeout expires, returning the last observed usage.
func waitForUsageAtOrBelow(t *testing.T, cache *Cache, limit int64, timeout time.Duration) int64 {
	t.Helper()

	deadline := time.Now().Add(timeout)

	var usage int64

	for time.Now().Before(deadline) {
		var err error

		usage, err = cache.UsageBytes(context.Background())
		if err != nil {
			t.Fatalf("UsageBytes failed: %v", err)
		}

		if usage <= limit {
			return usage
		}

		time.Sleep(25 * time.Millisecond)
	}

	return usage
}

func TestUsageBytesAccountsSourceAndVariantBytes(t *testing.T) {
	cache, _ := newEvictionTestCache(t, 1<<30)

	storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg",
		bytes.Repeat([]byte{0xAA}, 1000))
	storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg",
		bytes.Repeat([]byte{0xAB}, 2000))
	storeEvictionTestVariant(t, cache, "aabbccdd0001", bytes.Repeat([]byte{0xAC}, 500))
	storeEvictionTestVariant(t, cache, "aabbccdd0002", bytes.Repeat([]byte{0xAD}, 250))

	usage, err := cache.UsageBytes(context.Background())
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage != 3750 {
		t.Errorf("UsageBytes = %d, want 3750 (1000+2000+500+250)", usage)
	}
}

func TestUsageBytesCountsMultiReferencedBlobOnce(t *testing.T) {
	cache, _ := newEvictionTestCache(t, 1<<30)

	content := bytes.Repeat([]byte{0xCC}, 1200)

	hashOne := storeEvictionTestSource(t, cache, "src.example.com", "/one.jpg", content)
	hashTwo := storeEvictionTestSource(t, cache, "src.example.com", "/two.jpg", content)

	if hashOne != hashTwo {
		t.Fatalf("identical content produced different hashes: %s vs %s", hashOne, hashTwo)
	}

	usage, err := cache.UsageBytes(context.Background())
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage != 1200 {
		t.Errorf("UsageBytes = %d, want 1200 (deduplicated blob counted once)", usage)
	}
}

func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) {
	const limit = 3000

	cache, _ := newEvictionTestCache(t, limit)

	now := time.Now()
	keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003", "aabbccdd0004"}
	fills := []byte{0x01, 0x02, 0x03, 0x04}
	ages := []time.Duration{4 * time.Hour, 3 * time.Hour, 2 * time.Hour, 1 * time.Hour}

	for i, key := range keys {
		storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{fills[i]}, 1000))
		setVariantLastAccessed(t, cache, key, now.Add(-ages[i]))
	}

	if err := cache.EvictToLimit(context.Background()); err != nil {
		t.Fatalf("EvictToLimit failed: %v", err)
	}

	usage, err := cache.UsageBytes(context.Background())
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage > limit {
		t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
	}

	if cache.variants.Exists(keys[0]) {
		t.Errorf("least recently used variant %s must be evicted", keys[0])
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, string(keys[0]),
	); n != 0 {
		t.Errorf("evicted variant %s still has %d accounting rows", keys[0], n)
	}

	for _, key := range keys[1:] {
		if !cache.variants.Exists(key) {
			t.Errorf("more recently used variant %s must survive eviction", key)
		}
	}

	assertNoDanglingReferences(t, cache)
}

func TestEvictionRemovesMultiReferencedBlobTogetherWithAllReferences(t *testing.T) {
	const limit = 1000

	cache, _ := newEvictionTestCache(t, limit)

	now := time.Now()

	// One 800-byte blob referenced by two source paths.
	sharedContent := bytes.Repeat([]byte{0xDD}, 800)
	sharedHash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", sharedContent)

	if h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", sharedContent); h != sharedHash {
		t.Fatalf("identical content produced different hashes: %s vs %s", h, sharedHash)
	}

	// A newer 600-byte blob referenced by one source path.
	recentHash := storeEvictionTestSource(t, cache, "src.example.com", "/c.jpg",
		bytes.Repeat([]byte{0xEE}, 600))

	setSourceLastAccessed(t, cache, sharedHash, now.Add(-2*time.Hour))
	setSourceLastAccessed(t, cache, recentHash, now.Add(-time.Minute))

	if err := cache.EvictToLimit(context.Background()); err != nil {
		t.Fatalf("EvictToLimit failed: %v", err)
	}

	usage, err := cache.UsageBytes(context.Background())
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage > limit {
		t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
	}

	// The multi-referenced blob must be gone from disk, from
	// source_content, and from BOTH source_metadata rows: references
	// are removed together with the blob, never left dangling.
	if cache.srcContent.Exists(sharedHash) {
		t.Errorf("evicted blob %s still exists on disk", sharedHash)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(sharedHash),
	); n != 0 {
		t.Errorf("evicted blob %s still has %d source_content rows", sharedHash, n)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(sharedHash),
	); n != 0 {
		t.Errorf("evicted blob %s still has %d source_metadata references", sharedHash, n)
	}

	// The JSON metadata sidecars for both referencing paths must be
	// removed along with the rows.
	for _, path := range []string{"/a.jpg", "/b.jpg"} {
		pathHash := HashPath(path + "?")
		if cache.srcMetadata.Exists("src.example.com", pathHash) {
			t.Errorf("metadata sidecar for %s must be removed with its row", path)
		}
	}

	// The more recently used blob survives fully intact.
	if !cache.srcContent.Exists(recentHash) {
		t.Errorf("recently used blob %s must survive eviction", recentHash)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(recentHash),
	); n != 1 {
		t.Errorf("recently used blob %s has %d source_metadata rows, want 1", recentHash, n)
	}

	assertNoDanglingReferences(t, cache)
}

func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) {
	cache, _ := newEvictionTestCache(t, 1<<30)

	content := bytes.Repeat([]byte{0xDF}, 800)
	hash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content)

	if h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content); h != hash {
		t.Fatalf("identical content produced different hashes: %s vs %s", h, hash)
	}

	storeEvictionTestVariant(t, cache, "aabbccdd0001", bytes.Repeat([]byte{0xE0}, 500))

	if err := cache.EvictToLimit(context.Background()); err != nil {
		t.Fatalf("EvictToLimit failed: %v", err)
	}

	if !cache.srcContent.Exists(hash) {
		t.Errorf("blob %s must not be evicted while usage is under the limit", hash)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(hash),
	); n != 2 {
		t.Errorf("blob %s has %d source_metadata rows, want 2", hash, n)
	}

	if !cache.variants.Exists("aabbccdd0001") {
		t.Error("variant must not be evicted while usage is under the limit")
	}

	assertNoDanglingReferences(t, cache)
}

func TestZeroMaxBytesDisablesDiskCache(t *testing.T) {
	cache, tmpDir := newEvictionTestCache(t, 0)
	ctx := context.Background()

	req := &ImageRequest{
		SourceHost: "src.example.com",
		SourcePath: "/a.jpg",
		Format:     FormatJPEG,
		Quality:    85,
		FitMode:    FitCover,
	}

	// Writes are no-ops that report success.
	if err := cache.StoreVariant(CacheKey(req), bytes.NewReader([]byte("data")), "image/webp"); err != nil {
		t.Fatalf("StoreVariant on disabled cache must be a no-op, got error: %v", err)
	}

	result := &httpfetcher.FetchResult{
		StatusCode:    200,
		ContentType:   "image/jpeg",
		ContentLength: 4,
		Headers:       map[string][]string{},
	}

	hash, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result)
	if err != nil {
		t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err)
	}

	if hash != "" {
		t.Errorf("StoreSource on disabled cache returned hash %q, want empty", hash)
	}

	// Reads always miss.
	lookup, err := cache.Lookup(ctx, req)
	if err != nil {
		t.Fatalf("Lookup on disabled cache failed: %v", err)
	}

	if lookup.Hit {
		t.Error("Lookup on disabled cache must always miss")
	}

	srcHash, srcType, err := cache.LookupSource(ctx, req)
	if err != nil {
		t.Fatalf("LookupSource on disabled cache failed: %v", err)
	}

	if srcHash != "" || srcType != "" {
		t.Errorf("LookupSource on disabled cache = (%q, %q), want empty", srcHash, srcType)
	}

	// Nothing is tracked and nothing is written to disk.
	usage, err := cache.UsageBytes(ctx)
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage != 0 {
		t.Errorf("UsageBytes on disabled cache = %d, want 0", usage)
	}

	if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 {
		t.Errorf("disabled cache wrote %d source_content rows, want 0", n)
	}

	if n := countRows(t, cache, `SELECT COUNT(*) FROM source_metadata`); n != 0 {
		t.Errorf("disabled cache wrote %d source_metadata rows, want 0", n)
	}

	if _, err := os.Stat(filepath.Join(tmpDir, "cache")); !os.IsNotExist(err) {
		t.Errorf("disabled cache must not create the cache directory tree (stat err=%v)", err)
	}

	var foundFiles []string

	walkErr := filepath.WalkDir(tmpDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}

		if !d.IsDir() {
			foundFiles = append(foundFiles, path)
		}

		return nil
	})
	if walkErr != nil {
		t.Fatalf("failed to walk state dir: %v", walkErr)
	}

	if len(foundFiles) != 0 {
		t.Errorf("disabled cache wrote files to disk: %v", foundFiles)
	}
}

func TestEvictionRunsUnderWritePressure(t *testing.T) {
	const limit = 1500

	cache, _ := newEvictionTestCache(t, limit)

	// An interval far longer than the test ensures only write
	// pressure can trigger eviction here.
	cache.StartEviction(time.Hour)
	defer cache.StopEviction()

	keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003"}
	fills := []byte{0x11, 0x12, 0x13}

	for i, key := range keys {
		storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{fills[i]}, 1000))
	}

	usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second)
	if usage > limit {
		t.Errorf("write pressure did not trigger eviction: usage = %d, want <= %d",
			usage, limit)
	}

	assertNoDanglingReferences(t, cache)
}

func TestEvictionRunsOnPeriodicSchedule(t *testing.T) {
	const limit = 1500

	cache, _ := newEvictionTestCache(t, limit)

	// Start the evictor while the cache is empty, then create tracked
	// over-limit state WITHOUT going through the store methods, so no
	// write-pressure notification fires and only the periodic ticker
	// can trigger eviction.
	cache.StartEviction(100 * time.Millisecond)
	defer cache.StopEviction()

	keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003"}
	fills := []byte{0x21, 0x22, 0x23}

	for i, key := range keys {
		content := bytes.Repeat([]byte{fills[i]}, 1000)

		if _, err := cache.variants.Store(key, bytes.NewReader(content), "image/webp"); err != nil {
			t.Fatalf("failed to store variant file: %v", err)
		}

		if _, err := cache.db.Exec(
			`INSERT INTO variant_content (cache_key, size_bytes, content_type)
			 VALUES (?, ?, ?)`,
			string(key), len(content), "image/webp",
		); err != nil {
			t.Fatalf("failed to insert variant accounting row: %v", err)
		}
	}

	usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second)
	if usage > limit {
		t.Errorf("periodic schedule did not trigger eviction: usage = %d, want <= %d",
			usage, limit)
	}

	assertNoDanglingReferences(t, cache)
}

func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) {
	cache, _ := newEvictionTestCache(t, 1<<30)

	// An untracked variant file on disk (e.g. written before this
	// feature existed) must be adopted into the accounting.
	untracked := bytes.Repeat([]byte{0x31}, 1000)
	if _, err := cache.variants.Store("aabbccdd0001", bytes.NewReader(untracked), "image/webp"); err != nil {
		t.Fatalf("failed to store untracked variant file: %v", err)
	}

	// An accounting row whose file is missing must be dropped.
	if _, err := cache.db.Exec(
		`INSERT INTO variant_content (cache_key, size_bytes, content_type)
		 VALUES (?, ?, ?)`,
		"deadbeef0001", 700, "image/webp",
	); err != nil {
		t.Fatalf("failed to insert stale variant accounting row: %v", err)
	}

	cache.StartEviction(time.Hour)
	defer cache.StopEviction()

	deadline := time.Now().Add(5 * time.Second)

	var usage int64

	for time.Now().Before(deadline) {
		var err error

		usage, err = cache.UsageBytes(context.Background())
		if err != nil {
			t.Fatalf("UsageBytes failed: %v", err)
		}

		if usage == 1000 {
			break
		}

		time.Sleep(25 * time.Millisecond)
	}

	if usage != 1000 {
		t.Errorf("usage after reconciliation = %d, want 1000 "+
			"(untracked file adopted, stale row dropped)", usage)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "aabbccdd0001",
	); n != 1 {
		t.Errorf("untracked variant file was not adopted into accounting (rows=%d)", n)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "deadbeef0001",
	); n != 0 {
		t.Errorf("stale accounting row without a file was not dropped (rows=%d)", n)
	}
}

// TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves
// reconciliation is not a one-shot startup-only pass: it must also run
// on the periodic ticker, so a variant file that lands on disk with no
// accounting row well after startup (e.g. because StoreVariant's
// best-effort accounting insert failed under transient contention, or
// any other cause of an untracked file appearing during steady-state
// operation) is still adopted into accounting eventually, rather than
// staying invisible to UsageBytes/EvictToLimit until the next process
// restart.
func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
	cache, _ := newEvictionTestCache(t, 1<<30)

	const interval = 100 * time.Millisecond

	cache.StartEviction(interval)
	defer cache.StopEviction()

	// Let startup reconciliation run and settle on an empty cache
	// before introducing the untracked file, so the adoption we assert
	// below can only be the work of a later, periodic pass.
	time.Sleep(3 * interval)

	// Simulate a variant whose accounting insert failed after the
	// process was already running and serving requests: the content
	// file is written directly, bypassing StoreVariant's (and thus its
	// accounting insert) entirely, exactly as would happen if that
	// insert had failed and only the file write had succeeded.
	untracked := bytes.Repeat([]byte{0x41}, 900)
	if _, err := cache.variants.Store("aabbccdd0099", bytes.NewReader(untracked), "image/webp"); err != nil {
		t.Fatalf("failed to store untracked variant file: %v", err)
	}

	deadline := time.Now().Add(5 * time.Second)

	var usage int64

	for time.Now().Before(deadline) {
		var err error

		usage, err = cache.UsageBytes(context.Background())
		if err != nil {
			t.Fatalf("UsageBytes failed: %v", err)
		}

		if usage == 900 {
			break
		}

		time.Sleep(25 * time.Millisecond)
	}

	if usage != 900 {
		t.Errorf("usage after periodic reconciliation = %d, want 900 "+
			"(a file that appeared after startup reconciliation already ran must still "+
			"be adopted by a later periodic pass)", usage)
	}

	if n := countRows(t, cache,
		`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "aabbccdd0099",
	); n != 1 {
		t.Errorf("file that appeared after startup was not adopted by periodic "+
			"reconciliation (rows=%d)", n)
	}
}

// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
// the exact TOCTOU window between evictSourceBlob's row-deletion
// transaction commit and its content file unlink: a concurrent
// StoreSource for a different source path whose content hashes to the
// same value (real SHA-256 dedup, not a contrived case) must not be
// able to insert a fresh row referencing the file while eviction is
// mid-unlink, and must not lose its own store once eviction has fully
// released the content hash.
func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) {
	cache, _ := newEvictionTestCache(t, 1<<30)
	ctx := context.Background()

	content := bytes.Repeat([]byte{0x55}, 400)

	hash := storeEvictionTestSource(t, cache, "race.example.com", "/first.jpg", content)

	proceed := make(chan struct{})
	storeAttempted := make(chan struct{})

	cache.evictSourceBlobTestHook = func(gotHash ContentHash) {
		if gotHash != hash {
			t.Errorf("test hook invoked for hash %s, want %s", gotHash, hash)
		}

		close(storeAttempted)
		<-proceed
	}

	evictDone := make(chan error, 1)

	go func() {
		evictDone <- cache.evictSourceBlob(ctx, hash)
	}()

	// Wait until eviction has committed its delete transaction and is
	// paused (inside the test hook) immediately before unlinking the
	// content file: exactly the window the review flagged.
	<-storeAttempted

	storeDone := make(chan error, 1)

	go func() {
		req := &ImageRequest{
			SourceHost: "race.example.com",
			SourcePath: "/dup.jpg",
			Format:     FormatJPEG,
			Quality:    85,
			FitMode:    FitCover,
		}
		result := &httpfetcher.FetchResult{
			StatusCode:    200,
			ContentType:   "image/jpeg",
			ContentLength: int64(len(content)),
			Headers:       map[string][]string{"Content-Type": {"image/jpeg"}},
		}

		_, err := cache.StoreSource(ctx, req, bytes.NewReader(content), result)
		storeDone <- err
	}()

	// The concurrent store must not be able to complete while eviction
	// still holds the content hash (i.e. before the file is unlinked):
	// if it could, it would insert a row referencing a file about to be
	// removed out from under it.
	select {
	case err := <-storeDone:
		t.Fatalf("StoreSource for identical content completed (err=%v) while eviction "+
			"still held the content hash open between commit and unlink; the store and "+
			"the evict of identical content are not mutually exclusive", err)
	case <-time.After(200 * time.Millisecond):
		// Expected: the store is blocked behind eviction's exclusion.
	}

	close(proceed)

	if err := <-evictDone; err != nil {
		t.Fatalf("evictSourceBlob failed: %v", err)
	}

	if err := <-storeDone; err != nil {
		t.Fatalf("StoreSource failed: %v", err)
	}

	assertNoDanglingReferences(t, cache)

	dupHash, _, err := cache.LookupSource(ctx, &ImageRequest{
		SourceHost: "race.example.com",
		SourcePath: "/dup.jpg",
	})
	if err != nil {
		t.Fatalf("LookupSource failed: %v", err)
	}

	if dupHash == "" {
		t.Fatal("re-stored blob was lost: the store legitimately ran after eviction " +
			"released the content hash and must have recreated the file and row")
	}

	if !cache.srcContent.Exists(dupHash) {
		t.Errorf("source_content/source_metadata references %s but its file is missing", dupHash)
	}
}