sneak
41347a7e9f
reconcileAccounting currently runs exactly once, at evictor startup. Combined with StoreVariant's best-effort accounting insert, a variant file that lands on disk untracked during steady-state operation (e.g. insert failed under transient DB contention) stays invisible to UsageBytes/EvictToLimit until the next process restart -- the drift window the review flagged. Currently red: a file introduced after startup reconciliation has already run is never adopted.
838 lines
24 KiB
Go
838 lines
24 KiB
Go
package imgcache
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
"sneak.berlin/go/pixa/internal/database"
|
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
|
)
|
|
|
|
// sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP
|
|
// produces, so injected timestamps compare correctly against ones the
|
|
// implementation writes.
|
|
const sqliteTimestampFormat = "2006-01-02 15:04:05"
|
|
|
|
// evictionTestDB creates an in-memory SQLite database with the real
|
|
// production schema, limited to a single connection so the background
|
|
// eviction goroutine shares the same in-memory database as the test.
|
|
func evictionTestDB(t *testing.T) *sql.DB {
|
|
t.Helper()
|
|
|
|
db, err := sql.Open("sqlite", ":memory:")
|
|
if err != nil {
|
|
t.Fatalf("failed to open test db: %v", err)
|
|
}
|
|
|
|
db.SetMaxOpenConns(1)
|
|
|
|
if err := database.ApplyMigrations(context.Background(), db, nil); err != nil {
|
|
t.Fatalf("failed to apply migrations: %v", err)
|
|
}
|
|
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
|
|
return db
|
|
}
|
|
|
|
// newEvictionTestCache creates a Cache backed by a temp directory and
|
|
// an in-memory database, with the given size limit.
|
|
func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) {
|
|
t.Helper()
|
|
|
|
tmpDir := t.TempDir()
|
|
db := evictionTestDB(t)
|
|
|
|
// maxBytes zero mirrors the production mapping of
|
|
// cache_max_bytes: 0 (handlers sets DisableDiskCache); at the
|
|
// CacheConfig layer itself a zero MaxBytes means "no limit" for
|
|
// backwards compatibility with existing fixtures.
|
|
cache, err := NewCache(db, CacheConfig{
|
|
StateDir: tmpDir,
|
|
CacheTTL: time.Hour,
|
|
NegativeTTL: 5 * time.Minute,
|
|
MaxBytes: maxBytes,
|
|
DisableDiskCache: maxBytes == 0,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create cache: %v", err)
|
|
}
|
|
|
|
return cache, tmpDir
|
|
}
|
|
|
|
// storeEvictionTestSource stores content as a fetched source for
|
|
// host/path and returns the resulting content hash.
|
|
func storeEvictionTestSource(
|
|
t *testing.T, cache *Cache, host, path string, content []byte,
|
|
) ContentHash {
|
|
t.Helper()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: host,
|
|
SourcePath: path,
|
|
Format: FormatJPEG,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
result := &httpfetcher.FetchResult{
|
|
StatusCode: 200,
|
|
ContentType: "image/jpeg",
|
|
ContentLength: int64(len(content)),
|
|
Headers: map[string][]string{"Content-Type": {"image/jpeg"}},
|
|
}
|
|
|
|
hash, err := cache.StoreSource(context.Background(), req, bytes.NewReader(content), result)
|
|
if err != nil {
|
|
t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err)
|
|
}
|
|
|
|
return hash
|
|
}
|
|
|
|
// storeEvictionTestVariant stores content as a processed variant under
|
|
// the given cache key.
|
|
func storeEvictionTestVariant(t *testing.T, cache *Cache, key VariantKey, content []byte) {
|
|
t.Helper()
|
|
|
|
if err := cache.StoreVariant(key, bytes.NewReader(content), "image/webp"); err != nil {
|
|
t.Fatalf("StoreVariant(%s) failed: %v", key, err)
|
|
}
|
|
}
|
|
|
|
// setVariantLastAccessed backdates the last access time of a tracked
|
|
// variant, to make LRU ordering deterministic in tests.
|
|
func setVariantLastAccessed(t *testing.T, cache *Cache, key VariantKey, when time.Time) {
|
|
t.Helper()
|
|
|
|
res, err := cache.db.Exec(
|
|
`UPDATE variant_content SET last_accessed_at = ? WHERE cache_key = ?`,
|
|
when.UTC().Format(sqliteTimestampFormat), string(key),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to set variant last_accessed_at: %v", err)
|
|
}
|
|
|
|
affected, err := res.RowsAffected()
|
|
if err != nil {
|
|
t.Fatalf("failed to read affected rows: %v", err)
|
|
}
|
|
|
|
if affected != 1 {
|
|
t.Fatalf("variant %s has no accounting row (affected=%d); "+
|
|
"stores must track variants in the database", key, affected)
|
|
}
|
|
}
|
|
|
|
// setSourceLastAccessed backdates the last access time of a tracked
|
|
// source content blob.
|
|
func setSourceLastAccessed(t *testing.T, cache *Cache, hash ContentHash, when time.Time) {
|
|
t.Helper()
|
|
|
|
res, err := cache.db.Exec(
|
|
`UPDATE source_content SET last_accessed_at = ? WHERE content_hash = ?`,
|
|
when.UTC().Format(sqliteTimestampFormat), string(hash),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to set source last_accessed_at: %v", err)
|
|
}
|
|
|
|
affected, err := res.RowsAffected()
|
|
if err != nil {
|
|
t.Fatalf("failed to read affected rows: %v", err)
|
|
}
|
|
|
|
if affected != 1 {
|
|
t.Fatalf("source %s has no accounting row (affected=%d)", hash, affected)
|
|
}
|
|
}
|
|
|
|
// countRows returns the number of rows the given query yields.
|
|
func countRows(t *testing.T, cache *Cache, query string, args ...interface{}) int {
|
|
t.Helper()
|
|
|
|
var n int
|
|
if err := cache.db.QueryRow(query, args...).Scan(&n); err != nil {
|
|
t.Fatalf("count query %q failed: %v", query, err)
|
|
}
|
|
|
|
return n
|
|
}
|
|
|
|
// assertNoDanglingReferences verifies the core eviction invariant:
|
|
// every database row that references cache content on disk points at a
|
|
// file that actually exists.
|
|
func assertNoDanglingReferences(t *testing.T, cache *Cache) {
|
|
t.Helper()
|
|
|
|
rows, err := cache.db.Query(
|
|
`SELECT content_hash FROM source_metadata
|
|
WHERE content_hash IS NOT NULL AND content_hash != ''`,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to query source_metadata: %v", err)
|
|
}
|
|
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
for rows.Next() {
|
|
var hash string
|
|
if err := rows.Scan(&hash); err != nil {
|
|
t.Fatalf("failed to scan content_hash: %v", err)
|
|
}
|
|
|
|
if !cache.srcContent.Exists(ContentHash(hash)) {
|
|
t.Errorf("source_metadata references content %s but the file is missing", hash)
|
|
}
|
|
}
|
|
|
|
if err := rows.Err(); err != nil {
|
|
t.Fatalf("source_metadata iteration failed: %v", err)
|
|
}
|
|
|
|
variantRows, err := cache.db.Query(`SELECT cache_key FROM variant_content`)
|
|
if err != nil {
|
|
t.Fatalf("failed to query variant_content: %v", err)
|
|
}
|
|
|
|
defer func() { _ = variantRows.Close() }()
|
|
|
|
for variantRows.Next() {
|
|
var key string
|
|
if err := variantRows.Scan(&key); err != nil {
|
|
t.Fatalf("failed to scan cache_key: %v", err)
|
|
}
|
|
|
|
if !cache.variants.Exists(VariantKey(key)) {
|
|
t.Errorf("variant_content references key %s but the file is missing", key)
|
|
}
|
|
}
|
|
|
|
if err := variantRows.Err(); err != nil {
|
|
t.Fatalf("variant_content iteration failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// waitForUsageAtOrBelow polls UsageBytes until it reaches limit or the
|
|
// timeout expires, returning the last observed usage.
|
|
func waitForUsageAtOrBelow(t *testing.T, cache *Cache, limit int64, timeout time.Duration) int64 {
|
|
t.Helper()
|
|
|
|
deadline := time.Now().Add(timeout)
|
|
|
|
var usage int64
|
|
|
|
for time.Now().Before(deadline) {
|
|
var err error
|
|
|
|
usage, err = cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage <= limit {
|
|
return usage
|
|
}
|
|
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
|
|
return usage
|
|
}
|
|
|
|
func TestUsageBytesAccountsSourceAndVariantBytes(t *testing.T) {
|
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
|
|
|
storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg",
|
|
bytes.Repeat([]byte{0xAA}, 1000))
|
|
storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg",
|
|
bytes.Repeat([]byte{0xAB}, 2000))
|
|
storeEvictionTestVariant(t, cache, "aabbccdd0001", bytes.Repeat([]byte{0xAC}, 500))
|
|
storeEvictionTestVariant(t, cache, "aabbccdd0002", bytes.Repeat([]byte{0xAD}, 250))
|
|
|
|
usage, err := cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage != 3750 {
|
|
t.Errorf("UsageBytes = %d, want 3750 (1000+2000+500+250)", usage)
|
|
}
|
|
}
|
|
|
|
func TestUsageBytesCountsMultiReferencedBlobOnce(t *testing.T) {
|
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
|
|
|
content := bytes.Repeat([]byte{0xCC}, 1200)
|
|
|
|
hashOne := storeEvictionTestSource(t, cache, "src.example.com", "/one.jpg", content)
|
|
hashTwo := storeEvictionTestSource(t, cache, "src.example.com", "/two.jpg", content)
|
|
|
|
if hashOne != hashTwo {
|
|
t.Fatalf("identical content produced different hashes: %s vs %s", hashOne, hashTwo)
|
|
}
|
|
|
|
usage, err := cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage != 1200 {
|
|
t.Errorf("UsageBytes = %d, want 1200 (deduplicated blob counted once)", usage)
|
|
}
|
|
}
|
|
|
|
func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) {
|
|
const limit = 3000
|
|
|
|
cache, _ := newEvictionTestCache(t, limit)
|
|
|
|
now := time.Now()
|
|
keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003", "aabbccdd0004"}
|
|
fills := []byte{0x01, 0x02, 0x03, 0x04}
|
|
ages := []time.Duration{4 * time.Hour, 3 * time.Hour, 2 * time.Hour, 1 * time.Hour}
|
|
|
|
for i, key := range keys {
|
|
storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{fills[i]}, 1000))
|
|
setVariantLastAccessed(t, cache, key, now.Add(-ages[i]))
|
|
}
|
|
|
|
if err := cache.EvictToLimit(context.Background()); err != nil {
|
|
t.Fatalf("EvictToLimit failed: %v", err)
|
|
}
|
|
|
|
usage, err := cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage > limit {
|
|
t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
|
|
}
|
|
|
|
if cache.variants.Exists(keys[0]) {
|
|
t.Errorf("least recently used variant %s must be evicted", keys[0])
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, string(keys[0]),
|
|
); n != 0 {
|
|
t.Errorf("evicted variant %s still has %d accounting rows", keys[0], n)
|
|
}
|
|
|
|
for _, key := range keys[1:] {
|
|
if !cache.variants.Exists(key) {
|
|
t.Errorf("more recently used variant %s must survive eviction", key)
|
|
}
|
|
}
|
|
|
|
assertNoDanglingReferences(t, cache)
|
|
}
|
|
|
|
func TestEvictionRemovesMultiReferencedBlobTogetherWithAllReferences(t *testing.T) {
|
|
const limit = 1000
|
|
|
|
cache, _ := newEvictionTestCache(t, limit)
|
|
|
|
now := time.Now()
|
|
|
|
// One 800-byte blob referenced by two source paths.
|
|
sharedContent := bytes.Repeat([]byte{0xDD}, 800)
|
|
sharedHash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", sharedContent)
|
|
|
|
if h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", sharedContent); h != sharedHash {
|
|
t.Fatalf("identical content produced different hashes: %s vs %s", h, sharedHash)
|
|
}
|
|
|
|
// A newer 600-byte blob referenced by one source path.
|
|
recentHash := storeEvictionTestSource(t, cache, "src.example.com", "/c.jpg",
|
|
bytes.Repeat([]byte{0xEE}, 600))
|
|
|
|
setSourceLastAccessed(t, cache, sharedHash, now.Add(-2*time.Hour))
|
|
setSourceLastAccessed(t, cache, recentHash, now.Add(-time.Minute))
|
|
|
|
if err := cache.EvictToLimit(context.Background()); err != nil {
|
|
t.Fatalf("EvictToLimit failed: %v", err)
|
|
}
|
|
|
|
usage, err := cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage > limit {
|
|
t.Errorf("usage after eviction = %d, want <= %d", usage, limit)
|
|
}
|
|
|
|
// The multi-referenced blob must be gone from disk, from
|
|
// source_content, and from BOTH source_metadata rows: references
|
|
// are removed together with the blob, never left dangling.
|
|
if cache.srcContent.Exists(sharedHash) {
|
|
t.Errorf("evicted blob %s still exists on disk", sharedHash)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(sharedHash),
|
|
); n != 0 {
|
|
t.Errorf("evicted blob %s still has %d source_content rows", sharedHash, n)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(sharedHash),
|
|
); n != 0 {
|
|
t.Errorf("evicted blob %s still has %d source_metadata references", sharedHash, n)
|
|
}
|
|
|
|
// The JSON metadata sidecars for both referencing paths must be
|
|
// removed along with the rows.
|
|
for _, path := range []string{"/a.jpg", "/b.jpg"} {
|
|
pathHash := HashPath(path + "?")
|
|
if cache.srcMetadata.Exists("src.example.com", pathHash) {
|
|
t.Errorf("metadata sidecar for %s must be removed with its row", path)
|
|
}
|
|
}
|
|
|
|
// The more recently used blob survives fully intact.
|
|
if !cache.srcContent.Exists(recentHash) {
|
|
t.Errorf("recently used blob %s must survive eviction", recentHash)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(recentHash),
|
|
); n != 1 {
|
|
t.Errorf("recently used blob %s has %d source_metadata rows, want 1", recentHash, n)
|
|
}
|
|
|
|
assertNoDanglingReferences(t, cache)
|
|
}
|
|
|
|
func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) {
|
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
|
|
|
content := bytes.Repeat([]byte{0xDF}, 800)
|
|
hash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content)
|
|
|
|
if h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content); h != hash {
|
|
t.Fatalf("identical content produced different hashes: %s vs %s", h, hash)
|
|
}
|
|
|
|
storeEvictionTestVariant(t, cache, "aabbccdd0001", bytes.Repeat([]byte{0xE0}, 500))
|
|
|
|
if err := cache.EvictToLimit(context.Background()); err != nil {
|
|
t.Fatalf("EvictToLimit failed: %v", err)
|
|
}
|
|
|
|
if !cache.srcContent.Exists(hash) {
|
|
t.Errorf("blob %s must not be evicted while usage is under the limit", hash)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(hash),
|
|
); n != 2 {
|
|
t.Errorf("blob %s has %d source_metadata rows, want 2", hash, n)
|
|
}
|
|
|
|
if !cache.variants.Exists("aabbccdd0001") {
|
|
t.Error("variant must not be evicted while usage is under the limit")
|
|
}
|
|
|
|
assertNoDanglingReferences(t, cache)
|
|
}
|
|
|
|
func TestZeroMaxBytesDisablesDiskCache(t *testing.T) {
|
|
cache, tmpDir := newEvictionTestCache(t, 0)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "src.example.com",
|
|
SourcePath: "/a.jpg",
|
|
Format: FormatJPEG,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
// Writes are no-ops that report success.
|
|
if err := cache.StoreVariant(CacheKey(req), bytes.NewReader([]byte("data")), "image/webp"); err != nil {
|
|
t.Fatalf("StoreVariant on disabled cache must be a no-op, got error: %v", err)
|
|
}
|
|
|
|
result := &httpfetcher.FetchResult{
|
|
StatusCode: 200,
|
|
ContentType: "image/jpeg",
|
|
ContentLength: 4,
|
|
Headers: map[string][]string{},
|
|
}
|
|
|
|
hash, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result)
|
|
if err != nil {
|
|
t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err)
|
|
}
|
|
|
|
if hash != "" {
|
|
t.Errorf("StoreSource on disabled cache returned hash %q, want empty", hash)
|
|
}
|
|
|
|
// Reads always miss.
|
|
lookup, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup on disabled cache failed: %v", err)
|
|
}
|
|
|
|
if lookup.Hit {
|
|
t.Error("Lookup on disabled cache must always miss")
|
|
}
|
|
|
|
srcHash, srcType, err := cache.LookupSource(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("LookupSource on disabled cache failed: %v", err)
|
|
}
|
|
|
|
if srcHash != "" || srcType != "" {
|
|
t.Errorf("LookupSource on disabled cache = (%q, %q), want empty", srcHash, srcType)
|
|
}
|
|
|
|
// Nothing is tracked and nothing is written to disk.
|
|
usage, err := cache.UsageBytes(ctx)
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage != 0 {
|
|
t.Errorf("UsageBytes on disabled cache = %d, want 0", usage)
|
|
}
|
|
|
|
if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 {
|
|
t.Errorf("disabled cache wrote %d source_content rows, want 0", n)
|
|
}
|
|
|
|
if n := countRows(t, cache, `SELECT COUNT(*) FROM source_metadata`); n != 0 {
|
|
t.Errorf("disabled cache wrote %d source_metadata rows, want 0", n)
|
|
}
|
|
|
|
if _, err := os.Stat(filepath.Join(tmpDir, "cache")); !os.IsNotExist(err) {
|
|
t.Errorf("disabled cache must not create the cache directory tree (stat err=%v)", err)
|
|
}
|
|
|
|
var foundFiles []string
|
|
|
|
walkErr := filepath.WalkDir(tmpDir, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !d.IsDir() {
|
|
foundFiles = append(foundFiles, path)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if walkErr != nil {
|
|
t.Fatalf("failed to walk state dir: %v", walkErr)
|
|
}
|
|
|
|
if len(foundFiles) != 0 {
|
|
t.Errorf("disabled cache wrote files to disk: %v", foundFiles)
|
|
}
|
|
}
|
|
|
|
func TestEvictionRunsUnderWritePressure(t *testing.T) {
|
|
const limit = 1500
|
|
|
|
cache, _ := newEvictionTestCache(t, limit)
|
|
|
|
// An interval far longer than the test ensures only write
|
|
// pressure can trigger eviction here.
|
|
cache.StartEviction(time.Hour)
|
|
defer cache.StopEviction()
|
|
|
|
keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003"}
|
|
fills := []byte{0x11, 0x12, 0x13}
|
|
|
|
for i, key := range keys {
|
|
storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{fills[i]}, 1000))
|
|
}
|
|
|
|
usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second)
|
|
if usage > limit {
|
|
t.Errorf("write pressure did not trigger eviction: usage = %d, want <= %d",
|
|
usage, limit)
|
|
}
|
|
|
|
assertNoDanglingReferences(t, cache)
|
|
}
|
|
|
|
func TestEvictionRunsOnPeriodicSchedule(t *testing.T) {
|
|
const limit = 1500
|
|
|
|
cache, _ := newEvictionTestCache(t, limit)
|
|
|
|
// Start the evictor while the cache is empty, then create tracked
|
|
// over-limit state WITHOUT going through the store methods, so no
|
|
// write-pressure notification fires and only the periodic ticker
|
|
// can trigger eviction.
|
|
cache.StartEviction(100 * time.Millisecond)
|
|
defer cache.StopEviction()
|
|
|
|
keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003"}
|
|
fills := []byte{0x21, 0x22, 0x23}
|
|
|
|
for i, key := range keys {
|
|
content := bytes.Repeat([]byte{fills[i]}, 1000)
|
|
|
|
if _, err := cache.variants.Store(key, bytes.NewReader(content), "image/webp"); err != nil {
|
|
t.Fatalf("failed to store variant file: %v", err)
|
|
}
|
|
|
|
if _, err := cache.db.Exec(
|
|
`INSERT INTO variant_content (cache_key, size_bytes, content_type)
|
|
VALUES (?, ?, ?)`,
|
|
string(key), len(content), "image/webp",
|
|
); err != nil {
|
|
t.Fatalf("failed to insert variant accounting row: %v", err)
|
|
}
|
|
}
|
|
|
|
usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second)
|
|
if usage > limit {
|
|
t.Errorf("periodic schedule did not trigger eviction: usage = %d, want <= %d",
|
|
usage, limit)
|
|
}
|
|
|
|
assertNoDanglingReferences(t, cache)
|
|
}
|
|
|
|
func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) {
|
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
|
|
|
// An untracked variant file on disk (e.g. written before this
|
|
// feature existed) must be adopted into the accounting.
|
|
untracked := bytes.Repeat([]byte{0x31}, 1000)
|
|
if _, err := cache.variants.Store("aabbccdd0001", bytes.NewReader(untracked), "image/webp"); err != nil {
|
|
t.Fatalf("failed to store untracked variant file: %v", err)
|
|
}
|
|
|
|
// An accounting row whose file is missing must be dropped.
|
|
if _, err := cache.db.Exec(
|
|
`INSERT INTO variant_content (cache_key, size_bytes, content_type)
|
|
VALUES (?, ?, ?)`,
|
|
"deadbeef0001", 700, "image/webp",
|
|
); err != nil {
|
|
t.Fatalf("failed to insert stale variant accounting row: %v", err)
|
|
}
|
|
|
|
cache.StartEviction(time.Hour)
|
|
defer cache.StopEviction()
|
|
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
|
|
var usage int64
|
|
|
|
for time.Now().Before(deadline) {
|
|
var err error
|
|
|
|
usage, err = cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage == 1000 {
|
|
break
|
|
}
|
|
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
|
|
if usage != 1000 {
|
|
t.Errorf("usage after reconciliation = %d, want 1000 "+
|
|
"(untracked file adopted, stale row dropped)", usage)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "aabbccdd0001",
|
|
); n != 1 {
|
|
t.Errorf("untracked variant file was not adopted into accounting (rows=%d)", n)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "deadbeef0001",
|
|
); n != 0 {
|
|
t.Errorf("stale accounting row without a file was not dropped (rows=%d)", n)
|
|
}
|
|
}
|
|
|
|
// TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves
|
|
// reconciliation is not a one-shot startup-only pass: it must also run
|
|
// on the periodic ticker, so a variant file that lands on disk with no
|
|
// accounting row well after startup (e.g. because StoreVariant's
|
|
// best-effort accounting insert failed under transient contention, or
|
|
// any other cause of an untracked file appearing during steady-state
|
|
// operation) is still adopted into accounting eventually, rather than
|
|
// staying invisible to UsageBytes/EvictToLimit until the next process
|
|
// restart.
|
|
func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
|
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
|
|
|
const interval = 100 * time.Millisecond
|
|
|
|
cache.StartEviction(interval)
|
|
defer cache.StopEviction()
|
|
|
|
// Let startup reconciliation run and settle on an empty cache
|
|
// before introducing the untracked file, so the adoption we assert
|
|
// below can only be the work of a later, periodic pass.
|
|
time.Sleep(3 * interval)
|
|
|
|
// Simulate a variant whose accounting insert failed after the
|
|
// process was already running and serving requests: the content
|
|
// file is written directly, bypassing StoreVariant's (and thus its
|
|
// accounting insert) entirely, exactly as would happen if that
|
|
// insert had failed and only the file write had succeeded.
|
|
untracked := bytes.Repeat([]byte{0x41}, 900)
|
|
if _, err := cache.variants.Store("aabbccdd0099", bytes.NewReader(untracked), "image/webp"); err != nil {
|
|
t.Fatalf("failed to store untracked variant file: %v", err)
|
|
}
|
|
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
|
|
var usage int64
|
|
|
|
for time.Now().Before(deadline) {
|
|
var err error
|
|
|
|
usage, err = cache.UsageBytes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("UsageBytes failed: %v", err)
|
|
}
|
|
|
|
if usage == 900 {
|
|
break
|
|
}
|
|
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
|
|
if usage != 900 {
|
|
t.Errorf("usage after periodic reconciliation = %d, want 900 "+
|
|
"(a file that appeared after startup reconciliation already ran must still "+
|
|
"be adopted by a later periodic pass)", usage)
|
|
}
|
|
|
|
if n := countRows(t, cache,
|
|
`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "aabbccdd0099",
|
|
); n != 1 {
|
|
t.Errorf("file that appeared after startup was not adopted by periodic "+
|
|
"reconciliation (rows=%d)", n)
|
|
}
|
|
}
|
|
|
|
// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises
|
|
// the exact TOCTOU window between evictSourceBlob's row-deletion
|
|
// transaction commit and its content file unlink: a concurrent
|
|
// StoreSource for a different source path whose content hashes to the
|
|
// same value (real SHA-256 dedup, not a contrived case) must not be
|
|
// able to insert a fresh row referencing the file while eviction is
|
|
// mid-unlink, and must not lose its own store once eviction has fully
|
|
// released the content hash.
|
|
func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) {
|
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
|
ctx := context.Background()
|
|
|
|
content := bytes.Repeat([]byte{0x55}, 400)
|
|
|
|
hash := storeEvictionTestSource(t, cache, "race.example.com", "/first.jpg", content)
|
|
|
|
proceed := make(chan struct{})
|
|
storeAttempted := make(chan struct{})
|
|
|
|
cache.evictSourceBlobTestHook = func(gotHash ContentHash) {
|
|
if gotHash != hash {
|
|
t.Errorf("test hook invoked for hash %s, want %s", gotHash, hash)
|
|
}
|
|
|
|
close(storeAttempted)
|
|
<-proceed
|
|
}
|
|
|
|
evictDone := make(chan error, 1)
|
|
|
|
go func() {
|
|
evictDone <- cache.evictSourceBlob(ctx, hash)
|
|
}()
|
|
|
|
// Wait until eviction has committed its delete transaction and is
|
|
// paused (inside the test hook) immediately before unlinking the
|
|
// content file: exactly the window the review flagged.
|
|
<-storeAttempted
|
|
|
|
storeDone := make(chan error, 1)
|
|
|
|
go func() {
|
|
req := &ImageRequest{
|
|
SourceHost: "race.example.com",
|
|
SourcePath: "/dup.jpg",
|
|
Format: FormatJPEG,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
result := &httpfetcher.FetchResult{
|
|
StatusCode: 200,
|
|
ContentType: "image/jpeg",
|
|
ContentLength: int64(len(content)),
|
|
Headers: map[string][]string{"Content-Type": {"image/jpeg"}},
|
|
}
|
|
|
|
_, err := cache.StoreSource(ctx, req, bytes.NewReader(content), result)
|
|
storeDone <- err
|
|
}()
|
|
|
|
// The concurrent store must not be able to complete while eviction
|
|
// still holds the content hash (i.e. before the file is unlinked):
|
|
// if it could, it would insert a row referencing a file about to be
|
|
// removed out from under it.
|
|
select {
|
|
case err := <-storeDone:
|
|
t.Fatalf("StoreSource for identical content completed (err=%v) while eviction "+
|
|
"still held the content hash open between commit and unlink; the store and "+
|
|
"the evict of identical content are not mutually exclusive", err)
|
|
case <-time.After(200 * time.Millisecond):
|
|
// Expected: the store is blocked behind eviction's exclusion.
|
|
}
|
|
|
|
close(proceed)
|
|
|
|
if err := <-evictDone; err != nil {
|
|
t.Fatalf("evictSourceBlob failed: %v", err)
|
|
}
|
|
|
|
if err := <-storeDone; err != nil {
|
|
t.Fatalf("StoreSource failed: %v", err)
|
|
}
|
|
|
|
assertNoDanglingReferences(t, cache)
|
|
|
|
dupHash, _, err := cache.LookupSource(ctx, &ImageRequest{
|
|
SourceHost: "race.example.com",
|
|
SourcePath: "/dup.jpg",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("LookupSource failed: %v", err)
|
|
}
|
|
|
|
if dupHash == "" {
|
|
t.Fatal("re-stored blob was lost: the store legitimately ran after eviction " +
|
|
"released the content hash and must have recreated the file and row")
|
|
}
|
|
|
|
if !cache.srcContent.Exists(dupHash) {
|
|
t.Errorf("source_content/source_metadata references %s but its file is missing", dupHash)
|
|
}
|
|
}
|