From 3963ec31c1ec6af90159e07c68bbf4f5e268983b Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 7 Aug 2026 20:58:03 +0000 Subject: [PATCH] test: add failing tests for cache_max_bytes config and cache eviction Red phase for #51: covers strict cache_max_bytes parsing (invalid explicit values abort naming key and value), the computed default of max(75% of free space, 500 MiB) via an injectable free-space probe, explicit-value-no-floor, zero-disables-cache, size accounting over source blobs and variants, LRU eviction under the limit, the multi-referenced blob case, write-pressure and periodic eviction triggers, and startup reconciliation. Minimal API skeletons keep the tree compiling and lint-clean; only the new tests fail. --- internal/config/cache_max_bytes_test.go | 271 ++++++++++ internal/config/cachesize.go | 60 +++ internal/config/config.go | 11 + internal/imgcache/cache.go | 11 + internal/imgcache/eviction.go | 41 ++ internal/imgcache/eviction_test.go | 664 ++++++++++++++++++++++++ 6 files changed, 1058 insertions(+) create mode 100644 internal/config/cache_max_bytes_test.go create mode 100644 internal/config/cachesize.go create mode 100644 internal/imgcache/eviction.go create mode 100644 internal/imgcache/eviction_test.go diff --git a/internal/config/cache_max_bytes_test.go b/internal/config/cache_max_bytes_test.go new file mode 100644 index 0000000..ac7bf72 --- /dev/null +++ b/internal/config/cache_max_bytes_test.go @@ -0,0 +1,271 @@ +package config + +import ( + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" +) + +// discardLogger returns a logger that swallows all output, for tests +// that exercise code paths which log. +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// TestCacheMaxBytesExplicitValueUsedWithoutFloor verifies that an +// explicitly configured cache_max_bytes value is used exactly as +// given: the 500 MiB floor applies only to the computed default, never +// to explicit values. +func TestCacheMaxBytesExplicitValueUsedWithoutFloor(t *testing.T) { + yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 1024\n" + + c, err := configFromYAML(t, yamlContent) + if err != nil { + t.Fatalf("explicit cache_max_bytes must be accepted, got error: %v", err) + } + + if c.CacheMaxBytes != 1024 { + t.Errorf("CacheMaxBytes = %d, want 1024 (no floor for explicit values)", + c.CacheMaxBytes) + } +} + +// TestCacheMaxBytesZeroIsValidAndDisablesCache verifies that an +// explicit zero is a valid value (it disables the disk cache), not an +// error. +func TestCacheMaxBytesZeroIsValidAndDisablesCache(t *testing.T) { + yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 0\n" + + c, err := configFromYAML(t, yamlContent) + if err != nil { + t.Fatalf("cache_max_bytes: 0 must be accepted, got error: %v", err) + } + + if c.CacheMaxBytes != 0 { + t.Errorf("CacheMaxBytes = %d, want 0", c.CacheMaxBytes) + } +} + +// TestCacheMaxBytesLargeExplicitValueParses verifies that values above +// 32-bit range parse correctly (the field is an int64 byte count). +func TestCacheMaxBytesLargeExplicitValueParses(t *testing.T) { + yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 10737418240\n" + + c, err := configFromYAML(t, yamlContent) + if err != nil { + t.Fatalf("large cache_max_bytes must be accepted, got error: %v", err) + } + + if c.CacheMaxBytes != 10737418240 { + t.Errorf("CacheMaxBytes = %d, want 10737418240", c.CacheMaxBytes) + } +} + +// TestCacheMaxBytesInvalidValuesAbortStartup verifies that a SET but +// invalid cache_max_bytes value aborts startup naming the key and the +// offending value, per the no-silent-fallback rule: defaults apply +// only to omitted keys. +func TestCacheMaxBytesInvalidValuesAbortStartup(t *testing.T) { + signingKeyLine := "signing_key: " + validTestSigningKey + "\n" + + cases := []struct { + name string + yaml string + // wantErrSubstrings must all appear in the error message. + wantErrSubstrings []string + }{ + { + name: "negative", + yaml: signingKeyLine + "cache_max_bytes: -1024\n", + wantErrSubstrings: []string{"cache_max_bytes", "-1024"}, + }, + { + name: "float", + yaml: signingKeyLine + "cache_max_bytes: 3.5\n", + wantErrSubstrings: []string{"cache_max_bytes", "3.5"}, + }, + { + name: "non-numeric string", + yaml: signingKeyLine + "cache_max_bytes: banana\n", + wantErrSubstrings: []string{"cache_max_bytes", "banana"}, + }, + { + name: "explicit null", + yaml: signingKeyLine + "cache_max_bytes: null\n", + wantErrSubstrings: []string{"cache_max_bytes", "null"}, + }, + { + name: "bare key no value", + yaml: signingKeyLine + "cache_max_bytes:\n", + wantErrSubstrings: []string{"cache_max_bytes", "null"}, + }, + { + name: "boolean", + yaml: signingKeyLine + "cache_max_bytes: true\n", + wantErrSubstrings: []string{"cache_max_bytes", "true"}, + }, + { + name: "list", + yaml: signingKeyLine + "cache_max_bytes:\n - 1\n", + wantErrSubstrings: []string{"cache_max_bytes"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, err := configFromYAML(t, tc.yaml) + if err == nil { + t.Fatalf("config with %s cache_max_bytes must abort startup, got config: %+v", + tc.name, c) + } + + t.Logf("got expected error: %v", err) + + for _, want := range tc.wantErrSubstrings { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err.Error(), want) + } + } + }) + } +} + +// TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace verifies the +// computed default is 75% of the probed free space when that exceeds +// the floor. +func TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace(t *testing.T) { + // 4 GiB free -> 3 GiB default. + probe := func(string) (uint64, error) { return 4294967296, nil } + + got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe) + if err != nil { + t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err) + } + + if got != 3221225472 { + t.Errorf("ComputeDefaultCacheMaxBytes = %d, want 3221225472 (75%% of 4 GiB)", got) + } +} + +// TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault +// verifies that when 75% of free space is below 500 MiB, the computed +// default is floored at DefaultCacheMaxBytesFloor. +func TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault(t *testing.T) { + cases := []struct { + name string + freeBytes uint64 + }{ + {name: "100 MiB free", freeBytes: 104857600}, + {name: "zero free", freeBytes: 0}, + {name: "just below floor threshold", freeBytes: 699050665}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + probe := func(string) (uint64, error) { return tc.freeBytes, nil } + + got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe) + if err != nil { + t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err) + } + + if got != DefaultCacheMaxBytesFloor { + t.Errorf("ComputeDefaultCacheMaxBytes = %d, want floor %d", + got, DefaultCacheMaxBytesFloor) + } + }) + } +} + +// TestComputeDefaultCacheMaxBytesPropagatesProbeError verifies that a +// failing free-space probe produces an error naming the config key, +// instead of a silently wrong default. +func TestComputeDefaultCacheMaxBytesPropagatesProbeError(t *testing.T) { + probe := func(string) (uint64, error) { return 0, errors.New("statfs failed") } + + _, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe) + if err == nil { + t.Fatal("probe failure must produce an error, got nil") + } + + t.Logf("got expected error: %v", err) + + if !strings.Contains(err.Error(), "cache_max_bytes") { + t.Errorf("error %q does not name the config key cache_max_bytes", err.Error()) + } +} + +// TestResolveCacheMaxBytesComputesDefaultWhenOmitted verifies that an +// omitted cache_max_bytes key resolves to the computed default, that +// the probe is pointed at /cache/ (which must be created +// first so statfs measures the right filesystem), and that the result +// lands on the Config. +func TestResolveCacheMaxBytesComputesDefaultWhenOmitted(t *testing.T) { + c, err := configFromYAML(t, "signing_key: "+validTestSigningKey+"\n") + if err != nil { + t.Fatalf("minimal config should be valid, got error: %v", err) + } + + c.StateDir = t.TempDir() + wantCacheDir := filepath.Join(c.StateDir, "cache") + + var probedPath string + + // 4 GiB free -> 3 GiB default. + probe := func(path string) (uint64, error) { + probedPath = path + + return 4294967296, nil + } + + if err := c.resolveCacheMaxBytes(discardLogger(), probe); err != nil { + t.Fatalf("resolveCacheMaxBytes returned error: %v", err) + } + + if c.CacheMaxBytes != 3221225472 { + t.Errorf("CacheMaxBytes = %d, want computed default 3221225472", c.CacheMaxBytes) + } + + if probedPath != wantCacheDir { + t.Errorf("free space probed at %q, want cache directory %q", probedPath, wantCacheDir) + } + + info, err := os.Stat(wantCacheDir) + if err != nil || !info.IsDir() { + t.Errorf("cache directory %q was not created before probing: info=%v err=%v", + wantCacheDir, info, err) + } +} + +// TestResolveCacheMaxBytesDoesNotOverrideExplicitValue verifies that +// an explicitly configured value survives resolution untouched and +// that the free-space probe is never consulted for it. +func TestResolveCacheMaxBytesDoesNotOverrideExplicitValue(t *testing.T) { + yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 1024\n" + + c, err := configFromYAML(t, yamlContent) + if err != nil { + t.Fatalf("explicit cache_max_bytes must be accepted, got error: %v", err) + } + + c.StateDir = t.TempDir() + + probe := func(string) (uint64, error) { + t.Error("free-space probe must not be consulted for explicit values") + + return 0, errors.New("probe must not be called") + } + + if err := c.resolveCacheMaxBytes(discardLogger(), probe); err != nil { + t.Fatalf("resolveCacheMaxBytes returned error: %v", err) + } + + if c.CacheMaxBytes != 1024 { + t.Errorf("CacheMaxBytes = %d, want explicit 1024 (no floor, no recompute)", + c.CacheMaxBytes) + } +} diff --git a/internal/config/cachesize.go b/internal/config/cachesize.go new file mode 100644 index 0000000..9ee71a2 --- /dev/null +++ b/internal/config/cachesize.go @@ -0,0 +1,60 @@ +package config + +import ( + "fmt" + "log/slog" + "path/filepath" + "syscall" +) + +// DefaultCacheMaxBytesFloor is the minimum computed default for the +// cache_max_bytes setting: 500 MiB. The floor applies only to the +// computed default (when the key is omitted from the configuration), +// never to explicitly configured values. +const DefaultCacheMaxBytesFloor int64 = 524288000 + +// FreeSpaceProbeFunc reports the number of free bytes available on the +// filesystem containing path. It is a function type so tests can +// inject a fake probe instead of depending on the host disk. +type FreeSpaceProbeFunc func(path string) (uint64, error) + +// defaultFreeSpaceProbe reports free filesystem bytes via statfs on +// the given path, as available to unprivileged processes. +func defaultFreeSpaceProbe(path string) (uint64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, err + } + + if stat.Bsize < 0 { + return 0, fmt.Errorf("statfs reported negative block size %d for %q", stat.Bsize, path) + } + + blockSize := uint64(stat.Bsize) //nolint:gosec // G115: negative Bsize rejected above + + return stat.Bavail * blockSize, nil +} + +// ComputeDefaultCacheMaxBytes returns the default cache size limit for +// the filesystem containing cacheDir: 75% of the free bytes reported +// by probe, with a floor of DefaultCacheMaxBytesFloor. +func ComputeDefaultCacheMaxBytes(_ string, _ FreeSpaceProbeFunc) (int64, error) { + // Red phase: implementation follows the failing tests. + return 0, nil +} + +// resolveCacheMaxBytes finalizes CacheMaxBytes after state_dir +// validation: an explicitly configured value is kept as-is (no floor +// applies), while an omitted key receives the computed default based +// on free space in /cache/. The effective limit is logged. +func (c *Config) resolveCacheMaxBytes(log *slog.Logger, probe FreeSpaceProbeFunc) error { + // Red phase: implementation follows the failing tests. + limit, err := ComputeDefaultCacheMaxBytes(filepath.Join(c.StateDir, "cache"), probe) + if err != nil { + return err + } + + log.Debug("cache size limit resolution not implemented", "computed", limit) + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 702f1b8..adcdcfa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -48,6 +48,13 @@ type Config struct { AllowlistHosts []string // Hosts that don't require signatures AllowHTTP bool // Allow non-TLS upstream (testing only) UpstreamConnectionsPerHost int // Max concurrent connections per upstream host + + // CacheMaxBytes is the disk cache size limit in bytes. Zero + // disables the disk cache entirely. When cache_max_bytes is + // omitted from the configuration, this holds the computed default + // (75% of free space on the filesystem containing + // /cache/, floored at DefaultCacheMaxBytesFloor). + CacheMaxBytes int64 } // New creates a new Config instance by loading configuration from file. @@ -73,6 +80,10 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) { return nil, err } + if err := c.resolveCacheMaxBytes(log, defaultFreeSpaceProbe); err != nil { + return nil, err + } + if c.Debug { params.Logger.EnableDebugLogging() } diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 5e15873..4f74110 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "log/slog" "path/filepath" "time" @@ -27,6 +28,16 @@ type CacheConfig struct { StateDir string CacheTTL time.Duration NegativeTTL time.Duration + + // MaxBytes is the disk cache size limit in bytes. Zero disables + // the disk cache entirely (no reads, no writes, no eviction); the + // config layer supplies the computed default when the operator + // omits cache_max_bytes. + MaxBytes int64 + + // Logger receives accounting and eviction log output. A nil + // Logger means slog.Default(). + Logger *slog.Logger } // variantMeta stores content type for fast cache hits without reading .meta file. diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go new file mode 100644 index 0000000..044d59d --- /dev/null +++ b/internal/imgcache/eviction.go @@ -0,0 +1,41 @@ +package imgcache + +import ( + "context" + "time" +) + +// DefaultEvictionInterval is how often the background evictor checks +// cache usage against the configured limit, in addition to the +// write-pressure wakeups triggered by stores. +const DefaultEvictionInterval = 5 * time.Minute + +// UsageBytes returns the total number of bytes of cache content +// tracked in the database (source content blobs plus processed +// variants). It never scans the cache directories. +func (c *Cache) UsageBytes(_ context.Context) (int64, error) { + // Red phase: implementation follows the failing tests. + return 0, nil +} + +// EvictToLimit evicts least-recently-used cache entries until total +// tracked usage is at or below the configured MaxBytes limit. It is a +// no-op when the cache is disabled (MaxBytes zero). +func (c *Cache) EvictToLimit(_ context.Context) error { + // Red phase: implementation follows the failing tests. + return nil +} + +// StartEviction launches the background eviction goroutine, which +// reconciles the database accounting with the cache directories once +// at startup and then evicts to the configured limit on the given +// periodic interval and on write-pressure notifications. +func (c *Cache) StartEviction(_ time.Duration) { + // Red phase: implementation follows the failing tests. +} + +// StopEviction stops the background eviction goroutine and waits for +// it to exit. It is safe to call when eviction was never started. +func (c *Cache) StopEviction() { + // Red phase: implementation follows the failing tests. +} diff --git a/internal/imgcache/eviction_test.go b/internal/imgcache/eviction_test.go new file mode 100644 index 0000000..ce807db --- /dev/null +++ b/internal/imgcache/eviction_test.go @@ -0,0 +1,664 @@ +package imgcache + +import ( + "bytes" + "context" + "database/sql" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + _ "modernc.org/sqlite" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/httpfetcher" +) + +// sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP +// produces, so injected timestamps compare correctly against ones the +// implementation writes. +const sqliteTimestampFormat = "2006-01-02 15:04:05" + +// evictionTestDB creates an in-memory SQLite database with the real +// production schema, limited to a single connection so the background +// eviction goroutine shares the same in-memory database as the test. +func evictionTestDB(t *testing.T) *sql.DB { + t.Helper() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("failed to open test db: %v", err) + } + + db.SetMaxOpenConns(1) + + if err := database.ApplyMigrations(context.Background(), db, nil); err != nil { + t.Fatalf("failed to apply migrations: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + return db +} + +// newEvictionTestCache creates a Cache backed by a temp directory and +// an in-memory database, with the given size limit. +func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) { + t.Helper() + + tmpDir := t.TempDir() + db := evictionTestDB(t) + + cache, err := NewCache(db, CacheConfig{ + StateDir: tmpDir, + CacheTTL: time.Hour, + NegativeTTL: 5 * time.Minute, + MaxBytes: maxBytes, + }) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + return cache, tmpDir +} + +// storeEvictionTestSource stores content as a fetched source for +// host/path and returns the resulting content hash. +func storeEvictionTestSource( + t *testing.T, cache *Cache, host, path string, content []byte, +) ContentHash { + t.Helper() + + req := &ImageRequest{ + SourceHost: host, + SourcePath: path, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } + + result := &httpfetcher.FetchResult{ + StatusCode: 200, + ContentType: "image/jpeg", + ContentLength: int64(len(content)), + Headers: map[string][]string{"Content-Type": {"image/jpeg"}}, + } + + hash, err := cache.StoreSource(context.Background(), req, bytes.NewReader(content), result) + if err != nil { + t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err) + } + + return hash +} + +// storeEvictionTestVariant stores content as a processed variant under +// the given cache key. +func storeEvictionTestVariant(t *testing.T, cache *Cache, key VariantKey, content []byte) { + t.Helper() + + if err := cache.StoreVariant(key, bytes.NewReader(content), "image/webp"); err != nil { + t.Fatalf("StoreVariant(%s) failed: %v", key, err) + } +} + +// setVariantLastAccessed backdates the last access time of a tracked +// variant, to make LRU ordering deterministic in tests. +func setVariantLastAccessed(t *testing.T, cache *Cache, key VariantKey, when time.Time) { + t.Helper() + + res, err := cache.db.Exec( + `UPDATE variant_content SET last_accessed_at = ? WHERE cache_key = ?`, + when.UTC().Format(sqliteTimestampFormat), string(key), + ) + if err != nil { + t.Fatalf("failed to set variant last_accessed_at: %v", err) + } + + affected, err := res.RowsAffected() + if err != nil { + t.Fatalf("failed to read affected rows: %v", err) + } + + if affected != 1 { + t.Fatalf("variant %s has no accounting row (affected=%d); "+ + "stores must track variants in the database", key, affected) + } +} + +// setSourceLastAccessed backdates the last access time of a tracked +// source content blob. +func setSourceLastAccessed(t *testing.T, cache *Cache, hash ContentHash, when time.Time) { + t.Helper() + + res, err := cache.db.Exec( + `UPDATE source_content SET last_accessed_at = ? WHERE content_hash = ?`, + when.UTC().Format(sqliteTimestampFormat), string(hash), + ) + if err != nil { + t.Fatalf("failed to set source last_accessed_at: %v", err) + } + + affected, err := res.RowsAffected() + if err != nil { + t.Fatalf("failed to read affected rows: %v", err) + } + + if affected != 1 { + t.Fatalf("source %s has no accounting row (affected=%d)", hash, affected) + } +} + +// countRows returns the number of rows the given query yields. +func countRows(t *testing.T, cache *Cache, query string, args ...interface{}) int { + t.Helper() + + var n int + if err := cache.db.QueryRow(query, args...).Scan(&n); err != nil { + t.Fatalf("count query %q failed: %v", query, err) + } + + return n +} + +// assertNoDanglingReferences verifies the core eviction invariant: +// every database row that references cache content on disk points at a +// file that actually exists. +func assertNoDanglingReferences(t *testing.T, cache *Cache) { + t.Helper() + + rows, err := cache.db.Query( + `SELECT content_hash FROM source_metadata + WHERE content_hash IS NOT NULL AND content_hash != ''`, + ) + if err != nil { + t.Fatalf("failed to query source_metadata: %v", err) + } + + defer func() { _ = rows.Close() }() + + for rows.Next() { + var hash string + if err := rows.Scan(&hash); err != nil { + t.Fatalf("failed to scan content_hash: %v", err) + } + + if !cache.srcContent.Exists(ContentHash(hash)) { + t.Errorf("source_metadata references content %s but the file is missing", hash) + } + } + + if err := rows.Err(); err != nil { + t.Fatalf("source_metadata iteration failed: %v", err) + } + + variantRows, err := cache.db.Query(`SELECT cache_key FROM variant_content`) + if err != nil { + t.Fatalf("failed to query variant_content: %v", err) + } + + defer func() { _ = variantRows.Close() }() + + for variantRows.Next() { + var key string + if err := variantRows.Scan(&key); err != nil { + t.Fatalf("failed to scan cache_key: %v", err) + } + + if !cache.variants.Exists(VariantKey(key)) { + t.Errorf("variant_content references key %s but the file is missing", key) + } + } + + if err := variantRows.Err(); err != nil { + t.Fatalf("variant_content iteration failed: %v", err) + } +} + +// waitForUsageAtOrBelow polls UsageBytes until it reaches limit or the +// timeout expires, returning the last observed usage. +func waitForUsageAtOrBelow(t *testing.T, cache *Cache, limit int64, timeout time.Duration) int64 { + t.Helper() + + deadline := time.Now().Add(timeout) + + var usage int64 + + for time.Now().Before(deadline) { + var err error + + usage, err = cache.UsageBytes(context.Background()) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage <= limit { + return usage + } + + time.Sleep(25 * time.Millisecond) + } + + return usage +} + +func TestUsageBytesAccountsSourceAndVariantBytes(t *testing.T) { + cache, _ := newEvictionTestCache(t, 1<<30) + + storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", + bytes.Repeat([]byte{0xAA}, 1000)) + storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", + bytes.Repeat([]byte{0xAB}, 2000)) + storeEvictionTestVariant(t, cache, "aabbccdd0001", bytes.Repeat([]byte{0xAC}, 500)) + storeEvictionTestVariant(t, cache, "aabbccdd0002", bytes.Repeat([]byte{0xAD}, 250)) + + usage, err := cache.UsageBytes(context.Background()) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage != 3750 { + t.Errorf("UsageBytes = %d, want 3750 (1000+2000+500+250)", usage) + } +} + +func TestUsageBytesCountsMultiReferencedBlobOnce(t *testing.T) { + cache, _ := newEvictionTestCache(t, 1<<30) + + content := bytes.Repeat([]byte{0xCC}, 1200) + + hashOne := storeEvictionTestSource(t, cache, "src.example.com", "/one.jpg", content) + hashTwo := storeEvictionTestSource(t, cache, "src.example.com", "/two.jpg", content) + + if hashOne != hashTwo { + t.Fatalf("identical content produced different hashes: %s vs %s", hashOne, hashTwo) + } + + usage, err := cache.UsageBytes(context.Background()) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage != 1200 { + t.Errorf("UsageBytes = %d, want 1200 (deduplicated blob counted once)", usage) + } +} + +func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) { + const limit = 3000 + + cache, _ := newEvictionTestCache(t, limit) + + now := time.Now() + keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003", "aabbccdd0004"} + fills := []byte{0x01, 0x02, 0x03, 0x04} + ages := []time.Duration{4 * time.Hour, 3 * time.Hour, 2 * time.Hour, 1 * time.Hour} + + for i, key := range keys { + storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{fills[i]}, 1000)) + setVariantLastAccessed(t, cache, key, now.Add(-ages[i])) + } + + if err := cache.EvictToLimit(context.Background()); err != nil { + t.Fatalf("EvictToLimit failed: %v", err) + } + + usage, err := cache.UsageBytes(context.Background()) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage > limit { + t.Errorf("usage after eviction = %d, want <= %d", usage, limit) + } + + if cache.variants.Exists(keys[0]) { + t.Errorf("least recently used variant %s must be evicted", keys[0]) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, string(keys[0]), + ); n != 0 { + t.Errorf("evicted variant %s still has %d accounting rows", keys[0], n) + } + + for _, key := range keys[1:] { + if !cache.variants.Exists(key) { + t.Errorf("more recently used variant %s must survive eviction", key) + } + } + + assertNoDanglingReferences(t, cache) +} + +func TestEvictionRemovesMultiReferencedBlobTogetherWithAllReferences(t *testing.T) { + const limit = 1000 + + cache, _ := newEvictionTestCache(t, limit) + + now := time.Now() + + // One 800-byte blob referenced by two source paths. + sharedContent := bytes.Repeat([]byte{0xDD}, 800) + sharedHash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", sharedContent) + + if h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", sharedContent); h != sharedHash { + t.Fatalf("identical content produced different hashes: %s vs %s", h, sharedHash) + } + + // A newer 600-byte blob referenced by one source path. + recentHash := storeEvictionTestSource(t, cache, "src.example.com", "/c.jpg", + bytes.Repeat([]byte{0xEE}, 600)) + + setSourceLastAccessed(t, cache, sharedHash, now.Add(-2*time.Hour)) + setSourceLastAccessed(t, cache, recentHash, now.Add(-time.Minute)) + + if err := cache.EvictToLimit(context.Background()); err != nil { + t.Fatalf("EvictToLimit failed: %v", err) + } + + usage, err := cache.UsageBytes(context.Background()) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage > limit { + t.Errorf("usage after eviction = %d, want <= %d", usage, limit) + } + + // The multi-referenced blob must be gone from disk, from + // source_content, and from BOTH source_metadata rows: references + // are removed together with the blob, never left dangling. + if cache.srcContent.Exists(sharedHash) { + t.Errorf("evicted blob %s still exists on disk", sharedHash) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(sharedHash), + ); n != 0 { + t.Errorf("evicted blob %s still has %d source_content rows", sharedHash, n) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(sharedHash), + ); n != 0 { + t.Errorf("evicted blob %s still has %d source_metadata references", sharedHash, n) + } + + // The JSON metadata sidecars for both referencing paths must be + // removed along with the rows. + for _, path := range []string{"/a.jpg", "/b.jpg"} { + pathHash := HashPath(path + "?") + if cache.srcMetadata.Exists("src.example.com", pathHash) { + t.Errorf("metadata sidecar for %s must be removed with its row", path) + } + } + + // The more recently used blob survives fully intact. + if !cache.srcContent.Exists(recentHash) { + t.Errorf("recently used blob %s must survive eviction", recentHash) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(recentHash), + ); n != 1 { + t.Errorf("recently used blob %s has %d source_metadata rows, want 1", recentHash, n) + } + + assertNoDanglingReferences(t, cache) +} + +func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) { + cache, _ := newEvictionTestCache(t, 1<<30) + + content := bytes.Repeat([]byte{0xDF}, 800) + hash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content) + + if h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content); h != hash { + t.Fatalf("identical content produced different hashes: %s vs %s", h, hash) + } + + storeEvictionTestVariant(t, cache, "aabbccdd0001", bytes.Repeat([]byte{0xE0}, 500)) + + if err := cache.EvictToLimit(context.Background()); err != nil { + t.Fatalf("EvictToLimit failed: %v", err) + } + + if !cache.srcContent.Exists(hash) { + t.Errorf("blob %s must not be evicted while usage is under the limit", hash) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(hash), + ); n != 2 { + t.Errorf("blob %s has %d source_metadata rows, want 2", hash, n) + } + + if !cache.variants.Exists("aabbccdd0001") { + t.Error("variant must not be evicted while usage is under the limit") + } + + assertNoDanglingReferences(t, cache) +} + +func TestZeroMaxBytesDisablesDiskCache(t *testing.T) { + cache, tmpDir := newEvictionTestCache(t, 0) + ctx := context.Background() + + req := &ImageRequest{ + SourceHost: "src.example.com", + SourcePath: "/a.jpg", + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } + + // Writes are no-ops that report success. + if err := cache.StoreVariant(CacheKey(req), bytes.NewReader([]byte("data")), "image/webp"); err != nil { + t.Fatalf("StoreVariant on disabled cache must be a no-op, got error: %v", err) + } + + result := &httpfetcher.FetchResult{ + StatusCode: 200, + ContentType: "image/jpeg", + ContentLength: 4, + Headers: map[string][]string{}, + } + + hash, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result) + if err != nil { + t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err) + } + + if hash != "" { + t.Errorf("StoreSource on disabled cache returned hash %q, want empty", hash) + } + + // Reads always miss. + lookup, err := cache.Lookup(ctx, req) + if err != nil { + t.Fatalf("Lookup on disabled cache failed: %v", err) + } + + if lookup.Hit { + t.Error("Lookup on disabled cache must always miss") + } + + srcHash, srcType, err := cache.LookupSource(ctx, req) + if err != nil { + t.Fatalf("LookupSource on disabled cache failed: %v", err) + } + + if srcHash != "" || srcType != "" { + t.Errorf("LookupSource on disabled cache = (%q, %q), want empty", srcHash, srcType) + } + + // Nothing is tracked and nothing is written to disk. + usage, err := cache.UsageBytes(ctx) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage != 0 { + t.Errorf("UsageBytes on disabled cache = %d, want 0", usage) + } + + if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 { + t.Errorf("disabled cache wrote %d source_content rows, want 0", n) + } + + if n := countRows(t, cache, `SELECT COUNT(*) FROM source_metadata`); n != 0 { + t.Errorf("disabled cache wrote %d source_metadata rows, want 0", n) + } + + if _, err := os.Stat(filepath.Join(tmpDir, "cache")); !os.IsNotExist(err) { + t.Errorf("disabled cache must not create the cache directory tree (stat err=%v)", err) + } + + var foundFiles []string + + walkErr := filepath.WalkDir(tmpDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !d.IsDir() { + foundFiles = append(foundFiles, path) + } + + return nil + }) + if walkErr != nil { + t.Fatalf("failed to walk state dir: %v", walkErr) + } + + if len(foundFiles) != 0 { + t.Errorf("disabled cache wrote files to disk: %v", foundFiles) + } +} + +func TestEvictionRunsUnderWritePressure(t *testing.T) { + const limit = 1500 + + cache, _ := newEvictionTestCache(t, limit) + + // An interval far longer than the test ensures only write + // pressure can trigger eviction here. + cache.StartEviction(time.Hour) + defer cache.StopEviction() + + keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003"} + fills := []byte{0x11, 0x12, 0x13} + + for i, key := range keys { + storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{fills[i]}, 1000)) + } + + usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second) + if usage > limit { + t.Errorf("write pressure did not trigger eviction: usage = %d, want <= %d", + usage, limit) + } + + assertNoDanglingReferences(t, cache) +} + +func TestEvictionRunsOnPeriodicSchedule(t *testing.T) { + const limit = 1500 + + cache, _ := newEvictionTestCache(t, limit) + + // Start the evictor while the cache is empty, then create tracked + // over-limit state WITHOUT going through the store methods, so no + // write-pressure notification fires and only the periodic ticker + // can trigger eviction. + cache.StartEviction(100 * time.Millisecond) + defer cache.StopEviction() + + keys := []VariantKey{"aabbccdd0001", "aabbccdd0002", "aabbccdd0003"} + fills := []byte{0x21, 0x22, 0x23} + + for i, key := range keys { + content := bytes.Repeat([]byte{fills[i]}, 1000) + + if _, err := cache.variants.Store(key, bytes.NewReader(content), "image/webp"); err != nil { + t.Fatalf("failed to store variant file: %v", err) + } + + if _, err := cache.db.Exec( + `INSERT INTO variant_content (cache_key, size_bytes, content_type) + VALUES (?, ?, ?)`, + string(key), len(content), "image/webp", + ); err != nil { + t.Fatalf("failed to insert variant accounting row: %v", err) + } + } + + usage := waitForUsageAtOrBelow(t, cache, limit, 5*time.Second) + if usage > limit { + t.Errorf("periodic schedule did not trigger eviction: usage = %d, want <= %d", + usage, limit) + } + + assertNoDanglingReferences(t, cache) +} + +func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { + cache, _ := newEvictionTestCache(t, 1<<30) + + // An untracked variant file on disk (e.g. written before this + // feature existed) must be adopted into the accounting. + untracked := bytes.Repeat([]byte{0x31}, 1000) + if _, err := cache.variants.Store("aabbccdd0001", bytes.NewReader(untracked), "image/webp"); err != nil { + t.Fatalf("failed to store untracked variant file: %v", err) + } + + // An accounting row whose file is missing must be dropped. + if _, err := cache.db.Exec( + `INSERT INTO variant_content (cache_key, size_bytes, content_type) + VALUES (?, ?, ?)`, + "deadbeef0001", 700, "image/webp", + ); err != nil { + t.Fatalf("failed to insert stale variant accounting row: %v", err) + } + + cache.StartEviction(time.Hour) + defer cache.StopEviction() + + deadline := time.Now().Add(5 * time.Second) + + var usage int64 + + for time.Now().Before(deadline) { + var err error + + usage, err = cache.UsageBytes(context.Background()) + if err != nil { + t.Fatalf("UsageBytes failed: %v", err) + } + + if usage == 1000 { + break + } + + time.Sleep(25 * time.Millisecond) + } + + if usage != 1000 { + t.Errorf("usage after reconciliation = %d, want 1000 "+ + "(untracked file adopted, stale row dropped)", usage) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "aabbccdd0001", + ); n != 1 { + t.Errorf("untracked variant file was not adopted into accounting (rows=%d)", n) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, "deadbeef0001", + ); n != 0 { + t.Errorf("stale accounting row without a file was not dropped (rows=%d)", n) + } +}