Count what the cache holds in the default cache_max_bytes (closes #184)

The default limit was 75% of the space free at startup. The cache's own
files are not free space, so a fuller cache got a smaller limit after a
restart and eviction then deleted most of it. The default is now 75% of
the sum of the free space and what the cache already holds by its own
size accounting, at least 500 MiB. The cache works it out when it
opens, after the database is open, so the computation and its tests
moved from internal/config to internal/imgcache; the config only
records whether cache_max_bytes was set.

Model: opus-5-5

internal/imgcache/cache.go

@@ -37,11 +37,16 @@ type CacheConfig struct {
 	NegativeTTL time.Duration
 
 	// MaxBytes is the disk cache size limit in bytes that eviction
-	// enforces. Zero means no limit is enforced (no eviction). The
-	// config layer supplies the computed default when the operator
-	// omits cache_max_bytes.
+	// enforces. Zero means no limit is enforced (no eviction).
 	MaxBytes int64
 
+	// UseDefaultMaxBytes makes NewCache replace MaxBytes with the
+	// default limit: 75% of the sum of the space free on the filesystem
+	// holding the cache and the bytes the cache already holds, at least
+	// DefaultCacheMaxBytesFloor. The config layer sets this when the
+	// operator omits cache_max_bytes.
+	UseDefaultMaxBytes bool
+
 	// DisableDiskCache turns the disk cache off entirely: no cache
 	// directories are created, lookups always miss, stores are
 	// no-ops, and no eviction machinery runs. The config layer sets
@@ -145,6 +150,20 @@ func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) {
 	c.variants = variants
 	c.srcMetadata = srcMetadata
 
+	if config.UseDefaultMaxBytes {
+		limit, err := c.computeDefaultMaxBytes(context.Background(), defaultFreeSpaceProbe)
+		if err != nil {
+			return nil, err
+		}
+
+		c.config.MaxBytes = limit
+
+		log.Info("computed default cache size limit from free space and cache contents",
+			"cache_max_bytes", limit,
+			"cache_dir", filepath.Join(config.StateDir, "cache"),
+		)
+	}
+
 	return c, nil
 }
 

internal/imgcache/cachesize.go

@@ -0,0 +1,89 @@
+package imgcache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"path/filepath"
+	"syscall"
+)
+
+// DefaultCacheMaxBytesFloor is the minimum computed default for the
+// cache_max_bytes setting: 500 MiB. The floor applies only to the
+// computed default (when the key is omitted from the configuration),
+// never to explicitly configured values.
+const DefaultCacheMaxBytesFloor int64 = 524288000
+
+// freeSpaceFractionNumerator and freeSpaceFractionDenominator express
+// the 75% share used for the computed default limit as integer
+// arithmetic (dividing before multiplying avoids overflow).
+const (
+	freeSpaceFractionNumerator   uint64 = 3
+	freeSpaceFractionDenominator uint64 = 4
+)
+
+var errNegativeBlockSize = errors.New("statfs reported negative block size")
+
+// FreeSpaceProbeFunc reports the number of free bytes available on the
+// filesystem containing path. It is a function type so tests can
+// inject a fake probe instead of depending on the host disk.
+type FreeSpaceProbeFunc func(path string) (uint64, error)
+
+// defaultFreeSpaceProbe reports free filesystem bytes via statfs on
+// the given path, as available to unprivileged processes.
+func defaultFreeSpaceProbe(path string) (uint64, error) {
+	var stat syscall.Statfs_t
+
+	err := syscall.Statfs(path, &stat)
+	if err != nil {
+		return 0, err
+	}
+
+	if stat.Bsize < 0 {
+		return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path)
+	}
+
+	blockSize := uint64(stat.Bsize)
+
+	return stat.Bavail * blockSize, nil
+}
+
+// computeDefaultMaxBytes returns the default cache size limit: 75% of
+// the sum of the free bytes probe reports for <state_dir>/cache/ and
+// the bytes the cache already holds, with a floor of
+// DefaultCacheMaxBytesFloor. Counting what the cache holds keeps the
+// limit from shrinking as the cache fills.
+func (c *Cache) computeDefaultMaxBytes(
+	ctx context.Context, probe FreeSpaceProbeFunc,
+) (int64, error) {
+	cacheDir := filepath.Join(c.config.StateDir, "cache")
+
+	freeBytes, err := probe(cacheDir)
+	if err != nil {
+		return 0, fmt.Errorf(
+			"default cache_max_bytes: cannot determine free space for %q: %w",
+			cacheDir, err)
+	}
+
+	usedBytes, err := c.UsageBytes(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	// Both terms are at most math.MaxInt64, so the sum cannot overflow.
+	//nolint:gosec // G115: UsageBytes sums file sizes, never negative
+	spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes)
+
+	computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
+	computed = min(computed, math.MaxInt64)
+
+	// gosec cannot see that min() above bounds computed, so it reads
+	// this conversion as potentially overflowing. It cannot: computed is
+	// at most math.MaxInt64 on every path here.
+	//nolint:gosec // G115: clamped to MaxInt64 by min above
+	limit := int64(computed)
+	limit = max(limit, DefaultCacheMaxBytesFloor)
+
+	return limit, nil
+}

internal/imgcache/cachesize_internal_test.go

@@ -0,0 +1,178 @@
+package imgcache
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// errTestStatfsFailed is returned by the failing free-space probe below.
+var errTestStatfsFailed = errors.New("statfs failed")
+
+// TestComputeDefaultMaxBytesCountsWhatTheCacheHolds verifies that the
+// default limit is 75% of the free space plus what the cache already
+// holds, so a cache filled to its limit keeps that limit across a
+// restart instead of shrinking to 75% of the space left free.
+func TestComputeDefaultMaxBytesCountsWhatTheCacheHolds(t *testing.T) {
+	t.Parallel()
+
+	cache, _ := newEvictionTestCache(t, 1<<30)
+
+	// Empty cache, 4 GiB free -> 3 GiB default.
+	got, err := cache.computeDefaultMaxBytes(t.Context(),
+		func(string) (uint64, error) { return 4294967296, nil })
+	if err != nil {
+		t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
+	}
+
+	if got != 3221225472 {
+		t.Errorf("default for an empty cache = %d, want 3221225472 (75%% of 4 GiB)",
+			got)
+	}
+
+	// The cache now holds those 3 GiB, which leaves 1 GiB free.
+	_, err = cache.db.ExecContext(t.Context(),
+		`INSERT INTO variant_content (cache_key, size_bytes, content_type)
+		 VALUES (?, ?, ?)`,
+		string(testVariantKeyOne), 3221225472, testContentTypeWebP,
+	)
+	if err != nil {
+		t.Fatalf("failed to insert variant accounting row: %v", err)
+	}
+
+	got, err = cache.computeDefaultMaxBytes(t.Context(),
+		func(string) (uint64, error) { return 1073741824, nil })
+	if err != nil {
+		t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
+	}
+
+	if got != 3221225472 {
+		t.Errorf("default for a cache holding 3 GiB with 1 GiB free = %d, "+
+			"want 3221225472 (75%% of 1 GiB + 3 GiB)", got)
+	}
+}
+
+// TestComputeDefaultMaxBytesAppliesFloor verifies that when 75% of the
+// free space plus what the cache holds is below 500 MiB, the default
+// is floored at DefaultCacheMaxBytesFloor.
+func TestComputeDefaultMaxBytesAppliesFloor(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name      string
+		freeBytes uint64
+	}{
+		{name: "100 MiB free", freeBytes: 104857600},
+		{name: "zero free", freeBytes: 0},
+		{name: "just below floor threshold", freeBytes: 699050665},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			cache, _ := newEvictionTestCache(t, 1<<30)
+
+			got, err := cache.computeDefaultMaxBytes(t.Context(),
+				func(string) (uint64, error) { return tc.freeBytes, nil })
+			if err != nil {
+				t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
+			}
+
+			if got != DefaultCacheMaxBytesFloor {
+				t.Errorf("computeDefaultMaxBytes = %d, want floor %d",
+					got, DefaultCacheMaxBytesFloor)
+			}
+		})
+	}
+}
+
+// TestComputeDefaultMaxBytesPropagatesProbeError verifies that a
+// failing free-space probe produces an error naming cache_max_bytes,
+// instead of a silently wrong default.
+func TestComputeDefaultMaxBytesPropagatesProbeError(t *testing.T) {
+	t.Parallel()
+
+	cache, _ := newEvictionTestCache(t, 1<<30)
+
+	_, err := cache.computeDefaultMaxBytes(t.Context(),
+		func(string) (uint64, error) { return 0, errTestStatfsFailed })
+	if err == nil {
+		t.Fatal("probe failure must produce an error, got nil")
+	}
+
+	t.Logf("got expected error: %v", err)
+
+	if !strings.Contains(err.Error(), "cache_max_bytes") {
+		t.Errorf("error %q does not name cache_max_bytes", err.Error())
+	}
+}
+
+// TestComputeDefaultMaxBytesProbesTheCacheDirectory verifies that the
+// probe is pointed at <state_dir>/cache/ and that the directory exists
+// by then, so statfs measures the filesystem that holds the cache.
+func TestComputeDefaultMaxBytesProbesTheCacheDirectory(t *testing.T) {
+	t.Parallel()
+
+	cache, stateDir := newEvictionTestCache(t, 1<<30)
+	wantCacheDir := filepath.Join(stateDir, "cache")
+
+	var probedPath string
+
+	_, err := cache.computeDefaultMaxBytes(t.Context(),
+		func(path string) (uint64, error) {
+			probedPath = path
+
+			info, err := os.Stat(path)
+			if err != nil || !info.IsDir() {
+				t.Errorf("cache directory %q missing when probed: info=%v err=%v",
+					path, info, err)
+			}
+
+			return 4294967296, nil
+		})
+	if err != nil {
+		t.Fatalf("computeDefaultMaxBytes returned error: %v", err)
+	}
+
+	if probedPath != wantCacheDir {
+		t.Errorf("free space probed at %q, want cache directory %q",
+			probedPath, wantCacheDir)
+	}
+}
+
+// TestNewCacheUsesDefaultMaxBytesOnlyWhenAsked verifies that NewCache
+// replaces MaxBytes with the computed default when UseDefaultMaxBytes
+// is set, and keeps MaxBytes as given otherwise.
+func TestNewCacheUsesDefaultMaxBytesOnlyWhenAsked(t *testing.T) {
+	t.Parallel()
+
+	withDefault, err := NewCache(evictionTestDB(t), CacheConfig{
+		StateDir:           t.TempDir(),
+		MaxBytes:           1024,
+		UseDefaultMaxBytes: true,
+	})
+	if err != nil {
+		t.Fatalf("NewCache with UseDefaultMaxBytes failed: %v", err)
+	}
+
+	if withDefault.config.MaxBytes < DefaultCacheMaxBytesFloor {
+		t.Errorf("MaxBytes with UseDefaultMaxBytes = %d, want the default, "+
+			"at least %d", withDefault.config.MaxBytes, DefaultCacheMaxBytesFloor)
+	}
+
+	explicit, err := NewCache(evictionTestDB(t), CacheConfig{
+		StateDir: t.TempDir(),
+		MaxBytes: 1024,
+	})
+	if err != nil {
+		t.Fatalf("NewCache failed: %v", err)
+	}
+
+	if explicit.config.MaxBytes != 1024 {
+		t.Errorf("MaxBytes without UseDefaultMaxBytes = %d, want 1024",
+			explicit.config.MaxBytes)
+	}
+}