Eviction no longer reads a whole table while requests wait on the database (closes #227)

UsageBytes now reads the new cache_usage row, which triggers on
source_content and variant_content keep up to date in the statement
that adds, removes or resizes a row. The reconciliation pass reads both
tables 1000 rows per query, sums them, and corrects the total when it
differs, unless a row changed while it summed. Source rows now get
last_accessed_at when added, so choosing source images to evict reads
that column's index instead of sorting the whole table. Stats still
sums the tables, now in pages: an existing test drops both tables and
expects that sum to fail.

Model: opus-5-5

Dockerfile

@@ -22,8 +22,9 @@ FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66
 
 WORKDIR /src
 
-# script/bootstrap --cgo installs the build dependencies (a C compiler
-# and the libvips and libheif headers) and downloads the Go modules.
+# script/bootstrap --cgo installs the build dependencies (a C compiler,
+# the libvips and libheif headers, and libvips' JPEG XL support, which
+# the tests need) and downloads the Go modules.
 COPY script/ ./script/
 COPY go.mod go.sum ./
 RUN script/bootstrap --cgo
@@ -80,9 +81,11 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
 # alpine:3.21, 2026-02-25
 FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
 
-# Install runtime dependencies only
+# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
+# support, without which pixad does not start.
 RUN apk add --no-cache \
     vips \
+    vips-jxl \
     libheif \
     ca-certificates \
     tzdata \

README.md

@@ -89,7 +89,10 @@ another part of pixa failed to stop. A request not finished by then is cut off.
 `docker stop` waits 10 seconds before it kills the container.
 
 Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
-it uses libvips through CGO; building it also needs their development files,
+it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL
+support, which on Alpine is the `vips-jxl` package and which the nix and brew
+packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
+on. Building pixa also needs the development files of libvips and libheif,
 `pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
 as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
 `script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
@@ -583,8 +586,8 @@ provide:
 
 - `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and
   download the Go modules (idempotent); with `--cgo`, the C compiler and the
-  libvips and libheif libraries that compiling pixa needs instead of Node, Yarn
-  and prettier
+  libvips (with its JPEG XL support) and libheif libraries that compiling and
+  testing pixa need instead of Node, Yarn and prettier
 - `script/setup` — make a fresh clone ready for development (bootstrap, then
   install-precommit)
 - `script/projectname` — output the project name ("pixa")

TODO.md

@@ -30,6 +30,14 @@ P2: security: per-IP rate limiting on the image routes
 
 # Completed Steps
 
+- 2026-10-08 libvips' JPEG XL support is installed and required (part of #222):
+  `script/bootstrap --cgo` installs `vips-jxl` when its package manager is apk,
+  as Alpine's `vips` package lacks the support, and the runtime stage of the
+  `Dockerfile` installs it too. `imgcache.NewService` fails, naming the fix,
+  when `imageprocessor.CheckJPEGXLSupport` finds that libvips cannot load and
+  save JPEG XL, so pixad does not start without it. A test saves an image as
+  JPEG XL with govips and loads it back. JPEG XL is not yet a format pixa
+  serves.
 - 2026-10-08 requests no longer wait behind eviction queries that read a whole
   table (closes #227): the new `cache_usage` table holds the total cache usage,
   kept up to date by triggers on `source_content` and `variant_content` in the

internal/imageprocessor/imageprocessor.go

@@ -37,6 +37,24 @@ func initVips() {
 	})
 }
 
+// errNoJPEGXL is returned by CheckJPEGXLSupport.
+var errNoJPEGXL = errors.New("libvips lacks JPEG XL support: install " +
+	"vips-jxl on Alpine, or use a libvips built with libjxl")
+
+// CheckJPEGXLSupport returns an error, naming the fix, when libvips
+// cannot load and save JPEG XL.
+func CheckJPEGXLSupport() error {
+	initVips()
+
+	// govips counts a format as supported when libvips has its loader;
+	// libvips builds the JPEG XL loader and saver together.
+	if !vips.IsTypeSupported(vips.ImageTypeJXL) {
+		return errNoJPEGXL
+	}
+
+	return nil
+}
+
 // Format represents supported output image formats.
 type Format string
 

internal/imageprocessor/jpegxl_internal_test.go

@@ -0,0 +1,52 @@
+package imageprocessor
+
+import (
+	"testing"
+
+	"github.com/davidbyttow/govips/v2/vips"
+)
+
+// TestCheckJPEGXLSupport fails when libvips lacks JPEG XL support, as on
+// Alpine without the vips-jxl package.
+func TestCheckJPEGXLSupport(t *testing.T) {
+	t.Parallel()
+
+	err := CheckJPEGXLSupport()
+	if err != nil {
+		t.Fatalf("CheckJPEGXLSupport() error = %v", err)
+	}
+}
+
+// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and
+// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine
+// without the vips-jxl package.
+func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) {
+	t.Parallel()
+
+	img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48))
+	if err != nil {
+		t.Fatalf("failed to load test JPEG: %v", err)
+	}
+
+	defer img.Close()
+
+	jxl, _, err := img.ExportJxl(vips.NewJxlExportParams())
+	if err != nil {
+		t.Fatalf("ExportJxl() error = %v", err)
+	}
+
+	loaded, err := vips.NewImageFromBuffer(jxl)
+	if err != nil {
+		t.Fatalf("failed to load the JPEG XL image: %v", err)
+	}
+
+	defer loaded.Close()
+
+	if loaded.Format() != vips.ImageTypeJXL {
+		t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()])
+	}
+
+	if loaded.Width() != 64 || loaded.Height() != 48 {
+		t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height())
+	}
+}

internal/imgcache/cache.go

@@ -96,6 +96,17 @@ type Cache struct {
 	// deterministically pause inside that window to exercise
 	// concurrent stores against it; production code leaves it nil.
 	evictSourceBlobTestHook func(ContentHash)
+
+	// reconciliationPageSize is the most rows one read of a content table
+	// returns in the reconciliation pass and in Stats. newCache sets it to
+	// defaultReconciliationPageSize; tests set it smaller.
+	reconciliationPageSize int
+
+	// reconciliationReadTestHook, when set, is called after each of those
+	// reads with the number of rows the read covered, so tests can check
+	// that no read covers more than one page; production code leaves it
+	// nil.
+	reconciliationReadTestHook func(rows int)
 }
 
 // NewCache creates a new cache instance.
@@ -127,6 +138,8 @@ func newCache(
 		evictionDone:     make(chan struct{}),
 		metaCache:        metaCache,
 		contentLocks:     newContentLock(),
+
+		reconciliationPageSize: defaultReconciliationPageSize,
 	}
 
 	if c.disabled {

internal/imgcache/cache_usage_internal_test.go

@@ -2,7 +2,7 @@ package imgcache
 
 import (
 	"bytes"
-	"fmt"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -148,64 +148,43 @@ func TestUsageTotalNotCorrectedFromAnOlderSum(t *testing.T) {
 	}
 }
 
-// TestReconciliationReadsAPageAtATime adds one row more than a page to
-// each content table, none of them with a file. Each reconciliation read
-// must return one page at most, so a request's query waits for one page
-// at most, and the pass must still reach every row: it sums all of them
-// and drops all of them.
+// TestReconciliationReadsAPageAtATime stores five source images and five
+// variants, sets the page size to two rows and runs a reconciliation
+// pass. No read the pass makes of a content table, to check its rows or
+// to sum them, may cover more than two rows, so a request's query waits
+// for one page at most. Between them the reads must still cover every
+// row of both tables twice, once to check it and once to sum it, and the
+// sum must put a wrong total right.
 func TestReconciliationReadsAPageAtATime(t *testing.T) {
 	t.Parallel()
 
 	cache, _ := newEvictionTestCache(t, 1<<30)
 	ctx := t.Context()
 
-	const rowsPerTable = reconciliationPageSize + 1
+	const pageSize = 2
 
-	for i := range rowsPerTable {
-		_, err := cache.db.ExecContext(ctx, `
-			INSERT INTO variant_content (cache_key, size_bytes, content_type)
-			VALUES (?, 1, ?)
-		`, fmt.Sprintf("%012x", i), testContentTypeWebP)
+	cache.reconciliationPageSize = pageSize
+
+	// Sources of 100 bytes and variants of 10, five of each.
+	for i := range 5 {
+		content := []byte(strconv.Itoa(i))
+
+		storeEvictionTestSource(t, cache, "pages.example.com",
+			"/"+strconv.Itoa(i)+".jpg", bytes.Repeat(content, 100))
+		storeEvictionTestVariant(t, cache, VariantKey("aabbccdd000"+strconv.Itoa(i)),
+			bytes.Repeat(content, 10))
+	}
+
+	_, err := cache.db.ExecContext(ctx,
+		`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
 	if err != nil {
-			t.Fatalf("failed to insert variant row %d: %v", i, err)
+		t.Fatalf("failed to set a wrong total: %v", err)
 	}
 
-		_, err = cache.db.ExecContext(ctx, `
-			INSERT INTO source_content (content_hash, content_type, size_bytes)
-			VALUES (?, ?, 1)
-		`, fmt.Sprintf("%064x", i), testContentTypeJPEG)
-		if err != nil {
-			t.Fatalf("failed to insert source row %d: %v", i, err)
-		}
-	}
+	var rowsPerRead []int
 
-	keys, err := cache.variantKeysAfter(ctx, "")
-	if err != nil {
-		t.Fatalf("variantKeysAfter failed: %v", err)
-	}
-
-	if len(keys) != reconciliationPageSize {
-		t.Errorf("one read returned %d variant keys, want %d",
-			len(keys), reconciliationPageSize)
-	}
-
-	hashes, err := cache.sourceContentHashesAfter(ctx, "")
-	if err != nil {
-		t.Fatalf("sourceContentHashesAfter failed: %v", err)
-	}
-
-	if len(hashes) != reconciliationPageSize {
-		t.Errorf("one read returned %d source hashes, want %d",
-			len(hashes), reconciliationPageSize)
-	}
-
-	sum, err := cache.sumContentSizeBytes(ctx)
-	if err != nil {
-		t.Fatalf("sumContentSizeBytes failed: %v", err)
-	}
-
-	if sum != 2*rowsPerTable {
-		t.Errorf("sumContentSizeBytes() = %d, want %d", sum, 2*rowsPerTable)
+	cache.reconciliationReadTestHook = func(rows int) {
+		rowsPerRead = append(rowsPerRead, rows)
 	}
 
 	err = cache.reconcileAccounting(ctx)
@@ -213,12 +192,31 @@ func TestReconciliationReadsAPageAtATime(t *testing.T) {
 		t.Fatalf("reconcileAccounting failed: %v", err)
 	}
 
-	if n := countRows(t, cache, `SELECT COUNT(*) FROM variant_content`); n != 0 {
-		t.Errorf("%d variant rows without a file are left, want 0", n)
+	t.Logf("rows covered by each read, in order: %v", rowsPerRead)
+
+	coveredRows := 0
+
+	for _, rows := range rowsPerRead {
+		if rows > pageSize {
+			t.Errorf("a read covered %d rows, want at most %d", rows, pageSize)
 		}
 
-	if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 {
-		t.Errorf("%d source rows without a file are left, want 0", n)
+		coveredRows += rows
+	}
+
+	// Ten rows, each read once to check it and once to sum it.
+	if coveredRows != 20 {
+		t.Errorf("the reads covered %d rows in all, want 20", coveredRows)
+	}
+
+	usage, err := cache.UsageBytes(ctx)
+	if err != nil {
+		t.Fatalf("UsageBytes failed: %v", err)
+	}
+
+	if usage != 550 {
+		t.Errorf("UsageBytes() after reconciliation = %d, want 550 (5*100 + 5*10)",
+			usage)
 	}
 }
 

internal/imgcache/cachesize.go

@@ -72,7 +72,7 @@ func (c *Cache) computeDefaultMaxBytes(
 	}
 
 	// Both terms are at most math.MaxInt64, so the sum cannot overflow.
-	//nolint:gosec // G115: UsageBytes sums file sizes, never negative
+	//nolint:gosec // G115: UsageBytes returns the total cache usage, never negative
 	spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes)
 
 	computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator

internal/imgcache/eviction.go

@@ -21,10 +21,11 @@ const DefaultEvictionInterval = 5 * time.Minute
 // and source blobs) one eviction pass fetches from the database.
 const evictionBatchSize = 100
 
-// reconciliationPageSize is the most rows one read of the reconciliation
-// pass returns. Each read is a query of its own, so a request waits for
-// one page at most, however large the cache is.
-const reconciliationPageSize = 1000
+// defaultReconciliationPageSize is the most rows one read of the
+// reconciliation pass returns, unless a test sets
+// Cache.reconciliationPageSize smaller. Each read is a query of its own,
+// so a request waits for one page at most, however large the cache is.
+const defaultReconciliationPageSize = 1000
 
 // staleTempFileAge is how old an orphaned temp file (left behind by a
 // crashed write) must be before reconciliation removes it. Fresh temp
@@ -676,6 +677,10 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error {
 			return err
 		}
 
+		if c.reconciliationReadTestHook != nil {
+			c.reconciliationReadTestHook(len(keys))
+		}
+
 		if len(keys) == 0 {
 			return nil
 		}
@@ -702,7 +707,7 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error {
 	}
 }
 
-// variantKeysAfter returns, in order, up to reconciliationPageSize
+// variantKeysAfter returns, in order, up to c.reconciliationPageSize
 // tracked variant cache keys that sort after the given one.
 func (c *Cache) variantKeysAfter(
 	ctx context.Context, after VariantKey,
@@ -710,7 +715,7 @@ func (c *Cache) variantKeysAfter(
 	return queryStringColumn[VariantKey](ctx, c.db, `
 		SELECT cache_key FROM variant_content
 		WHERE cache_key > ? ORDER BY cache_key LIMIT ?
-	`, "variant keys", "variant key", string(after), reconciliationPageSize)
+	`, "variant keys", "variant key", string(after), c.reconciliationPageSize)
 }
 
 // queryStringColumn runs a single-column query with args and returns the
@@ -823,6 +828,10 @@ func (c *Cache) reconcileSourceRows(ctx context.Context) error {
 			return err
 		}
 
+		if c.reconciliationReadTestHook != nil {
+			c.reconciliationReadTestHook(len(hashes))
+		}
+
 		if len(hashes) == 0 {
 			return nil
 		}
@@ -851,7 +860,7 @@ func (c *Cache) reconcileSourceRows(ctx context.Context) error {
 }
 
 // sourceContentHashesAfter returns, in order, up to
-// reconciliationPageSize tracked source content hashes that sort after
+// c.reconciliationPageSize tracked source content hashes that sort after
 // the given one.
 func (c *Cache) sourceContentHashesAfter(
 	ctx context.Context, after ContentHash,
@@ -859,20 +868,24 @@ func (c *Cache) sourceContentHashesAfter(
 	return queryStringColumn[ContentHash](ctx, c.db, `
 		SELECT content_hash FROM source_content
 		WHERE content_hash > ? ORDER BY content_hash LIMIT ?
-	`, "source content hashes", "content hash", string(after), reconciliationPageSize)
+	`, "source content hashes", "content hash", string(after),
+		c.reconciliationPageSize)
 }
 
 // Each of these queries sums size_bytes over the next page of rows of
 // one content table, the rows that sort after a key, and returns the
-// page's last key with the sum. Past the last row the key is NULL.
+// page's last key, the sum and the number of rows in the page. Past the
+// last row the page has no rows and the key is NULL.
 const (
 	sourceSizePageQuery = `
-		SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0) FROM (
+		SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0), COUNT(*)
+		FROM (
 			SELECT content_hash, size_bytes FROM source_content
 			WHERE content_hash > ? ORDER BY content_hash LIMIT ?
 		)`
 	variantSizePageQuery = `
-		SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0) FROM (
+		SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0), COUNT(*)
+		FROM (
 			SELECT cache_key, size_bytes FROM variant_content
 			WHERE cache_key > ? ORDER BY cache_key LIMIT ?
 		)`
@@ -908,13 +921,19 @@ func (c *Cache) sumSizeBytesInPages(
 
 		var pageBytes int64
 
-		err := c.db.QueryRowContext(ctx, pageQuery, after, reconciliationPageSize).
-			Scan(&lastKey, &pageBytes)
+		var pageRows int
+
+		err := c.db.QueryRowContext(ctx, pageQuery, after, c.reconciliationPageSize).
+			Scan(&lastKey, &pageBytes, &pageRows)
 		if err != nil {
 			return 0, fmt.Errorf("failed to sum cache content sizes: %w", err)
 		}
 
-		if !lastKey.Valid {
+		if c.reconciliationReadTestHook != nil {
+			c.reconciliationReadTestHook(pageRows)
+		}
+
+		if pageRows == 0 {
 			return total, nil
 		}
 

internal/imgcache/service.go

@@ -100,6 +100,13 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
 		allowHTTP = cfg.FetcherConfig.AllowHTTP
 	}
 
+	// JPEG XL is to become the default output format, so pixad does not
+	// start without it.
+	err := imageprocessor.CheckJPEGXLSupport()
+	if err != nil {
+		return nil, err
+	}
+
 	maxResponseSize := fetcherCfg.MaxResponseSize
 	processor := imageprocessor.New(imageprocessor.Params{
 		MaxInputBytes:           maxResponseSize,

script/bootstrap

@@ -14,11 +14,12 @@
 #                            script/fmt-check: all the host needs, as
 #                            the checks compile pixa in Docker
 #   script/bootstrap --cgo   git, make, Go, and a C compiler and the
-#                            CGO image libraries (pkg-config, vips,
-#                            libheif) for the govips bindings instead
-#                            of Node: to compile pixa, in the
-#                            Dockerfile's test phase and build stage,
-#                            which format nothing
+#                            CGO image libraries (pkg-config, vips
+#                            with its JPEG XL support, libheif) for
+#                            the govips bindings instead of Node: to
+#                            compile pixa, in the Dockerfile's test
+#                            phase and build stage, which format
+#                            nothing
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -150,6 +151,16 @@ ensure_cgo_deps() {
     if ! pkg-config --exists vips; then
         pkg_install vips libvips-dev vips vips-dev
     fi
+    # libvips' JPEG XL loader and saver are in the nix and brew vips
+    # packages, and in apt's from Debian 12 and Ubuntu 24.04 on, but in
+    # the package vips-jxl on Alpine. detect_pkgmgr is called only where
+    # apk exists, as on apt it updates the package lists.
+    if ! missing apk; then
+        detect_pkgmgr
+    fi
+    if [ "$PKGMGR" = "apk" ] && ! apk info -e vips-jxl >/dev/null; then
+        apk add --no-cache vips-jxl
+    fi
     if ! pkg-config --exists libheif; then
         pkg_install libheif libheif-dev libheif libheif-dev
     fi