Eviction no longer reads a whole table while requests wait on the database (closes #227)

UsageBytes now reads the new cache_usage row, which triggers on
source_content and variant_content keep up to date in the statement
that adds, removes or resizes a row. The reconciliation pass reads both
tables 1000 rows per query, sums them, and corrects the total when it
differs, unless a row changed while it summed. Source rows now get
last_accessed_at when added, so choosing source images to evict reads
that column's index instead of sorting the whole table. Stats still
sums the tables, now in pages: an existing test drops both tables and
expects that sum to fail.

Model: opus-5-5

Dockerfile

@@ -22,9 +22,8 @@ FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66
 
 WORKDIR /src
 
-# script/bootstrap --cgo installs the build dependencies (a C compiler,
+# script/bootstrap --cgo installs the build dependencies (a C compiler
-# the libvips and libheif headers, and libvips' JPEG XL support, which
+# and the libvips and libheif headers) and downloads the Go modules.
-# the tests need) and downloads the Go modules.
 COPY script/ ./script/
 COPY go.mod go.sum ./
 RUN script/bootstrap --cgo
@@ -81,11 +80,9 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
 # alpine:3.21, 2026-02-25
 FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
 
-# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
+# Install runtime dependencies only
-# support, without which pixad does not start.
 RUN apk add --no-cache \
     vips \
-    vips-jxl \
     libheif \
     ca-certificates \
     tzdata \

README.md

@@ -89,10 +89,7 @@ another part of pixa failed to stop. A request not finished by then is cut off.
 `docker stop` waits 10 seconds before it kills the container.
 
 Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
-it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL
+it uses libvips through CGO; building it also needs their development files,
-support, which on Alpine is the `vips-jxl` package and which the nix and brew
-packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
-on. Building pixa also needs the development files of libvips and libheif,
 `pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
 as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
 `script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
@@ -586,8 +583,8 @@ provide:
 
 - `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and
   download the Go modules (idempotent); with `--cgo`, the C compiler and the
-  libvips (with its JPEG XL support) and libheif libraries that compiling and
+  libvips and libheif libraries that compiling pixa needs instead of Node, Yarn
-  testing pixa need instead of Node, Yarn and prettier
+  and prettier
 - `script/setup` — make a fresh clone ready for development (bootstrap, then
   install-precommit)
 - `script/projectname` — output the project name ("pixa")

TODO.md

@@ -30,14 +30,6 @@ P2: security: per-IP rate limiting on the image routes
 
 # Completed Steps
 
-- 2026-10-08 libvips' JPEG XL support is installed and required (part of #222):
-  `script/bootstrap --cgo` installs `vips-jxl` when its package manager is apk,
-  as Alpine's `vips` package lacks the support, and the runtime stage of the
-  `Dockerfile` installs it too. `imgcache.NewService` fails, naming the fix,
-  when `imageprocessor.CheckJPEGXLSupport` finds that libvips cannot load and
-  save JPEG XL, so pixad does not start without it. A test saves an image as
-  JPEG XL with govips and loads it back. JPEG XL is not yet a format pixa
-  serves.
 - 2026-10-08 requests no longer wait behind eviction queries that read a whole
   table (closes #227): the new `cache_usage` table holds the total cache usage,
   kept up to date by triggers on `source_content` and `variant_content` in the

internal/imageprocessor/imageprocessor.go

@@ -37,24 +37,6 @@ func initVips() {
 	})
 }
 
-// errNoJPEGXL is returned by CheckJPEGXLSupport.
-var errNoJPEGXL = errors.New("libvips lacks JPEG XL support: install " +
-	"vips-jxl on Alpine, or use a libvips built with libjxl")
-
-// CheckJPEGXLSupport returns an error, naming the fix, when libvips
-// cannot load and save JPEG XL.
-func CheckJPEGXLSupport() error {
-	initVips()
-
-	// govips counts a format as supported when libvips has its loader;
-	// libvips builds the JPEG XL loader and saver together.
-	if !vips.IsTypeSupported(vips.ImageTypeJXL) {
-		return errNoJPEGXL
-	}
-
-	return nil
-}
-
 // Format represents supported output image formats.
 type Format string
 

internal/imageprocessor/jpegxl_internal_test.go

@@ -1,52 +0,0 @@
-package imageprocessor
-
-import (
-	"testing"
-
-	"github.com/davidbyttow/govips/v2/vips"
-)
-
-// TestCheckJPEGXLSupport fails when libvips lacks JPEG XL support, as on
-// Alpine without the vips-jxl package.
-func TestCheckJPEGXLSupport(t *testing.T) {
-	t.Parallel()
-
-	err := CheckJPEGXLSupport()
-	if err != nil {
-		t.Fatalf("CheckJPEGXLSupport() error = %v", err)
-	}
-}
-
-// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and
-// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine
-// without the vips-jxl package.
-func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) {
-	t.Parallel()
-
-	img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48))
-	if err != nil {
-		t.Fatalf("failed to load test JPEG: %v", err)
-	}
-
-	defer img.Close()
-
-	jxl, _, err := img.ExportJxl(vips.NewJxlExportParams())
-	if err != nil {
-		t.Fatalf("ExportJxl() error = %v", err)
-	}
-
-	loaded, err := vips.NewImageFromBuffer(jxl)
-	if err != nil {
-		t.Fatalf("failed to load the JPEG XL image: %v", err)
-	}
-
-	defer loaded.Close()
-
-	if loaded.Format() != vips.ImageTypeJXL {
-		t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()])
-	}
-
-	if loaded.Width() != 64 || loaded.Height() != 48 {
-		t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height())
-	}
-}

internal/imgcache/cache.go

@@ -96,17 +96,6 @@ type Cache struct {
 	// deterministically pause inside that window to exercise
 	// concurrent stores against it; production code leaves it nil.
 	evictSourceBlobTestHook func(ContentHash)
-
-	// reconciliationPageSize is the most rows one read of a content table
-	// returns in the reconciliation pass and in Stats. newCache sets it to
-	// defaultReconciliationPageSize; tests set it smaller.
-	reconciliationPageSize int
-
-	// reconciliationReadTestHook, when set, is called after each of those
-	// reads with the number of rows the read covered, so tests can check
-	// that no read covers more than one page; production code leaves it
-	// nil.
-	reconciliationReadTestHook func(rows int)
 }
 
 // NewCache creates a new cache instance.
@@ -138,8 +127,6 @@ func newCache(
 		evictionDone:     make(chan struct{}),
 		metaCache:        metaCache,
 		contentLocks:     newContentLock(),
-
-		reconciliationPageSize: defaultReconciliationPageSize,
 	}
 
 	if c.disabled {

internal/imgcache/cache_usage_internal_test.go

@@ -2,7 +2,7 @@ package imgcache
 
 import (
 	"bytes"
-	"strconv"
+	"fmt"
 	"strings"
 	"testing"
 )
@@ -148,43 +148,64 @@ func TestUsageTotalNotCorrectedFromAnOlderSum(t *testing.T) {
 	}
 }
 
-// TestReconciliationReadsAPageAtATime stores five source images and five
+// TestReconciliationReadsAPageAtATime adds one row more than a page to
-// variants, sets the page size to two rows and runs a reconciliation
+// each content table, none of them with a file. Each reconciliation read
-// pass. No read the pass makes of a content table, to check its rows or
+// must return one page at most, so a request's query waits for one page
-// to sum them, may cover more than two rows, so a request's query waits
+// at most, and the pass must still reach every row: it sums all of them
-// for one page at most. Between them the reads must still cover every
+// and drops all of them.
-// row of both tables twice, once to check it and once to sum it, and the
-// sum must put a wrong total right.
 func TestReconciliationReadsAPageAtATime(t *testing.T) {
 	t.Parallel()
 
 	cache, _ := newEvictionTestCache(t, 1<<30)
 	ctx := t.Context()
 
-	const pageSize = 2
+	const rowsPerTable = reconciliationPageSize + 1
 
-	cache.reconciliationPageSize = pageSize
+	for i := range rowsPerTable {
+		_, err := cache.db.ExecContext(ctx, `
+			INSERT INTO variant_content (cache_key, size_bytes, content_type)
+			VALUES (?, 1, ?)
+		`, fmt.Sprintf("%012x", i), testContentTypeWebP)
+		if err != nil {
+			t.Fatalf("failed to insert variant row %d: %v", i, err)
+		}
 
-	// Sources of 100 bytes and variants of 10, five of each.
+		_, err = cache.db.ExecContext(ctx, `
-	for i := range 5 {
+			INSERT INTO source_content (content_hash, content_type, size_bytes)
-		content := []byte(strconv.Itoa(i))
+			VALUES (?, ?, 1)
-
+		`, fmt.Sprintf("%064x", i), testContentTypeJPEG)
-		storeEvictionTestSource(t, cache, "pages.example.com",
+		if err != nil {
-			"/"+strconv.Itoa(i)+".jpg", bytes.Repeat(content, 100))
+			t.Fatalf("failed to insert source row %d: %v", i, err)
-		storeEvictionTestVariant(t, cache, VariantKey("aabbccdd000"+strconv.Itoa(i)),
+		}
-			bytes.Repeat(content, 10))
 	}
 
-	_, err := cache.db.ExecContext(ctx,
+	keys, err := cache.variantKeysAfter(ctx, "")
-		`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
 	if err != nil {
-		t.Fatalf("failed to set a wrong total: %v", err)
+		t.Fatalf("variantKeysAfter failed: %v", err)
 	}
 
-	var rowsPerRead []int
+	if len(keys) != reconciliationPageSize {
+		t.Errorf("one read returned %d variant keys, want %d",
+			len(keys), reconciliationPageSize)
+	}
 
-	cache.reconciliationReadTestHook = func(rows int) {
+	hashes, err := cache.sourceContentHashesAfter(ctx, "")
-		rowsPerRead = append(rowsPerRead, rows)
+	if err != nil {
+		t.Fatalf("sourceContentHashesAfter failed: %v", err)
+	}
+
+	if len(hashes) != reconciliationPageSize {
+		t.Errorf("one read returned %d source hashes, want %d",
+			len(hashes), reconciliationPageSize)
+	}
+
+	sum, err := cache.sumContentSizeBytes(ctx)
+	if err != nil {
+		t.Fatalf("sumContentSizeBytes failed: %v", err)
+	}
+
+	if sum != 2*rowsPerTable {
+		t.Errorf("sumContentSizeBytes() = %d, want %d", sum, 2*rowsPerTable)
 	}
 
 	err = cache.reconcileAccounting(ctx)
@@ -192,31 +213,12 @@ func TestReconciliationReadsAPageAtATime(t *testing.T) {
 		t.Fatalf("reconcileAccounting failed: %v", err)
 	}
 
-	t.Logf("rows covered by each read, in order: %v", rowsPerRead)
+	if n := countRows(t, cache, `SELECT COUNT(*) FROM variant_content`); n != 0 {
-
+		t.Errorf("%d variant rows without a file are left, want 0", n)
-	coveredRows := 0
-
-	for _, rows := range rowsPerRead {
-		if rows > pageSize {
-			t.Errorf("a read covered %d rows, want at most %d", rows, pageSize)
-		}
-
-		coveredRows += rows
 	}
 
-	// Ten rows, each read once to check it and once to sum it.
+	if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 {
-	if coveredRows != 20 {
+		t.Errorf("%d source rows without a file are left, want 0", n)
-		t.Errorf("the reads covered %d rows in all, want 20", coveredRows)
-	}
-
-	usage, err := cache.UsageBytes(ctx)
-	if err != nil {
-		t.Fatalf("UsageBytes failed: %v", err)
-	}
-
-	if usage != 550 {
-		t.Errorf("UsageBytes() after reconciliation = %d, want 550 (5*100 + 5*10)",
-			usage)
 	}
 }
 

internal/imgcache/cachesize.go

@@ -72,7 +72,7 @@ func (c *Cache) computeDefaultMaxBytes(
 	}
 
 	// Both terms are at most math.MaxInt64, so the sum cannot overflow.
-	//nolint:gosec // G115: UsageBytes returns the total cache usage, never negative
+	//nolint:gosec // G115: UsageBytes sums file sizes, never negative
 	spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes)
 
 	computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator

internal/imgcache/eviction.go

@@ -21,11 +21,10 @@ const DefaultEvictionInterval = 5 * time.Minute
 // and source blobs) one eviction pass fetches from the database.
 const evictionBatchSize = 100
 
-// defaultReconciliationPageSize is the most rows one read of the
+// reconciliationPageSize is the most rows one read of the reconciliation
-// reconciliation pass returns, unless a test sets
+// pass returns. Each read is a query of its own, so a request waits for
-// Cache.reconciliationPageSize smaller. Each read is a query of its own,
+// one page at most, however large the cache is.
-// so a request waits for one page at most, however large the cache is.
+const reconciliationPageSize = 1000
-const defaultReconciliationPageSize = 1000
 
 // staleTempFileAge is how old an orphaned temp file (left behind by a
 // crashed write) must be before reconciliation removes it. Fresh temp
@@ -677,10 +676,6 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error {
 			return err
 		}
 
-		if c.reconciliationReadTestHook != nil {
-			c.reconciliationReadTestHook(len(keys))
-		}
-
 		if len(keys) == 0 {
 			return nil
 		}
@@ -707,7 +702,7 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error {
 	}
 }
 
-// variantKeysAfter returns, in order, up to c.reconciliationPageSize
+// variantKeysAfter returns, in order, up to reconciliationPageSize
 // tracked variant cache keys that sort after the given one.
 func (c *Cache) variantKeysAfter(
 	ctx context.Context, after VariantKey,
@@ -715,7 +710,7 @@ func (c *Cache) variantKeysAfter(
 	return queryStringColumn[VariantKey](ctx, c.db, `
 		SELECT cache_key FROM variant_content
 		WHERE cache_key > ? ORDER BY cache_key LIMIT ?
-	`, "variant keys", "variant key", string(after), c.reconciliationPageSize)
+	`, "variant keys", "variant key", string(after), reconciliationPageSize)
 }
 
 // queryStringColumn runs a single-column query with args and returns the
@@ -828,10 +823,6 @@ func (c *Cache) reconcileSourceRows(ctx context.Context) error {
 			return err
 		}
 
-		if c.reconciliationReadTestHook != nil {
-			c.reconciliationReadTestHook(len(hashes))
-		}
-
 		if len(hashes) == 0 {
 			return nil
 		}
@@ -860,7 +851,7 @@ func (c *Cache) reconcileSourceRows(ctx context.Context) error {
 }
 
 // sourceContentHashesAfter returns, in order, up to
-// c.reconciliationPageSize tracked source content hashes that sort after
+// reconciliationPageSize tracked source content hashes that sort after
 // the given one.
 func (c *Cache) sourceContentHashesAfter(
 	ctx context.Context, after ContentHash,
@@ -868,24 +859,20 @@ func (c *Cache) sourceContentHashesAfter(
 	return queryStringColumn[ContentHash](ctx, c.db, `
 		SELECT content_hash FROM source_content
 		WHERE content_hash > ? ORDER BY content_hash LIMIT ?
-	`, "source content hashes", "content hash", string(after),
+	`, "source content hashes", "content hash", string(after), reconciliationPageSize)
-		c.reconciliationPageSize)
 }
 
 // Each of these queries sums size_bytes over the next page of rows of
 // one content table, the rows that sort after a key, and returns the
-// page's last key, the sum and the number of rows in the page. Past the
+// page's last key with the sum. Past the last row the key is NULL.
-// last row the page has no rows and the key is NULL.
 const (
 	sourceSizePageQuery = `
-		SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0), COUNT(*)
+		SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0) FROM (
-		FROM (
 			SELECT content_hash, size_bytes FROM source_content
 			WHERE content_hash > ? ORDER BY content_hash LIMIT ?
 		)`
 	variantSizePageQuery = `
-		SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0), COUNT(*)
+		SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0) FROM (
-		FROM (
 			SELECT cache_key, size_bytes FROM variant_content
 			WHERE cache_key > ? ORDER BY cache_key LIMIT ?
 		)`
@@ -921,19 +908,13 @@ func (c *Cache) sumSizeBytesInPages(
 
 		var pageBytes int64
 
-		var pageRows int
+		err := c.db.QueryRowContext(ctx, pageQuery, after, reconciliationPageSize).
-
+			Scan(&lastKey, &pageBytes)
-		err := c.db.QueryRowContext(ctx, pageQuery, after, c.reconciliationPageSize).
-			Scan(&lastKey, &pageBytes, &pageRows)
 		if err != nil {
 			return 0, fmt.Errorf("failed to sum cache content sizes: %w", err)
 		}
 
-		if c.reconciliationReadTestHook != nil {
+		if !lastKey.Valid {
-			c.reconciliationReadTestHook(pageRows)
-		}
-
-		if pageRows == 0 {
 			return total, nil
 		}
 

internal/imgcache/service.go

@@ -100,13 +100,6 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
 		allowHTTP = cfg.FetcherConfig.AllowHTTP
 	}
 
-	// JPEG XL is to become the default output format, so pixad does not
-	// start without it.
-	err := imageprocessor.CheckJPEGXLSupport()
-	if err != nil {
-		return nil, err
-	}
-
 	maxResponseSize := fetcherCfg.MaxResponseSize
 	processor := imageprocessor.New(imageprocessor.Params{
 		MaxInputBytes:           maxResponseSize,

script/bootstrap

@@ -14,12 +14,11 @@
 #                            script/fmt-check: all the host needs, as
 #                            the checks compile pixa in Docker
 #   script/bootstrap --cgo   git, make, Go, and a C compiler and the
-#                            CGO image libraries (pkg-config, vips
+#                            CGO image libraries (pkg-config, vips,
-#                            with its JPEG XL support, libheif) for
+#                            libheif) for the govips bindings instead
-#                            the govips bindings instead of Node: to
+#                            of Node: to compile pixa, in the
-#                            compile pixa, in the Dockerfile's test
+#                            Dockerfile's test phase and build stage,
-#                            phase and build stage, which format
+#                            which format nothing
-#                            nothing
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -151,16 +150,6 @@ ensure_cgo_deps() {
     if ! pkg-config --exists vips; then
         pkg_install vips libvips-dev vips vips-dev
     fi
-    # libvips' JPEG XL loader and saver are in the nix and brew vips
-    # packages, and in apt's from Debian 12 and Ubuntu 24.04 on, but in
-    # the package vips-jxl on Alpine. detect_pkgmgr is called only where
-    # apk exists, as on apt it updates the package lists.
-    if ! missing apk; then
-        detect_pkgmgr
-    fi
-    if [ "$PKGMGR" = "apk" ] && ! apk info -e vips-jxl >/dev/null; then
-        apk add --no-cache vips-jxl
-    fi
     if ! pkg-config --exists libheif; then
         pkg_install libheif libheif-dev libheif libheif-dev
     fi