Export cache, eviction, upstream and transform metrics (closes #85)

New pixa_ metrics, none with labels:
- source cache hits and misses, counted once per variant made or checked
- cache usage and limit gauges; usage is set each time eviction reads it
- eviction count and bytes, from the size_bytes the deleted row returns
- upstream fetch duration histogram and fetch error counter
- transform duration histogram and an in-progress gauge, both including
  the wait for a processing slot

Removing a damaged file during reconciliation is not counted as an
eviction. README lists every metric; the metrics config comment now says
the pixa_ metrics are measured whether or not /metrics is served.

Model: opus-5-5

README.md

@@ -1167,10 +1167,33 @@ Each item below is one setting, named by its config key:
     - the Go runtime and process metrics
     - `pixa_cache_hits_total`: image requests served from the cache
     - `pixa_cache_misses_total`: image requests not served from the cache
+    - `pixa_source_cache_hits_total`: variants made or checked whose source was
+      in the cache
+    - `pixa_source_cache_misses_total`: variants made or checked whose source
+      was not in the cache
+    - `pixa_cache_usage_bytes`: the bytes of sources and variants in the cache,
+      as eviction last read them; eviction runs after each store and every five
+      minutes
+    - `pixa_cache_max_bytes`: the cache size limit, `cache_max_bytes`
+    - `pixa_cache_evictions_total`: sources and variants removed by eviction
+    - `pixa_cache_evicted_bytes_total`: the bytes those evictions freed
     - `pixa_upstream_fetches_total`: fetches of a source image that read any
       bytes
     - `pixa_upstream_fetch_bytes_total`: the bytes those fetches read
+    - `pixa_upstream_fetch_duration_seconds`: a histogram of how long those
+      fetches took, up to the last byte of the response
+    - `pixa_upstream_fetch_errors_total`: fetches of a source image that failed,
+      before or while reading the response
     - `pixa_transforms_total`: images converted
+    - `pixa_transform_duration_seconds`: a histogram of how long those
+      conversions took, including the wait for a processing slot
+    - `pixa_transforms_in_progress`: images being converted or waiting for a
+      processing slot
+
+    pixa measures the `pixa_` metrics whether or not `/metrics` is served. They
+    have no labels, so their number never grows with the hosts and paths pixa
+    serves. Requests that share work (see Limits) count one source cache hit or
+    miss between them.
 
     A request's `handler` label is the pattern of the route it matches, such as
     `/v1/image/*`, never its path. Requests that match no route share the label

TODO.md

@@ -27,12 +27,5 @@ P2: security: per-origin rate limiting
 - P2: HTTP response handling
     - Last-Modified headers
 - P2: operational
-    - Prometheus metrics still missing:
-        - cache hits and misses, split into source images and variants
-        - cache size on disk and its limit
-        - evictions and the bytes they free
-        - upstream fetch duration and errors
-        - transcode duration
-        - images in processing
     - measure the 1k to 5k req/s target with `script/loadtest` on a machine not
       shared with other work

configs/config.example.yml

@@ -158,8 +158,9 @@ access_control_allow_origin: "*"
 # Sentry DSN for error reporting (default: empty, which turns it off)
 sentry_dsn: ""
 
-# Username and password for /metrics, set together (default: unset). Metrics
+# Username and password for /metrics, set together (default: unset). pixa
-# are measured and /metrics is served only when both are set.
+# serves /metrics, and measures HTTP requests, only when both are set. It
+# measures the pixa_ metrics either way.
 # metrics:
 #   username: "admin"
 #   password: "secret"

internal/imageproxy/imageproxy.go

@@ -37,11 +37,15 @@ type Proxy struct {
 	// same time share one fetch and one transcode.
 	variantsInProgress singleflight.Group
 
-	// The counts of upstream fetches and transforms, made and registered by
+	// The metrics of upstream fetches and transforms, made and registered by
-	// New. They are kept in memory, so adding to one waits for nothing.
+	// New. They are kept in memory, so updating one waits for nothing.
-	upstreamFetches    prometheus.Counter
+	upstreamFetches        prometheus.Counter
-	upstreamFetchBytes prometheus.Counter
+	upstreamFetchBytes     prometheus.Counter
-	transforms         prometheus.Counter
+	upstreamFetchDurations prometheus.Histogram
+	upstreamFetchErrors    prometheus.Counter
+	transforms             prometheus.Counter
+	transformDurations     prometheus.Histogram
+	transformsInProgress   prometheus.Gauge
 }
 
 // Config holds the proxy's configuration.
@@ -62,9 +66,9 @@ type Config struct {
 // errCacheRequired is returned when the proxy is built without a cache.
 var errCacheRequired = errors.New("cache is required")
 
-// New creates a new image proxy and registers its counters on
+// New creates a new image proxy and registers its metrics on
 // metricsRegistry. pixad passes Prometheus' default registry, the one
-// /metrics serves. It fails when metricsRegistry already holds counters of
+// /metrics serves. It fails when metricsRegistry already holds metrics of
 // the same names.
 func New(
 	cfg *Config, metricsRegistry prometheus.Registerer,
@@ -99,18 +103,39 @@ func New(
 			Name: "pixa_upstream_fetch_bytes_total",
 			Help: "Bytes read from upstream by fetches of source images.",
 		}),
+		upstreamFetchDurations: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name: "pixa_upstream_fetch_duration_seconds",
+			Help: "Duration of the fetches pixa_upstream_fetches_total counts, " +
+				"until the response's last byte.",
+		}),
+		upstreamFetchErrors: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "pixa_upstream_fetch_errors_total",
+			Help: "Fetches of a source image from upstream that failed, " +
+				"before or while reading the response.",
+		}),
 		transforms: prometheus.NewCounter(prometheus.CounterOpts{
 			Name: "pixa_transforms_total",
 			Help: "Images the image processor transformed.",
 		}),
+		transformDurations: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name: "pixa_transform_duration_seconds",
+			Help: "Duration of the transforms pixa_transforms_total counts, " +
+				"waiting for a processing slot included.",
+		}),
+		transformsInProgress: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "pixa_transforms_in_progress",
+			Help: "Images being transformed or waiting for a processing slot.",
+		}),
 	}
 
-	for _, counter := range []prometheus.Counter{
+	for _, metric := range []prometheus.Collector{
-		p.upstreamFetches, p.upstreamFetchBytes, p.transforms,
+		p.upstreamFetches, p.upstreamFetchBytes, p.upstreamFetchDurations,
+		p.upstreamFetchErrors, p.transforms, p.transformDurations,
+		p.transformsInProgress,
 	} {
-		err = metricsRegistry.Register(counter)
+		err = metricsRegistry.Register(metric)
 		if err != nil {
-			return nil, fmt.Errorf("failed to register the proxy's counters: %w", err)
+			return nil, fmt.Errorf("failed to register the proxy's metrics: %w", err)
 		}
 	}
 
@@ -341,7 +366,8 @@ func (p *Proxy) loadCachedSource(
 }
 
 // processFromSourceOrFetch makes the variant req asks for, from the cached
-// source when there is one and from a fetched source otherwise.
+// source when there is one and from a fetched source otherwise. It counts
+// whether the source was in the cache.
 //
 // A cached source last checked with upstream CacheTTL or more ago is checked
 // first: upstream is asked to send it only if it changed since. When it
@@ -362,9 +388,13 @@ func (p *Proxy) processFromSourceOrFetch(
 			p.log.Warn("failed to look up cached source, fetching", "error", err)
 		}
 
+		p.cache.IncrementSourceStats(false)
+
 		return p.fetchAndProcess(ctx, req, cacheKey)
 	}
 
+	p.cache.IncrementSourceStats(true)
+
 	if p.cache.Stale(cached.CheckedAt) {
 		fetched, err := p.fetchSource(ctx, req, cached.ETag, cached.LastModified)
 
@@ -581,36 +611,32 @@ type fetchedSource struct {
 // fetchSource fetches req's source from upstream and stores it in the cache.
 // A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source
 // only if it changed since; errSourceNotModified says it did not. It counts
-// the fetch with the bytes read from upstream, including when reading the
+// the fetch with the bytes read from upstream and its duration, including
-// response or a later step fails. It fetches nothing while the source URL is
+// when reading the response or a later step fails, and counts a fetch that
-// in the negative cache. The caller closes the returned response.
+// fails or whose response cannot be read as an upstream fetch error. It
+// fetches nothing while the source URL is in the negative cache. The caller
+// closes the returned response.
 func (p *Proxy) fetchSource(
 	ctx context.Context,
 	req *imagerequest.Request,
 	ifNoneMatch, ifModifiedSince string,
 ) (*fetchedSource, error) {
-	// Check negative cache - skip fetching for recently-failed URLs
+	err := p.refuseNegativeCached(ctx, req)
-	negHit, err := p.cache.Failure(ctx, req.Source)
 	if err != nil {
-		p.log.Warn("negative cache check failed", "error", err)
+		return nil, err
-	}
-
-	if negHit {
-		p.log.Debug("negative cache hit",
-			"host", req.Source.Host,
-			"path", req.Source.Path,
-		)
-
-		return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
 	}
 
 	// Fetch from upstream
 	p.log.Debug("fetching from upstream", "host", req.Source.Host,
 		"path", req.Source.Path, "query", req.Source.Query)
 
+	fetchStart := time.Now()
+
 	fetchResult, err := p.fetcher.Fetch(ctx, req.Source,
 		ifNoneMatch, ifModifiedSince)
 	if err != nil {
+		p.upstreamFetchErrors.Inc()
+
 		// Store negative cache for certain errors
 		if isNegativeCacheable(err) {
 			p.storeNegative(ctx, req)
@@ -634,9 +660,11 @@ func (p *Proxy) fetchSource(
 	fetchBytes := int64(len(sourceData))
 
 	// Counted also when the request context has ended meanwhile
-	p.incrementUpstreamFetch(fetchBytes)
+	p.incrementUpstreamFetch(fetchBytes, time.Since(fetchStart))
 
 	if err != nil {
+		p.upstreamFetchErrors.Inc()
+
 		_ = fetchResult.Content.Close()
 
 		return nil, fmt.Errorf("failed to read upstream response: %w", err)
@@ -677,6 +705,29 @@ func (p *Proxy) fetchSource(
 	}, nil
 }
 
+// refuseNegativeCached returns an error wrapping ErrNegativeCached while
+// req's source URL is in the negative cache, and nil otherwise. A failure to
+// read the negative cache is only logged, and returns nil.
+func (p *Proxy) refuseNegativeCached(
+	ctx context.Context, req *imagerequest.Request,
+) error {
+	negHit, err := p.cache.Failure(ctx, req.Source)
+	if err != nil {
+		p.log.Warn("negative cache check failed", "error", err)
+	}
+
+	if !negHit {
+		return nil
+	}
+
+	p.log.Debug("negative cache hit",
+		"host", req.Source.Host,
+		"path", req.Source.Path,
+	)
+
+	return fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
+}
+
 // fetchedSourceInfo returns what the cache keeps about a source upstream
 // just sent: fetched and checked now, with the ETag and Last-Modified
 // headers of fetchResult.
@@ -722,15 +773,16 @@ func (p *Proxy) logUpstreamFetch(
 	)
 }
 
-// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes.
+// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes
-// A fetch that read no bytes is not counted.
+// and took duration. A fetch that read no bytes is not counted.
-func (p *Proxy) incrementUpstreamFetch(fetchBytes int64) {
+func (p *Proxy) incrementUpstreamFetch(fetchBytes int64, duration time.Duration) {
 	if fetchBytes <= 0 {
 		return
 	}
 
 	p.upstreamFetches.Inc()
 	p.upstreamFetchBytes.Add(float64(fetchBytes))
+	p.upstreamFetchDurations.Observe(duration.Seconds())
 }
 
 // storeNegative stores a negative cache entry for req's source, whose fetch
@@ -772,8 +824,7 @@ func (p *Proxy) processAndStore(
 
 	// ctx has the deadline of the request doing the processing, which can
 	// end while a request waiting for its image has time left
-	processResult, err := p.processor.Process(ctx, source, sourceFormat,
+	processResult, err := p.transform(ctx, source, sourceFormat, &req.Transform)
-		&req.Transform)
 	if errors.Is(err, context.DeadlineExceeded) {
 		return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
 	}
@@ -786,6 +837,7 @@ func (p *Proxy) processAndStore(
 
 	// Counted also when the request context has ended meanwhile
 	p.transforms.Inc()
+	p.transformDurations.Observe(processDuration.Seconds())
 
 	// Read processed content
 	processedData, err := io.ReadAll(processResult.Content)
@@ -838,6 +890,21 @@ func (p *Proxy) processAndStore(
 	}, nil
 }
 
+// transform has the image processor transform the image read from source
+// as requested says, and counts it in pixa_transforms_in_progress until the
+// processor returns or panics.
+func (p *Proxy) transform(
+	ctx context.Context,
+	source io.Reader,
+	sourceFormat imagerequest.Format,
+	requested *imagerequest.Transform,
+) (*imageprocessor.Result, error) {
+	p.transformsInProgress.Inc()
+	defer p.transformsInProgress.Dec()
+
+	return p.processor.Process(ctx, source, sourceFormat, requested)
+}
+
 // isNegativeCacheable returns true if the error should be cached.
 func isNegativeCacheable(err error) bool {
 	return errors.Is(err, httpfetcher.ErrUpstreamError)

internal/imageproxy/metrics_internal_test.go

@@ -0,0 +1,112 @@
+package imageproxy
+
+import (
+	"testing"
+
+	"sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// metricValues holds the metrics of a proxy and its cache that
+// TestProxy_Get_CountsSourceCacheFetchesAndTransforms checks. A histogram
+// is given by its count of observations.
+type metricValues struct {
+	sourceCacheHits   uint64
+	sourceCacheMisses uint64
+	fetchDurations    uint64
+	fetchErrors       uint64
+	transformDuration uint64
+	inProgress        float64
+}
+
+// readMetricValues reads the metrics of svc and its cache that
+// metricValues holds from their metrics registry.
+func readMetricValues(t *testing.T, svc *testProxy) metricValues {
+	t.Helper()
+
+	families, err := svc.registry.Gather()
+	if err != nil {
+		t.Fatalf("failed to gather the metrics: %v", err)
+	}
+
+	var values metricValues
+
+	for _, family := range families {
+		metric := family.GetMetric()[0]
+
+		switch family.GetName() {
+		case "pixa_source_cache_hits_total":
+			values.sourceCacheHits = uint64(metric.GetCounter().GetValue())
+		case "pixa_source_cache_misses_total":
+			values.sourceCacheMisses = uint64(metric.GetCounter().GetValue())
+		case "pixa_upstream_fetch_duration_seconds":
+			values.fetchDurations = metric.GetHistogram().GetSampleCount()
+		case "pixa_upstream_fetch_errors_total":
+			values.fetchErrors = uint64(metric.GetCounter().GetValue())
+		case "pixa_transform_duration_seconds":
+			values.transformDuration = metric.GetHistogram().GetSampleCount()
+		case "pixa_transforms_in_progress":
+			values.inProgress = metric.GetGauge().GetValue()
+		}
+	}
+
+	return values
+}
+
+// TestProxy_Get_CountsSourceCacheFetchesAndTransforms walks Get through a
+// miss that fetches the source, a hit, a miss that uses the cached source
+// and a miss whose source is not found upstream, checking the source cache,
+// upstream fetch and transform metrics after each.
+func TestProxy_Get_CountsSourceCacheFetchesAndTransforms(t *testing.T) {
+	t.Parallel()
+
+	svc, fixtures := setupTestProxy(t)
+
+	// want is source cache hits, source cache misses, fetch durations
+	// observed, fetch errors, transform durations observed, transforms in
+	// progress.
+	steps := []struct {
+		name    string
+		path    string
+		size    int
+		wantErr bool
+		want    metricValues
+	}{
+		{"miss that fetches the source", testPathPhoto, 50, false,
+			metricValues{0, 1, 1, 0, 1, 0}},
+		{"hit", testPathPhoto, 50, false,
+			metricValues{0, 1, 1, 0, 1, 0}},
+		{"miss that uses the cached source", testPathPhoto, 25, false,
+			metricValues{1, 1, 1, 0, 2, 0}},
+		{"miss whose source is not found", "/images/absent.jpg", 50, true,
+			metricValues{1, 2, 1, 1, 2, 0}},
+	}
+
+	for _, step := range steps {
+		resp, err := svc.Get(t.Context(), &imagerequest.Request{
+			Source: imagerequest.Source{
+				Host: fixtures.GoodHost,
+				Path: step.path,
+			},
+			Transform: imagerequest.Transform{
+				Size:    imagerequest.Size{Width: step.size, Height: step.size},
+				Format:  imagerequest.FormatJPEG,
+				Quality: 85,
+				FitMode: imagerequest.FitCover,
+			},
+		})
+		if (err != nil) != step.wantErr {
+			t.Fatalf("%s: Get() error = %v, want error %t", step.name, err, step.wantErr)
+		}
+
+		if err == nil {
+			_ = resp.Content.Close()
+		}
+
+		got := readMetricValues(t, svc)
+		t.Logf("after the %s: metrics = %+v", step.name, got)
+
+		if got != step.want {
+			t.Fatalf("after the %s: metrics = %+v, want %+v", step.name, got, step.want)
+		}
+	}
+}

internal/imgcache/cache.go

@@ -111,10 +111,16 @@ type Cache struct {
 	pendingAccessTimesDone    chan struct{}
 	pendingAccessTimesCancel  context.CancelFunc
 
-	// The counts of cache hits and misses, made and registered by
+	// The cache's metrics, made and registered by registerMetrics (see
-	// registerCounters (see counters.go)
+	// metrics.go)
-	cacheHits   prometheus.Counter
+	cacheHits         prometheus.Counter
-	cacheMisses prometheus.Counter
+	cacheMisses       prometheus.Counter
+	sourceCacheHits   prometheus.Counter
+	sourceCacheMisses prometheus.Counter
+	usageBytes        prometheus.Gauge
+	maxBytes          prometheus.Gauge
+	evictions         prometheus.Counter
+	evictedBytes      prometheus.Counter
 
 	// reconciliationPageSize is the most rows one read of a content table
 	// returns in the reconciliation pass. NewCache sets it to
@@ -128,7 +134,7 @@ type Cache struct {
 	reconciliationReadTestHook func(rows int)
 }
 
-// NewCache creates a new cache instance and registers its counters on
+// NewCache creates a new cache instance and registers its metrics on
 // metricsRegistry. pixad passes Prometheus' default registry, the one
 // /metrics serves. It makes no database call.
 func NewCache(
@@ -157,7 +163,7 @@ func NewCache(
 		reconciliationPageSize: defaultReconciliationPageSize,
 	}
 
-	err := c.registerCounters(metricsRegistry)
+	err := c.registerMetrics(metricsRegistry)
 	if err != nil {
 		return nil, err
 	}
@@ -656,17 +662,6 @@ func (c *Cache) CleanExpired(ctx context.Context) error {
 	}
 }
 
-// IncrementStats counts a cache hit or miss. It only adds to the cache's
-// Prometheus counters in memory (see counters.go), so it never waits for the
-// database or fails.
-func (c *Cache) IncrementStats(hit bool) {
-	if hit {
-		c.cacheHits.Inc()
-	} else {
-		c.cacheMisses.Inc()
-	}
-}
-
 // touchVariant hands the access time of a variant served from the cache to
 // the goroutine StartPendingAccessTimeWrites starts, which writes it in its
 // next batch. It never waits: while maxPendingAccessTimes access times of

internal/imgcache/counters.go

@@ -1,31 +0,0 @@
-package imgcache
-
-import (
-	"fmt"
-
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-// registerCounters makes the cache's counters and registers them on
-// metricsRegistry. The counters are kept in memory, so adding to one waits
-// for nothing and makes no database call. It fails when metricsRegistry
-// already holds counters of the same names.
-func (c *Cache) registerCounters(metricsRegistry prometheus.Registerer) error {
-	c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "pixa_cache_hits_total",
-		Help: "Image requests served from the cache.",
-	})
-	c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "pixa_cache_misses_total",
-		Help: "Image requests not served from the cache.",
-	})
-
-	for _, counter := range []prometheus.Counter{c.cacheHits, c.cacheMisses} {
-		err := metricsRegistry.Register(counter)
-		if err != nil {
-			return fmt.Errorf("failed to register the cache's counters: %w", err)
-		}
-	}
-
-	return nil
-}

internal/imgcache/eviction.go

@@ -92,8 +92,9 @@ type evictionCandidate struct {
 }
 
 // EvictToLimit evicts least-recently-used cache entries until total
-// tracked usage is at or below the configured MaxBytes limit. It is a
+// tracked usage is at or below the configured MaxBytes limit. It sets the
-// no-op when the cache is disabled.
+// pixa_cache_usage_bytes gauge to each usage it reads. It is a no-op when
+// the cache is disabled.
 func (c *Cache) EvictToLimit(ctx context.Context) error {
 	if c.config.MaxBytes == 0 {
 		return nil
@@ -105,6 +106,8 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
 			return err
 		}
 
+		c.usageBytes.Set(float64(usage))
+
 		if usage <= c.config.MaxBytes {
 			return nil
 		}
@@ -345,37 +348,95 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro
 	return candidates, nil
 }
 
-// evictVariant removes one variant: accounting row first, then the
+// evictVariant removes one variant (see removeVariant) and counts the
-// file, so the database never references a deleted file.
+// eviction with the size its row recorded.
 func (c *Cache) evictVariant(
 	ctx context.Context, cacheKey imagerequest.VariantKey,
 ) error {
-	_, err := c.db.ExecContext(ctx,
+	sizeBytes, err := c.removeVariant(ctx, cacheKey)
-		`DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey))
-	if err != nil {
-		return fmt.Errorf("failed to delete variant accounting row: %w", err)
-	}
-
-	err = c.variants.Delete(string(cacheKey))
 	if err != nil {
 		return err
 	}
 
+	c.countEviction(sizeBytes)
+
 	return nil
 }
 
-// evictSourceBlob removes one source: its row first, then its file, as
+// removeVariant removes one variant: accounting row first, then the
-// evictVariant does for a variant. It holds no lock: a store of the same
+// file, so the database never references a deleted file. It returns the
-// source in between can leave the source's row with no file, which
+// size the row recorded, or 0 when the variant had no row.
-// Source treats as a miss.
+func (c *Cache) removeVariant(
-func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
+	ctx context.Context, cacheKey imagerequest.VariantKey,
-	_, err := c.db.ExecContext(ctx,
+) (int64, error) {
-		`DELETE FROM source_content WHERE source_key = ?`, string(key))
+	sizeBytes, err := c.deleteRowReturningSize(ctx,
+		`DELETE FROM variant_content WHERE cache_key = ? RETURNING size_bytes`,
+		string(cacheKey))
 	if err != nil {
-		return fmt.Errorf("failed to delete source content row: %w", err)
+		return 0, fmt.Errorf("failed to delete variant accounting row: %w", err)
 	}
 
-	return c.srcContent.Delete(string(key))
+	err = c.variants.Delete(string(cacheKey))
+	if err != nil {
+		return 0, err
+	}
+
+	return sizeBytes, nil
+}
+
+// evictSourceBlob removes one source (see removeSourceBlob) and counts the
+// eviction with the size its row recorded.
+func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
+	sizeBytes, err := c.removeSourceBlob(ctx, key)
+	if err != nil {
+		return err
+	}
+
+	c.countEviction(sizeBytes)
+
+	return nil
+}
+
+// removeSourceBlob removes one source: its row first, then its file, as
+// removeVariant does for a variant, and returns the size the row recorded,
+// or 0 when the source had no row. It holds no lock: a store of the same
+// source in between can leave the source's row with no file, which Source
+// treats as a miss.
+func (c *Cache) removeSourceBlob(
+	ctx context.Context, key imagerequest.SourceKey,
+) (int64, error) {
+	sizeBytes, err := c.deleteRowReturningSize(ctx,
+		`DELETE FROM source_content WHERE source_key = ? RETURNING size_bytes`,
+		string(key))
+	if err != nil {
+		return 0, fmt.Errorf("failed to delete source content row: %w", err)
+	}
+
+	err = c.srcContent.Delete(string(key))
+	if err != nil {
+		return 0, err
+	}
+
+	return sizeBytes, nil
+}
+
+// deleteRowReturningSize runs deleteQuery, which deletes the row of key and
+// returns its size_bytes, and returns that size, or 0 when there was no row.
+func (c *Cache) deleteRowReturningSize(
+	ctx context.Context, deleteQuery string, key string,
+) (int64, error) {
+	var sizeBytes int64
+
+	err := c.db.QueryRowContext(ctx, deleteQuery, key).Scan(&sizeBytes)
+	if errors.Is(err, sql.ErrNoRows) {
+		return 0, nil
+	}
+
+	if err != nil {
+		return 0, err
+	}
+
+	return sizeBytes, nil
 }
 
 // notifyWritePressure wakes the background evictor after a store, so
@@ -673,7 +734,9 @@ func (c *Cache) reconcileVariantFile(
 	c.log.Warn("removing damaged cached variant file", "path", path,
 		"file_bytes", info.Size(), "recorded_bytes", recordedSize)
 
-	return c.evictVariant(ctx, cacheKey)
+	_, err = c.removeVariant(ctx, cacheKey)
+
+	return err
 }
 
 // adoptVariantFile inserts an accounting row for a variant file that
@@ -835,7 +898,9 @@ func (c *Cache) reconcileSourceFile(
 	c.log.Warn("removing damaged cached source file", "path", path,
 		"file_bytes", info.Size(), "recorded_bytes", recordedSize)
 
-	return c.evictSourceBlob(ctx, key)
+	_, err = c.removeSourceBlob(ctx, key)
+
+	return err
 }
 
 // removeUntrackedSourceFile deletes a source blob file that has no

internal/imgcache/metrics.go

@@ -0,0 +1,88 @@
+package imgcache
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// registerMetrics makes the cache's metrics and registers them on
+// metricsRegistry. The metrics are kept in memory, so updating one waits
+// for nothing and makes no database call. It fails when metricsRegistry
+// already holds metrics of the same names.
+func (c *Cache) registerMetrics(metricsRegistry prometheus.Registerer) error {
+	c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "pixa_cache_hits_total",
+		Help: "Image requests served from the cache.",
+	})
+	c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "pixa_cache_misses_total",
+		Help: "Image requests not served from the cache.",
+	})
+	c.sourceCacheHits = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "pixa_source_cache_hits_total",
+		Help: "Variants made or checked whose source was in the cache.",
+	})
+	c.sourceCacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "pixa_source_cache_misses_total",
+		Help: "Variants made or checked whose source was not in the cache.",
+	})
+	c.usageBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "pixa_cache_usage_bytes",
+		Help: "Bytes of sources and variants in the cache, as eviction last read them.",
+	})
+	c.maxBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "pixa_cache_max_bytes",
+		Help: "The cache size limit in bytes, cache_max_bytes.",
+	})
+	c.evictions = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "pixa_cache_evictions_total",
+		Help: "Sources and variants removed by eviction.",
+	})
+	c.evictedBytes = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "pixa_cache_evicted_bytes_total",
+		Help: "Bytes of the sources and variants removed by eviction.",
+	})
+
+	c.maxBytes.Set(float64(c.config.MaxBytes))
+
+	for _, metric := range []prometheus.Collector{
+		c.cacheHits, c.cacheMisses, c.sourceCacheHits, c.sourceCacheMisses,
+		c.usageBytes, c.maxBytes, c.evictions, c.evictedBytes,
+	} {
+		err := metricsRegistry.Register(metric)
+		if err != nil {
+			return fmt.Errorf("failed to register the cache's metrics: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// IncrementStats counts a cache hit or miss of an image request. It only
+// adds to the cache's counters in memory, so it never waits for the
+// database or fails.
+func (c *Cache) IncrementStats(hit bool) {
+	if hit {
+		c.cacheHits.Inc()
+	} else {
+		c.cacheMisses.Inc()
+	}
+}
+
+// IncrementSourceStats counts whether a variant being made or checked found
+// its source in the cache. Like IncrementStats, it never waits or fails.
+func (c *Cache) IncrementSourceStats(hit bool) {
+	if hit {
+		c.sourceCacheHits.Inc()
+	} else {
+		c.sourceCacheMisses.Inc()
+	}
+}
+
+// countEviction counts one source or variant of sizeBytes bytes removed by
+// eviction.
+func (c *Cache) countEviction(sizeBytes int64) {
+	c.evictions.Inc()
+	c.evictedBytes.Add(float64(sizeBytes))
+}

internal/imgcache/metrics_internal_test.go

@@ -0,0 +1,137 @@
+package imgcache
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// gatheredValue returns the value of the metric name on registry: a
+// counter's or a gauge's value.
+func gatheredValue(t *testing.T, registry *prometheus.Registry, name string) float64 {
+	t.Helper()
+
+	families, err := registry.Gather()
+	if err != nil {
+		t.Fatalf("failed to gather the metrics: %v", err)
+	}
+
+	for _, family := range families {
+		if family.GetName() != name {
+			continue
+		}
+
+		metric := family.GetMetric()[0]
+		if metric.GetGauge() != nil {
+			return metric.GetGauge().GetValue()
+		}
+
+		return metric.GetCounter().GetValue()
+	}
+
+	t.Fatalf("the metric %s is not on the registry", name)
+
+	return 0
+}
+
+// TestEvictToLimit_CountsEvictionsAndUsage stores four variants of 1000
+// bytes in a cache limited to 3000 bytes. EvictToLimit must count the one
+// eviction and its bytes, and set the usage and limit gauges.
+func TestEvictToLimit_CountsEvictionsAndUsage(t *testing.T) {
+	t.Parallel()
+
+	const limit = 3000
+
+	registry := prometheus.NewRegistry()
+
+	cache, err := NewCache(evictionTestDB(t),
+		Config{StateDir: t.TempDir(), MaxBytes: limit}, registry)
+	if err != nil {
+		t.Fatalf("NewCache() error = %v", err)
+	}
+
+	keys := []imagerequest.VariantKey{
+		testVariantKeyOne, testVariantKeyTwo,
+		testVariantKeyThree, testVariantKeyFour,
+	}
+
+	for i, key := range keys {
+		storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{0x01}, 1000))
+		setVariantLastAccessed(t, cache, key,
+			time.Now().Add(-time.Duration(len(keys)-i)*time.Hour))
+	}
+
+	err = cache.EvictToLimit(t.Context())
+	if err != nil {
+		t.Fatalf("EvictToLimit() error = %v", err)
+	}
+
+	want := map[string]float64{
+		"pixa_cache_evictions_total":     1,
+		"pixa_cache_evicted_bytes_total": 1000,
+		"pixa_cache_usage_bytes":         3000,
+		"pixa_cache_max_bytes":           limit,
+	}
+
+	for name, value := range want {
+		got := gatheredValue(t, registry, name)
+		t.Logf("%s = %v", name, got)
+
+		if got != value {
+			t.Errorf("%s = %v, want %v", name, got, value)
+		}
+	}
+}
+
+// TestEvictNotCheckedRecently_CountsEvictions stores a source of 6 bytes and
+// a variant of 7 bytes last checked with upstream 31 days ago.
+// EvictNotCheckedRecently must count both evictions and their 13 bytes.
+func TestEvictNotCheckedRecently_CountsEvictions(t *testing.T) {
+	t.Parallel()
+
+	registry := prometheus.NewRegistry()
+
+	cache, err := NewCache(evictionTestDB(t),
+		Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry)
+	if err != nil {
+		t.Fatalf("NewCache() error = %v", err)
+	}
+
+	old := time.Now().Add(-31 * 24 * time.Hour)
+
+	_, err = cache.PutSource(t.Context(),
+		imagerequest.Source{Host: testHostCDN, Path: "/old.jpg"},
+		bytes.NewReader([]byte("source")),
+		SourceInfo{FetchedAt: old, CheckedAt: old})
+	if err != nil {
+		t.Fatalf("PutSource() error = %v", err)
+	}
+
+	err = cache.PutVariant(t.Context(), testVariantKeyOne,
+		SourceInfo{CheckedAt: old}, bytes.NewReader([]byte("variant")))
+	if err != nil {
+		t.Fatalf("PutVariant() error = %v", err)
+	}
+
+	err = cache.EvictNotCheckedRecently(t.Context())
+	if err != nil {
+		t.Fatalf("EvictNotCheckedRecently() error = %v", err)
+	}
+
+	want := map[string]float64{
+		"pixa_cache_evictions_total":     2,
+		"pixa_cache_evicted_bytes_total": 13,
+	}
+
+	for name, value := range want {
+		got := gatheredValue(t, registry, name)
+		t.Logf("%s = %v", name, got)
+
+		if got != value {
+			t.Errorf("%s = %v, want %v", name, got, value)
+		}
+	}
+}

internal/providers/providers.go

@@ -35,7 +35,7 @@ import (
 
 // Constructors returns the constructors of the parts this package builds,
 // for fx.Provide. The fx graph must also provide the prometheus.Registerer
-// the cache and the image service register their counters on.
+// the cache and the image service register their metrics on.
 func Constructors() []any {
 	return []any{
 		newKeys,