Export cache, eviction, upstream and transform metrics (closes #85)

New pixa_ metrics, none with labels:
- source cache hits and misses, counted once per variant made or checked;
  a source whose cached file cannot be used is a miss
- cache usage and limit gauges; usage is set each time eviction reads it
- eviction count and bytes, from the size_bytes the deleted row returns
- upstream fetch duration histogram and fetch error counter
- transform duration histogram and an in-progress gauge, both including
  the wait for a processing slot

Removing a damaged file during reconciliation is not counted as an
eviction. README lists every metric; the metrics config comment now says
the pixa_ metrics are measured whether or not /metrics is served.

Model: opus-5-5

internal/imgcache/eviction.go

@@ -92,8 +92,9 @@ type evictionCandidate struct {
 }
 
 // EvictToLimit evicts least-recently-used cache entries until total
-// tracked usage is at or below the configured MaxBytes limit. It is a
-// no-op when the cache is disabled.
+// tracked usage is at or below the configured MaxBytes limit. It sets the
+// pixa_cache_usage_bytes gauge to each usage it reads. It is a no-op when
+// the cache is disabled.
 func (c *Cache) EvictToLimit(ctx context.Context) error {
 	if c.config.MaxBytes == 0 {
 		return nil
@@ -105,6 +106,8 @@ func (c *Cache) EvictToLimit(ctx context.Context) error {
 			return err
 		}
 
+		c.usageBytes.Set(float64(usage))
+
 		if usage <= c.config.MaxBytes {
 			return nil
 		}
@@ -345,37 +348,95 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro
 	return candidates, nil
 }
 
-// evictVariant removes one variant: accounting row first, then the
-// file, so the database never references a deleted file.
+// evictVariant removes one variant (see removeVariant) and counts the
+// eviction with the size its row recorded.
 func (c *Cache) evictVariant(
 	ctx context.Context, cacheKey imagerequest.VariantKey,
 ) error {
-	_, err := c.db.ExecContext(ctx,
-		`DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey))
-	if err != nil {
-		return fmt.Errorf("failed to delete variant accounting row: %w", err)
-	}
-
-	err = c.variants.Delete(string(cacheKey))
+	sizeBytes, err := c.removeVariant(ctx, cacheKey)
 	if err != nil {
 		return err
 	}
 
+	c.countEviction(sizeBytes)
+
 	return nil
 }
 
-// evictSourceBlob removes one source: its row first, then its file, as
-// evictVariant does for a variant. It holds no lock: a store of the same
-// source in between can leave the source's row with no file, which
-// Source treats as a miss.
-func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
-	_, err := c.db.ExecContext(ctx,
-		`DELETE FROM source_content WHERE source_key = ?`, string(key))
+// removeVariant removes one variant: accounting row first, then the
+// file, so the database never references a deleted file. It returns the
+// size the row recorded, or 0 when the variant had no row.
+func (c *Cache) removeVariant(
+	ctx context.Context, cacheKey imagerequest.VariantKey,
+) (int64, error) {
+	sizeBytes, err := c.deleteRowReturningSize(ctx,
+		`DELETE FROM variant_content WHERE cache_key = ? RETURNING size_bytes`,
+		string(cacheKey))
 	if err != nil {
-		return fmt.Errorf("failed to delete source content row: %w", err)
+		return 0, fmt.Errorf("failed to delete variant accounting row: %w", err)
 	}
 
-	return c.srcContent.Delete(string(key))
+	err = c.variants.Delete(string(cacheKey))
+	if err != nil {
+		return 0, err
+	}
+
+	return sizeBytes, nil
+}
+
+// evictSourceBlob removes one source (see removeSourceBlob) and counts the
+// eviction with the size its row recorded.
+func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error {
+	sizeBytes, err := c.removeSourceBlob(ctx, key)
+	if err != nil {
+		return err
+	}
+
+	c.countEviction(sizeBytes)
+
+	return nil
+}
+
+// removeSourceBlob removes one source: its row first, then its file, as
+// removeVariant does for a variant, and returns the size the row recorded,
+// or 0 when the source had no row. It holds no lock: a store of the same
+// source in between can leave the source's row with no file, which Source
+// treats as a miss.
+func (c *Cache) removeSourceBlob(
+	ctx context.Context, key imagerequest.SourceKey,
+) (int64, error) {
+	sizeBytes, err := c.deleteRowReturningSize(ctx,
+		`DELETE FROM source_content WHERE source_key = ? RETURNING size_bytes`,
+		string(key))
+	if err != nil {
+		return 0, fmt.Errorf("failed to delete source content row: %w", err)
+	}
+
+	err = c.srcContent.Delete(string(key))
+	if err != nil {
+		return 0, err
+	}
+
+	return sizeBytes, nil
+}
+
+// deleteRowReturningSize runs deleteQuery, which deletes the row of key and
+// returns its size_bytes, and returns that size, or 0 when there was no row.
+func (c *Cache) deleteRowReturningSize(
+	ctx context.Context, deleteQuery string, key string,
+) (int64, error) {
+	var sizeBytes int64
+
+	err := c.db.QueryRowContext(ctx, deleteQuery, key).Scan(&sizeBytes)
+	if errors.Is(err, sql.ErrNoRows) {
+		return 0, nil
+	}
+
+	if err != nil {
+		return 0, err
+	}
+
+	return sizeBytes, nil
 }
 
 // notifyWritePressure wakes the background evictor after a store, so
@@ -673,7 +734,9 @@ func (c *Cache) reconcileVariantFile(
 	c.log.Warn("removing damaged cached variant file", "path", path,
 		"file_bytes", info.Size(), "recorded_bytes", recordedSize)
 
-	return c.evictVariant(ctx, cacheKey)
+	_, err = c.removeVariant(ctx, cacheKey)
+
+	return err
 }
 
 // adoptVariantFile inserts an accounting row for a variant file that
@@ -835,7 +898,9 @@ func (c *Cache) reconcileSourceFile(
 	c.log.Warn("removing damaged cached source file", "path", path,
 		"file_bytes", info.Size(), "recorded_bytes", recordedSize)
 
-	return c.evictSourceBlob(ctx, key)
+	_, err = c.removeSourceBlob(ctx, key)
+
+	return err
 }
 
 // removeUntrackedSourceFile deletes a source blob file that has no