Sync cached files and refuse ones left empty or partial (closes #252)

A crash could leave a cached source or transformed image empty or cut
short, and pixa then served it, or fetched it again forever. Source and
transformed image files, and the .meta file beside each transformed
image, are now written under a temporary name, synced, renamed into
place and their directory synced; a failed directory sync removes the
renamed file. The reconciliation pass compares each cached file's size
with the size its row records and removes a file that is empty or of
another size, with its rows and a warning. Requests compare no sizes.

Not verified by a test: the syncs, and the removal after a failed sync.

Model: opus-5-5

internal/imgcache/eviction.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -560,10 +561,11 @@ func (c *Cache) runCleanExpired(ctx context.Context) {
 // (or whose accounting insert failed, e.g. StoreVariant's best-effort
 // insert under transient DB contention), drops accounting rows whose
 // files are missing, removes source blob files the database does not
-// know (and rows whose files are gone), sweeps stale temp files left
-// behind by crashed writes, and last checks the total cache usage
-// against the tables. It reads the tables a page at a time. Running it
-// periodically, not just once, bounds how long such drift can
+// know (and rows whose files are gone), removes source and variant files
+// that are empty or not the size their rows record, sweeps stale temp
+// files left behind by crashed writes, and last checks the total cache
+// usage against the tables. It reads the tables a page at a time.
+// Running it periodically, not just once, bounds how long such drift can
 // accumulate unaccounted for on a long-running process to one eviction
 // interval. Once ctx is cancelled, it stops at the next file or row and
 // returns ctx's error.
@@ -595,8 +597,9 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
 	return c.reconcileUsageTotal(ctx)
 }
 
-// reconcileVariantFiles walks the variant storage directory, adopting
-// files without accounting rows and sweeping stale temp files.
+// reconcileVariantFiles walks the variant storage directory, checking
+// each variant file against its accounting row (reconcileVariantFile)
+// and sweeping stale temp files.
 func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
 	return filepath.WalkDir(
 		c.variants.baseDir,
@@ -621,38 +624,55 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
 				return nil
 			}
 
-			return c.adoptVariantFile(ctx, path, entry, VariantKey(name))
+			return c.reconcileVariantFile(ctx, path, entry, VariantKey(name))
 		},
 	)
 }
 
-// adoptVariantFile inserts an accounting row for a variant file that
-// has none, using the file's size and modification time.
-func (c *Cache) adoptVariantFile(
+// reconcileVariantFile checks one variant file against its accounting
+// row. A file without a row is adopted into the accounting, unless it is
+// empty. A file that is empty, or not the size its row records, as a
+// crash or a restored snapshot can leave it, is removed with its row and
+// a warning naming it, so the variant is converted again.
+func (c *Cache) reconcileVariantFile(
 	ctx context.Context, path string, entry fs.DirEntry, cacheKey VariantKey,
 ) error {
-	var rowExists int
-
-	err := c.db.QueryRowContext(ctx,
-		`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, string(cacheKey),
-	).Scan(&rowExists)
-	if err != nil {
-		return fmt.Errorf("failed to check variant accounting row: %w", err)
-	}
-
-	if rowExists > 0 {
-		return nil
-	}
-
 	info, err := entry.Info()
 	if err != nil {
 		return fmt.Errorf("failed to stat variant file: %w", err)
 	}
 
+	var recordedSize int64
+
+	err = c.db.QueryRowContext(ctx,
+		`SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey),
+	).Scan(&recordedSize)
+
+	switch {
+	case errors.Is(err, sql.ErrNoRows) && info.Size() > 0:
+		return c.adoptVariantFile(ctx, path, info, cacheKey)
+	case err != nil && !errors.Is(err, sql.ErrNoRows):
+		return fmt.Errorf("failed to check variant accounting row: %w", err)
+	case info.Size() > 0 && info.Size() == recordedSize:
+		return nil
+	}
+
+	// The file is empty, or not the size its row records
+	c.log.Warn("removing damaged cached variant file", "path", path,
+		"file_bytes", info.Size(), "recorded_bytes", recordedSize)
+
+	return c.evictVariant(ctx, cacheKey)
+}
+
+// adoptVariantFile inserts an accounting row for a variant file that
+// has none, using the file's size and modification time.
+func (c *Cache) adoptVariantFile(
+	ctx context.Context, path string, info fs.FileInfo, cacheKey VariantKey,
+) error {
 	modTime := info.ModTime().UTC().Format(sqliteTimestampLayout)
 	contentType := c.variantContentTypeFromSidecar(path)
 
-	_, err = c.db.ExecContext(ctx, `
+	_, err := c.db.ExecContext(ctx, `
 		INSERT INTO variant_content
 			(cache_key, size_bytes, content_type, created_at, last_accessed_at)
 		VALUES (?, ?, ?, ?, ?)
@@ -771,10 +791,9 @@ func queryStringColumn[T ~string](
 	return values, nil
 }
 
-// reconcileSourceFiles walks the source content directory, removing
-// blob files the database does not track (they are unreachable: source
-// lookups always go through source_metadata) and sweeping stale temp
-// files.
+// reconcileSourceFiles walks the source content directory, checking
+// each blob file against its source_content row (reconcileSourceFile)
+// and sweeping stale temp files.
 func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
 	return filepath.WalkDir(
 		c.srcContent.baseDir,
@@ -795,31 +814,55 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
 				return nil
 			}
 
-			return c.removeUntrackedSourceFile(ctx, path, ContentHash(name))
+			return c.reconcileSourceFile(ctx, path, entry, ContentHash(name))
 		},
 	)
 }
 
+// reconcileSourceFile checks one source blob file against its
+// source_content row. A file without a row is removed: it is
+// unreachable, as source lookups always go through source_metadata. A
+// file that is empty, or not the size its row records, as a crash or a
+// restored snapshot can leave it, is removed with its rows and a warning
+// naming it, so the source is fetched again.
+func (c *Cache) reconcileSourceFile(
+	ctx context.Context, path string, entry fs.DirEntry, contentHash ContentHash,
+) error {
+	info, err := entry.Info()
+	if err != nil {
+		return fmt.Errorf("failed to stat source content file: %w", err)
+	}
+
+	var recordedSize int64
+
+	err = c.db.QueryRowContext(ctx,
+		`SELECT size_bytes FROM source_content WHERE content_hash = ?`,
+		string(contentHash),
+	).Scan(&recordedSize)
+
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return c.removeUntrackedSourceFile(ctx, path, contentHash)
+	case err != nil:
+		return fmt.Errorf("failed to check source content row: %w", err)
+	case info.Size() > 0 && info.Size() == recordedSize:
+		return nil
+	}
+
+	// The file is empty, or not the size its row records
+	c.log.Warn("removing damaged cached source file", "path", path,
+		"file_bytes", info.Size(), "recorded_bytes", recordedSize)
+
+	return c.evictSourceBlob(ctx, contentHash)
+}
+
 // removeUntrackedSourceFile deletes a source blob file that has no
 // source_content row. Any source_metadata rows referencing the hash
 // are removed first so no row ever points at a deleted file.
 func (c *Cache) removeUntrackedSourceFile(
 	ctx context.Context, path string, contentHash ContentHash,
 ) error {
-	var rowExists int
-
-	err := c.db.QueryRowContext(ctx,
-		`SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(contentHash),
-	).Scan(&rowExists)
-	if err != nil {
-		return fmt.Errorf("failed to check source content row: %w", err)
-	}
-
-	if rowExists > 0 {
-		return nil
-	}
-
-	_, err = c.db.ExecContext(ctx,
+	_, err := c.db.ExecContext(ctx,
 		`DELETE FROM source_metadata WHERE content_hash = ?`, string(contentHash))
 	if err != nil {
 		return fmt.Errorf("failed to delete metadata rows for untracked blob: %w", err)