Sync cached files and refuse ones left empty or partial (closes #252)

Source and variant files are synced to disk before the rename into
place, and their directory after. Lookup and LookupSource compare the
file's size with the size its row records: a file that is empty or of
another size is removed with its rows and a warning naming it, and the
request converts the image or fetches the source again. A variant whose
row cannot be read is judged by its file alone. Storing a source over a
file of another size replaces it. A source that fails to decode is not
removed for that alone.

#252

Model: opus-5-5

internal/imgcache/storage.go

@@ -79,8 +79,8 @@ func (s *ContentStorage) Store(r io.Reader) (ContentHash, int64, error) {
 // from hash, without recomputing it. Callers that already know the
 // hash before writing (e.g. because they must hold a hash-keyed lock
 // across the whole store operation) use this instead of Store. Like
-// Store, it is idempotent: content already on disk at that path is
-// left untouched.
+// Store, it is idempotent: a file of the content's size already at that
+// path is left untouched, and a file of another size is replaced.
 func (s *ContentStorage) StoreHashed(
 	hash ContentHash, data []byte,
 ) (int64, error) {
@@ -151,16 +151,31 @@ func (s *ContentStorage) Exists(hash ContentHash) bool {
 	return err == nil
 }
 
-// writeIfAbsent writes data to the path derived from hash, unless
-// content already exists there, via a temp-file-plus-rename so
-// concurrent readers never observe a partial file.
+// Size returns the size in bytes of the content with the given hash.
+func (s *ContentStorage) Size(hash ContentHash) (int64, error) {
+	info, err := os.Stat(s.hashToPath(hash))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, ErrNotFound
+		}
+
+		return 0, fmt.Errorf("failed to stat content: %w", err)
+	}
+
+	return info.Size(), nil
+}
+
+// writeIfAbsent writes data to the path derived from hash, unless a
+// file of the data's size is already there. It writes a temp file,
+// syncs it to disk and renames it into place, so neither a concurrent
+// reader nor a crash finds a partial file under that path.
 func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error {
 	// Build path: <basedir>/<ab>/<cd>/<hash>
 	path := s.hashToPath(hash)
 
-	// Check if already exists
-	_, statErr := os.Stat(path)
-	if statErr == nil {
+	// A file of another size there is damaged, and is replaced
+	info, statErr := os.Stat(path)
+	if statErr == nil && info.Size() == int64(len(data)) {
 		return nil
 	}
 
@@ -194,6 +209,14 @@ func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error {
 		return fmt.Errorf("failed to write content: %w", err)
 	}
 
+	err = tmpFile.Sync()
+	if err != nil {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpPath)
+
+		return fmt.Errorf("failed to sync temp file: %w", err)
+	}
+
 	err = tmpFile.Close()
 	if err != nil {
 		_ = os.Remove(tmpPath)
@@ -209,6 +232,24 @@ func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error {
 		return fmt.Errorf("failed to rename temp file: %w", err)
 	}
 
+	return syncDirectory(dir)
+}
+
+// syncDirectory syncs the directory dir to disk, so that a file just
+// renamed into it is still there after a crash.
+func syncDirectory(dir string) error {
+	d, err := os.Open(filepath.Clean(dir))
+	if err != nil {
+		return fmt.Errorf("failed to open directory: %w", err)
+	}
+
+	err = d.Sync()
+	_ = d.Close()
+
+	if err != nil {
+		return fmt.Errorf("failed to sync directory: %w", err)
+	}
+
 	return nil
 }
 
@@ -437,7 +478,9 @@ func (s *VariantStorage) Store(
 		return 0, fmt.Errorf("failed to create directory: %w", err)
 	}
 
-	// Write content to temp file first, then rename for atomicity
+	// Write content to temp file first, sync it to disk, then rename for
+	// atomicity, so neither a concurrent reader nor a crash finds a
+	// partial file under the key's path
 	tmpFile, err := os.CreateTemp(dir, ".tmp-*")
 	if err != nil {
 		return 0, fmt.Errorf("failed to create temp file: %w", err)
@@ -453,6 +496,14 @@ func (s *VariantStorage) Store(
 		return 0, fmt.Errorf("failed to write content: %w", err)
 	}
 
+	err = tmpFile.Sync()
+	if err != nil {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpPath)
+
+		return 0, fmt.Errorf("failed to sync temp file: %w", err)
+	}
+
 	err = tmpFile.Close()
 	if err != nil {
 		_ = os.Remove(tmpPath)
@@ -468,6 +519,11 @@ func (s *VariantStorage) Store(
 		return 0, fmt.Errorf("failed to rename temp file: %w", err)
 	}
 
+	err = syncDirectory(dir)
+	if err != nil {
+		return 0, err
+	}
+
 	// Write metadata
 	meta := VariantMeta{
 		ContentType: contentType,
@@ -564,6 +620,20 @@ func (s *VariantStorage) Exists(key VariantKey) bool {
 	return err == nil
 }
 
+// Size returns the size in bytes of the content at the given key.
+func (s *VariantStorage) Size(key VariantKey) (int64, error) {
+	info, err := os.Stat(s.keyToPath(key))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, ErrNotFound
+		}
+
+		return 0, fmt.Errorf("failed to stat content: %w", err)
+	}
+
+	return info.Size(), nil
+}
+
 // Delete removes the content at the given key together with its .meta
 // sidecar file. A missing file is not an error.
 func (s *VariantStorage) Delete(key VariantKey) error {