Store a source the way a variant is stored, one row and one file per source URL (closes #308)

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5

internal/imgcache/cache_usage_internal_test.go

@@ -31,8 +31,8 @@ func TestEvictionReadsTheUsageTotal(t *testing.T) {
 	storeEvictionTestVariant(t, cache, testVariantKeyTwo,
 		bytes.Repeat([]byte{0x75}, 200))
 
-	// pixa never changes a source's size, but a statement that does must
-	// change the total too.
+	// Storing a source URL again changes its size, so the total must
+	// follow a statement that changes it.
 	_, err := cache.db.ExecContext(ctx,
 		`UPDATE source_content SET size_bytes = 900 WHERE source_key = ?`,
 		string(kept))

internal/imgcache/eviction_internal_test.go

@@ -948,14 +948,15 @@ func TestStopEvictionInterruptsPassInProgress(t *testing.T) {
 }
 
 // TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context
-// while the oldest of three sources is being evicted, and checks that
-// EvictToLimit then returns context.Canceled, with that source gone and
-// the other two kept, and logs no warning. The test database has one
-// connection. When the deletion of the oldest source's row commits, a
-// commit hook holds the deletion until the test waits for the
-// connection, so the test takes it as soon as the deletion is done. The
-// eviction of the next source then waits for the connection, and the
-// test cancels the context while it waits.
+// once the oldest of three sources is evicted, while the next one waits
+// for the database, and checks that EvictToLimit then returns
+// context.Canceled, with that source gone and the other two kept, and
+// logs no warning. The test database has one connection. When the
+// deletion of the oldest source's row commits, a commit hook holds the
+// deletion until the test waits for the connection, so the test takes it
+// as soon as the deletion is done. The eviction of the next source then
+// waits for the connection, and the test cancels the context while it
+// waits.
 func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
 	t.Parallel()
 
@@ -1006,24 +1007,26 @@ func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
 
 	go func() { evicted <- cache.EvictToLimit(ctx) }()
 
-	<-oldestRowDeleted
-
-	conn, err := cache.db.Conn(t.Context())
-	if err != nil {
-		t.Fatalf("failed to take the database connection: %v", err)
+	select {
+	case <-oldestRowDeleted:
+	case err := <-evicted:
+		t.Fatalf("EvictToLimit() returned %v without deleting a row", err)
+	case <-time.After(5 * time.Second):
+		t.Fatal("EvictToLimit never deleted a row")
 	}
 
-	defer func() { _ = conn.Close() }()
+	releaseDatabase := holdDatabase(t, cache)
+	defer releaseDatabase()
 
 	waitForConnectionWaits(t, cache.db, waitsBefore+2)
 	cancel()
 
-	err = <-evicted
+	err := <-evicted
 	t.Logf("EvictToLimit() error = %v", err)
 	t.Logf("log output: %s", logBuf.String())
 
 	// The checks below read the database, so the connection goes back.
-	_ = conn.Close()
+	releaseDatabase()
 
 	setCommitHook(t, cache, nil)
 

internal/imgcache/max_concurrent_processing_internal_test.go

@@ -103,7 +103,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) {
 	// before waiting, it would have read it by now.
 	time.Sleep(100 * time.Millisecond)
 
-	err = os.WriteFile(svc.cache.srcContent.hashToPath(sourceKey),
+	err = os.WriteFile(svc.cache.srcContent.keyToPath(sourceKey),
 		generateTestJPEG(t, 100, 50, color.RGBA{0, 0, 255, 255}), 0o600)
 	if err != nil {
 		t.Fatalf("failed to rewrite the cached source: %v", err)

internal/imgcache/partial_cache_files_internal_test.go

@@ -70,7 +70,7 @@ func cachedSourcePath(
 		t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err)
 	}
 
-	return svc.cache.srcContent.hashToPath(sourceKey)
+	return svc.cache.srcContent.keyToPath(sourceKey)
 }
 
 // reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
@@ -327,7 +327,7 @@ func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
 
 	content := []byte("the whole source image")
 	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
-	path := storage.hashToPath(key)
+	path := storage.keyToPath(key)
 
 	err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
 	if err != nil {

internal/imgcache/service_internal_test.go

@@ -619,7 +619,7 @@ func backdateSource(
 		dbTime, string(key))
 	setSourceLastAccessed(t, cache, key, when)
 
-	path := cache.srcContent.hashToPath(key)
+	path := cache.srcContent.keyToPath(key)
 
 	err := os.Chtimes(path, when, when)
 	if err != nil {

internal/imgcache/storage.go

@@ -49,7 +49,7 @@ func NewContentStorage(baseDir string) (*ContentStorage, error) {
 func (s *ContentStorage) Store(
 	key imagerequest.SourceKey, data []byte,
 ) (int64, error) {
-	path := s.hashToPath(key)
+	path := s.keyToPath(key)
 
 	err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
 	if err != nil {
@@ -66,7 +66,7 @@ func (s *ContentStorage) Store(
 
 // Load returns a reader for the content with the given key.
 func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) {
-	path := s.hashToPath(key)
+	path := s.keyToPath(key)
 
 	f, err := os.Open(path) //nolint:gosec // path derived from source key
 	if err != nil {
@@ -84,7 +84,7 @@ func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error)
 func (s *ContentStorage) LoadWithSize(
 	key imagerequest.SourceKey,
 ) (io.ReadCloser, int64, error) {
-	path := s.hashToPath(key)
+	path := s.keyToPath(key)
 
 	f, err := os.Open(path) //nolint:gosec // path derived from source key
 	if err != nil {
@@ -107,7 +107,7 @@ func (s *ContentStorage) LoadWithSize(
 
 // Delete removes content with the given key.
 func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
-	path := s.hashToPath(key)
+	path := s.keyToPath(key)
 
 	err := os.Remove(path)
 	if err != nil && !os.IsNotExist(err) {
@@ -119,7 +119,7 @@ func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
 
 // Exists checks if content with the given key exists.
 func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool {
-	path := s.hashToPath(key)
+	path := s.keyToPath(key)
 	_, err := os.Stat(path)
 
 	return err == nil
@@ -199,14 +199,14 @@ func syncDirectory(dir string) error {
 	return nil
 }
 
-// hashToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
-func (s *ContentStorage) hashToPath(key imagerequest.SourceKey) string {
-	h := string(key)
-	if len(h) < MinHashLength {
-		return filepath.Clean(filepath.Join(s.baseDir, h))
+// keyToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
+func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string {
+	k := string(key)
+	if len(k) < MinHashLength {
+		return filepath.Clean(filepath.Join(s.baseDir, k))
 	}
 
-	return filepath.Clean(filepath.Join(s.baseDir, h[0:2], h[2:4], h))
+	return filepath.Clean(filepath.Join(s.baseDir, k[0:2], k[2:4], k))
 }
 
 // VariantStorage handles key-based file storage for processed image variants.

internal/imgcache/storage_internal_test.go

@@ -3,14 +3,66 @@ package imgcache
 import (
 	"bytes"
 	"errors"
+	"io"
 	"log/slog"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"sneak.berlin/go/pixa/internal/imagerequest"
 )
 
+func TestContentStorage_StoreAndLoad(t *testing.T) {
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+
+	storage, err := NewContentStorage(tmpDir)
+	if err != nil {
+		t.Fatalf("NewContentStorage() error = %v", err)
+	}
+
+	content := []byte("hello world")
+	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
+
+	size, err := storage.Store(key, content)
+	if err != nil {
+		t.Fatalf("Store() error = %v", err)
+	}
+
+	if size != int64(len(content)) {
+		t.Errorf("Store() size = %d, want %d", size, len(content))
+	}
+
+	// Verify file exists at expected path
+	keyStr := string(key)
+
+	expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr)
+
+	_, err = os.Stat(expectedPath)
+	if err != nil {
+		t.Errorf("File not at expected path %s: %v", expectedPath, err)
+	}
+
+	// Load and verify content
+	r, err := storage.Load(key)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	defer func() { _ = r.Close() }()
+
+	loaded, err := io.ReadAll(r)
+	if err != nil {
+		t.Fatalf("ReadAll() error = %v", err)
+	}
+
+	if !bytes.Equal(loaded, content) {
+		t.Errorf("Load() content = %q, want %q", loaded, content)
+	}
+}
+
 func TestContentStorage_LoadNotFound(t *testing.T) {
 	t.Parallel()
 
@@ -27,6 +79,38 @@ func TestContentStorage_LoadNotFound(t *testing.T) {
 	}
 }
 
+func TestContentStorage_Delete(t *testing.T) {
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+
+	storage, err := NewContentStorage(tmpDir)
+	if err != nil {
+		t.Fatalf("NewContentStorage() error = %v", err)
+	}
+
+	content := []byte("to be deleted")
+	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
+
+	_, err = storage.Store(key, content)
+	if err != nil {
+		t.Fatalf("Store() error = %v", err)
+	}
+
+	if !storage.Exists(key) {
+		t.Error("Exists() = false, want true")
+	}
+
+	err = storage.Delete(key)
+	if err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+
+	if storage.Exists(key) {
+		t.Error("Exists() = true after delete, want false")
+	}
+}
+
 func TestContentStorage_DeleteNonexistent(t *testing.T) {
 	t.Parallel()
 
@@ -44,6 +128,35 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) {
 	}
 }
 
+func TestContentStorage_HashToPath(t *testing.T) {
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+
+	storage, err := NewContentStorage(tmpDir)
+	if err != nil {
+		t.Fatalf("NewContentStorage() error = %v", err)
+	}
+
+	// Test by storing and verifying the resulting path structure
+	content := []byte("test content for path verification")
+	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
+
+	_, err = storage.Store(key, content)
+	if err != nil {
+		t.Fatalf("Store() error = %v", err)
+	}
+
+	keyStr := string(key)
+
+	expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr)
+
+	_, err = os.Stat(expectedPath)
+	if err != nil {
+		t.Errorf("File not at expected path %s: %v", expectedPath, err)
+	}
+}
+
 // TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
 // that fails is logged at warn and the store still succeeds.
 func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {