Store a source the way a variant is stored, one row and one file per source URL (closes #308)

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing, so
TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20.
Two source URLs serving the same bytes store them twice. The tests of
the removed sharing and lock go with them; the two eviction
cancellation tests no longer use the hook.

Model: opus-5-5

internal/imgcache/cache_usage_internal_test.go

@@ -31,8 +31,8 @@ func TestEvictionReadsTheUsageTotal(t *testing.T) {
 	storeEvictionTestVariant(t, cache, testVariantKeyTwo,
 		bytes.Repeat([]byte{0x75}, 200))
 
-	// Storing a source URL again changes its size, so the total must
-	// follow a statement that changes it.
+	// pixa never changes a source's size, but a statement that does must
+	// change the total too.
 	_, err := cache.db.ExecContext(ctx,
 		`UPDATE source_content SET size_bytes = 900 WHERE source_key = ?`,
 		string(kept))

internal/imgcache/eviction_internal_test.go

@@ -948,15 +948,14 @@ func TestStopEvictionInterruptsPassInProgress(t *testing.T) {
 }
 
 // TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context
-// once the oldest of three sources is evicted, while the next one waits
-// for the database, and checks that EvictToLimit then returns
-// context.Canceled, with that source gone and the other two kept, and
-// logs no warning. The test database has one connection. When the
-// deletion of the oldest source's row commits, a commit hook holds the
-// deletion until the test waits for the connection, so the test takes it
-// as soon as the deletion is done. The eviction of the next source then
-// waits for the connection, and the test cancels the context while it
-// waits.
+// while the oldest of three sources is being evicted, and checks that
+// EvictToLimit then returns context.Canceled, with that source gone and
+// the other two kept, and logs no warning. The test database has one
+// connection. When the deletion of the oldest source's row commits, a
+// commit hook holds the deletion until the test waits for the
+// connection, so the test takes it as soon as the deletion is done. The
+// eviction of the next source then waits for the connection, and the
+// test cancels the context while it waits.
 func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
 	t.Parallel()
 
@@ -1007,26 +1006,24 @@ func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) {
 
 	go func() { evicted <- cache.EvictToLimit(ctx) }()
 
-	select {
-	case <-oldestRowDeleted:
-	case err := <-evicted:
-		t.Fatalf("EvictToLimit() returned %v without deleting a row", err)
-	case <-time.After(5 * time.Second):
-		t.Fatal("EvictToLimit never deleted a row")
+	<-oldestRowDeleted
+
+	conn, err := cache.db.Conn(t.Context())
+	if err != nil {
+		t.Fatalf("failed to take the database connection: %v", err)
 	}
 
-	releaseDatabase := holdDatabase(t, cache)
-	defer releaseDatabase()
+	defer func() { _ = conn.Close() }()
 
 	waitForConnectionWaits(t, cache.db, waitsBefore+2)
 	cancel()
 
-	err := <-evicted
+	err = <-evicted
 	t.Logf("EvictToLimit() error = %v", err)
 	t.Logf("log output: %s", logBuf.String())
 
 	// The checks below read the database, so the connection goes back.
-	releaseDatabase()
+	_ = conn.Close()
 
 	setCommitHook(t, cache, nil)
 

internal/imgcache/max_concurrent_processing_internal_test.go

@@ -103,7 +103,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) {
 	// before waiting, it would have read it by now.
 	time.Sleep(100 * time.Millisecond)
 
-	err = os.WriteFile(svc.cache.srcContent.keyToPath(sourceKey),
+	err = os.WriteFile(svc.cache.srcContent.hashToPath(sourceKey),
 		generateTestJPEG(t, 100, 50, color.RGBA{0, 0, 255, 255}), 0o600)
 	if err != nil {
 		t.Fatalf("failed to rewrite the cached source: %v", err)

internal/imgcache/partial_cache_files_internal_test.go

@@ -70,7 +70,7 @@ func cachedSourcePath(
 		t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err)
 	}
 
-	return svc.cache.srcContent.keyToPath(sourceKey)
+	return svc.cache.srcContent.hashToPath(sourceKey)
 }
 
 // reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
@@ -327,7 +327,7 @@ func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
 
 	content := []byte("the whole source image")
 	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
-	path := storage.keyToPath(key)
+	path := storage.hashToPath(key)
 
 	err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
 	if err != nil {

internal/imgcache/service_internal_test.go

@@ -619,7 +619,7 @@ func backdateSource(
 		dbTime, string(key))
 	setSourceLastAccessed(t, cache, key, when)
 
-	path := cache.srcContent.keyToPath(key)
+	path := cache.srcContent.hashToPath(key)
 
 	err := os.Chtimes(path, when, when)
 	if err != nil {

internal/imgcache/storage.go

@@ -49,7 +49,7 @@ func NewContentStorage(baseDir string) (*ContentStorage, error) {
 func (s *ContentStorage) Store(
 	key imagerequest.SourceKey, data []byte,
 ) (int64, error) {
-	path := s.keyToPath(key)
+	path := s.hashToPath(key)
 
 	err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
 	if err != nil {
@@ -66,7 +66,7 @@ func (s *ContentStorage) Store(
 
 // Load returns a reader for the content with the given key.
 func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) {
-	path := s.keyToPath(key)
+	path := s.hashToPath(key)
 
 	f, err := os.Open(path) //nolint:gosec // path derived from source key
 	if err != nil {
@@ -84,7 +84,7 @@ func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error)
 func (s *ContentStorage) LoadWithSize(
 	key imagerequest.SourceKey,
 ) (io.ReadCloser, int64, error) {
-	path := s.keyToPath(key)
+	path := s.hashToPath(key)
 
 	f, err := os.Open(path) //nolint:gosec // path derived from source key
 	if err != nil {
@@ -107,7 +107,7 @@ func (s *ContentStorage) LoadWithSize(
 
 // Delete removes content with the given key.
 func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
-	path := s.keyToPath(key)
+	path := s.hashToPath(key)
 
 	err := os.Remove(path)
 	if err != nil && !os.IsNotExist(err) {
@@ -119,7 +119,7 @@ func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
 
 // Exists checks if content with the given key exists.
 func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool {
-	path := s.keyToPath(key)
+	path := s.hashToPath(key)
 	_, err := os.Stat(path)
 
 	return err == nil
@@ -199,14 +199,14 @@ func syncDirectory(dir string) error {
 	return nil
 }
 
-// keyToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
-func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string {
-	k := string(key)
-	if len(k) < MinHashLength {
-		return filepath.Clean(filepath.Join(s.baseDir, k))
+// hashToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
+func (s *ContentStorage) hashToPath(key imagerequest.SourceKey) string {
+	h := string(key)
+	if len(h) < MinHashLength {
+		return filepath.Clean(filepath.Join(s.baseDir, h))
 	}
 
-	return filepath.Clean(filepath.Join(s.baseDir, k[0:2], k[2:4], k))
+	return filepath.Clean(filepath.Join(s.baseDir, h[0:2], h[2:4], h))
 }
 
 // VariantStorage handles key-based file storage for processed image variants.

internal/imgcache/storage_internal_test.go

@@ -3,66 +3,14 @@ package imgcache
 import (
 	"bytes"
 	"errors"
-	"io"
 	"log/slog"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
 	"sneak.berlin/go/pixa/internal/imagerequest"
 )
 
-func TestContentStorage_StoreAndLoad(t *testing.T) {
-	t.Parallel()
-
-	tmpDir := t.TempDir()
-
-	storage, err := NewContentStorage(tmpDir)
-	if err != nil {
-		t.Fatalf("NewContentStorage() error = %v", err)
-	}
-
-	content := []byte("hello world")
-	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
-
-	size, err := storage.Store(key, content)
-	if err != nil {
-		t.Fatalf("Store() error = %v", err)
-	}
-
-	if size != int64(len(content)) {
-		t.Errorf("Store() size = %d, want %d", size, len(content))
-	}
-
-	// Verify file exists at expected path
-	keyStr := string(key)
-
-	expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr)
-
-	_, err = os.Stat(expectedPath)
-	if err != nil {
-		t.Errorf("File not at expected path %s: %v", expectedPath, err)
-	}
-
-	// Load and verify content
-	r, err := storage.Load(key)
-	if err != nil {
-		t.Fatalf("Load() error = %v", err)
-	}
-
-	defer func() { _ = r.Close() }()
-
-	loaded, err := io.ReadAll(r)
-	if err != nil {
-		t.Fatalf("ReadAll() error = %v", err)
-	}
-
-	if !bytes.Equal(loaded, content) {
-		t.Errorf("Load() content = %q, want %q", loaded, content)
-	}
-}
-
 func TestContentStorage_LoadNotFound(t *testing.T) {
 	t.Parallel()
 
@@ -79,38 +27,6 @@ func TestContentStorage_LoadNotFound(t *testing.T) {
 	}
 }
 
-func TestContentStorage_Delete(t *testing.T) {
-	t.Parallel()
-
-	tmpDir := t.TempDir()
-
-	storage, err := NewContentStorage(tmpDir)
-	if err != nil {
-		t.Fatalf("NewContentStorage() error = %v", err)
-	}
-
-	content := []byte("to be deleted")
-	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
-
-	_, err = storage.Store(key, content)
-	if err != nil {
-		t.Fatalf("Store() error = %v", err)
-	}
-
-	if !storage.Exists(key) {
-		t.Error("Exists() = false, want true")
-	}
-
-	err = storage.Delete(key)
-	if err != nil {
-		t.Fatalf("Delete() error = %v", err)
-	}
-
-	if storage.Exists(key) {
-		t.Error("Exists() = true after delete, want false")
-	}
-}
-
 func TestContentStorage_DeleteNonexistent(t *testing.T) {
 	t.Parallel()
 
@@ -128,35 +44,6 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) {
 	}
 }
 
-func TestContentStorage_HashToPath(t *testing.T) {
-	t.Parallel()
-
-	tmpDir := t.TempDir()
-
-	storage, err := NewContentStorage(tmpDir)
-	if err != nil {
-		t.Fatalf("NewContentStorage() error = %v", err)
-	}
-
-	// Test by storing and verifying the resulting path structure
-	content := []byte("test content for path verification")
-	key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
-
-	_, err = storage.Store(key, content)
-	if err != nil {
-		t.Fatalf("Store() error = %v", err)
-	}
-
-	keyStr := string(key)
-
-	expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr)
-
-	_, err = os.Stat(expectedPath)
-	if err != nil {
-		t.Errorf("File not at expected path %s: %v", expectedPath, err)
-	}
-}
-
 // TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
 // that fails is logged at warn and the store still succeeds.
 func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {