Split internal/imgcache into the image proxy and the cache (closes #317)

The new package internal/imageproxy holds Proxy, which was imgcache.Service,
with its Config, ImageResponse, CacheStatus and errors. NewProxy makes and
registers the counters of upstream fetches and transforms. internal/imgcache
holds the cache only, with its hit and miss counters, and imgcache.CacheConfig
is imgcache.Config. The cache reads and writes each kind it keeps with one pair
of methods: Variant and PutVariant, Source and PutSource, Failure and
PutFailure. Source replaces LookupSource followed by GetSourceContent.
imgcache.go and module.go are gone. The proxy's tests moved with it. The
cache's tests check that the reconciliation pass removes an empty or cut file
and its row. The proxy's tests remove them by hand, then check that Get fetches
or converts again.

Model: opus-5-5

internal/imgcache/partial_cache_files_internal_test.go

@@ -12,67 +12,6 @@ import (
 	"sneak.berlin/go/pixa/internal/imagerequest"
 )
 
-// getImage calls Get for req and returns the image it answered with and
-// its cache status.
-func getImage(
-	t *testing.T, svc *Service, req *imagerequest.Request,
-) ([]byte, CacheStatus) {
-	t.Helper()
-
-	resp, err := svc.Get(t.Context(), req)
-	if err != nil {
-		t.Fatalf("Get() of quality %d error = %v", req.Transform.Quality, err)
-	}
-
-	defer func() { _ = resp.Content.Close() }()
-
-	body, err := io.ReadAll(resp.Content)
-	if err != nil {
-		t.Fatalf("reading the image of quality %d: %v", req.Transform.Quality, err)
-	}
-
-	return body, resp.CacheStatus
-}
-
-// fileSize returns the size of the file at path.
-func fileSize(t *testing.T, path string) int64 {
-	t.Helper()
-
-	info, err := os.Stat(path)
-	if err != nil {
-		t.Fatalf("stat %s: %v", path, err)
-	}
-
-	return info.Size()
-}
-
-// truncateFile cuts the file at path down to size bytes.
-func truncateFile(t *testing.T, path string, size int64) {
-	t.Helper()
-
-	t.Logf("cutting %s from %d to %d bytes",
-		filepath.Base(path), fileSize(t, path), size)
-
-	err := os.Truncate(path, size)
-	if err != nil {
-		t.Fatalf("truncating %s: %v", path, err)
-	}
-}
-
-// cachedSourcePath returns the path of the cached source file req uses.
-func cachedSourcePath(
-	t *testing.T, svc *Service, req *imagerequest.Request,
-) string {
-	t.Helper()
-
-	sourceKey, err := svc.cache.LookupSource(t.Context(), req)
-	if err != nil || sourceKey == "" {
-		t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err)
-	}
-
-	return svc.cache.srcContent.keyToPath(sourceKey)
-}
-
 // reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
 // when it starts, and returns what the pass logged.
 func reconcileAsAtStartup(t *testing.T, cache *Cache) string {
@@ -107,127 +46,65 @@ func assertWarningNames(t *testing.T, logs, path string) {
 	t.Errorf("reconciliation logged no warning naming %s", path)
 }
 
-// TestService_Get_FetchesAgainForEmptyCachedSource empties the cached source
-// file, as a crash or a restored snapshot could, and runs the reconciliation
-// pass, which must remove it with a warning naming it. It then asks for two
-// new sizes of the source: the first must fetch it again and cache it whole,
-// so that the second is made from the cache without a third fetch.
-func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) {
+// TestReconciliationRemovesDamagedFiles empties a cached source file and a
+// cached variant file, or cuts them to half their size, as a crash or a
+// restored snapshot could. It checks that the reconciliation pass removes
+// each file and its row, with a warning naming the file.
+func TestReconciliationRemovesDamagedFiles(t *testing.T) {
 	t.Parallel()
 
-	svc, fixtures := SetupTestService(t)
+	content := []byte("the whole cached image")
 
-	getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
+	for name, size := range map[string]int64{
+		"empty":   0,
+		"partial": int64(len(content) / 2),
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
 
-	path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
-	wholeSize := fileSize(t, path)
+			cache, _ := newEvictionTestCache(t, 1<<30)
 
-	truncateFile(t, path, 0)
-	assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)
+			sourceKey := storeEvictionTestSource(t, cache, testHostCDN, testPathCat, content)
+			storeEvictionTestVariant(t, cache, testVariantKeyOne, content)
 
-	getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover))
-	getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover))
+			sourcePath := cache.srcContent.keyToPath(sourceKey)
+			variantPath := cache.variants.keyToPath(testVariantKeyOne)
 
-	fetches := readCounters(svc.cache).upstreamFetchCount
-	t.Logf("upstream fetches = %d", fetches)
+			for _, path := range []string{sourcePath, variantPath} {
+				err := os.Truncate(path, size)
+				if err != nil {
+					t.Fatalf("cutting %s to %d bytes: %v", path, size, err)
+				}
+			}
 
-	if fetches != 2 {
-		t.Errorf("upstream fetches = %d, want 2: the empty cached source "+
-			"was kept and fetched again on every new size", fetches)
+			logs := reconcileAsAtStartup(t, cache)
+
+			assertWarningNames(t, logs, sourcePath)
+			assertWarningNames(t, logs, variantPath)
+
+			if cache.srcContent.Exists(sourceKey) {
+				t.Error("the damaged source file is still on disk")
+			}
+
+			if cache.variants.Exists(testVariantKeyOne) {
+				t.Error("the damaged variant file is still on disk")
+			}
+
+			rows := countRows(t, cache,
+				`SELECT COUNT(*) FROM source_content WHERE source_key = ?`,
+				string(sourceKey))
+			if rows != 0 {
+				t.Errorf("the damaged source file kept %d rows, want 0", rows)
+			}
+
+			rows = countRows(t, cache,
+				`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`,
+				string(testVariantKeyOne))
+			if rows != 0 {
+				t.Errorf("the damaged variant file kept %d rows, want 0", rows)
+			}
+		})
 	}
-
-	size := fileSize(t, path)
-	if size != wholeSize {
-		t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
-	}
-}
-
-// TestService_Get_FetchesAgainForPartialCachedSource cuts the cached source
-// file to half its size, as a crash or a restored snapshot could, and runs
-// the reconciliation pass, which must remove it with a warning naming it.
-// It then asks for two new sizes of the source: the first must fetch it
-// again rather than fail to decode the partial file, and the second is
-// made from the cache.
-func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) {
-	t.Parallel()
-
-	svc, fixtures := SetupTestService(t)
-
-	getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
-
-	path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
-	wholeSize := fileSize(t, path)
-
-	truncateFile(t, path, wholeSize/2)
-	assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)
-
-	getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover))
-	getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover))
-
-	fetches := readCounters(svc.cache).upstreamFetchCount
-	t.Logf("upstream fetches = %d", fetches)
-
-	if fetches != 2 {
-		t.Errorf("upstream fetches = %d, want 2", fetches)
-	}
-
-	size := fileSize(t, path)
-	if size != wholeSize {
-		t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
-	}
-}
-
-// assertConvertedAgain caches a variant of the photo, cuts its file to size
-// bytes and runs the reconciliation pass, which must remove it with a
-// warning naming it. It then checks that Get converts the image again
-// instead of serving the file, and that the next Get serves the whole
-// variant from the cache.
-func assertConvertedAgain(t *testing.T, size int64) {
-	t.Helper()
-
-	svc, fixtures := SetupTestService(t)
-	req := photoVariant(fixtures, 85, imagerequest.FitCover)
-
-	want, _ := getImage(t, svc, req)
-
-	path := svc.cache.variants.keyToPath(imagerequest.CacheKey(req))
-
-	truncateFile(t, path, size)
-	assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)
-
-	got, status := getImage(t, svc, req)
-	t.Logf("after the cut: %d bytes, %s", len(got), status)
-
-	if status != CacheMiss || !bytes.Equal(got, want) {
-		t.Errorf("after the cut: Get() = %d bytes, %s; want the %d bytes "+
-			"converted again, %s", len(got), status, len(want), CacheMiss)
-	}
-
-	got, status = getImage(t, svc, req)
-	t.Logf("next request: %d bytes, %s", len(got), status)
-
-	if status != CacheHit || !bytes.Equal(got, want) {
-		t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s",
-			len(got), status, len(want), CacheHit)
-	}
-}
-
-// TestService_Get_ConvertsAgainForEmptyCachedVariant empties a cached
-// variant file, as a crash or a restored snapshot could, and checks that it
-// is converted again rather than served empty.
-func TestService_Get_ConvertsAgainForEmptyCachedVariant(t *testing.T) {
-	t.Parallel()
-
-	assertConvertedAgain(t, 0)
-}
-
-// TestService_Get_ConvertsAgainForPartialCachedVariant cuts a cached
-// variant file to ten bytes, as a crash or a restored snapshot could, and
-// checks that it is converted again rather than served cut short.
-func TestService_Get_ConvertsAgainForPartialCachedVariant(t *testing.T) {
-	t.Parallel()
-
-	assertConvertedAgain(t, 10)
 }
 
 // TestReconciliationRemovesEmptyVariantFileWithoutRow checks that the