Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s

A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5
This commit was merged in pull request #324.
This commit is contained in:
2026-10-21 16:31:33 +02:00
parent 00d8ba26c5
commit a19cca2f34
24 changed files with 622 additions and 1468 deletions
@@ -2,8 +2,6 @@ package imgcache
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"io"
"log/slog"
"os"
@@ -67,12 +65,12 @@ func cachedSourcePath(
) string {
t.Helper()
contentHash, _, err := svc.cache.LookupSource(t.Context(), req)
if err != nil || contentHash == "" {
t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err)
sourceKey, _, err := svc.cache.LookupSource(t.Context(), req)
if err != nil || sourceKey == "" {
t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err)
}
return svc.cache.srcContent.hashToPath(contentHash)
return svc.cache.srcContent.keyToPath(sourceKey)
}
// reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
@@ -316,10 +314,10 @@ func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) {
}
}
// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a
// store over a file of another size at the content's path, such as a
// TestContentStorage_StoreReplacesFileOfAnotherSize checks that a
// store over a file of another size at the source's path, such as a
// partial file a crash left, replaces it with the content.
func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
t.Parallel()
storage, err := NewContentStorage(t.TempDir())
@@ -328,13 +326,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
}
content := []byte("the whole source image")
sum := sha256.Sum256(content)
hash := ContentHash(hex.EncodeToString(sum[:]))
path := storage.hashToPath(hash)
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
path := storage.keyToPath(key)
err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
if err != nil {
t.Fatalf("creating the content's directory: %v", err)
t.Fatalf("creating the source's directory: %v", err)
}
err = os.WriteFile(path, content[:5], StorageFilePerm)
@@ -342,12 +339,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
t.Fatalf("writing a partial file: %v", err)
}
_, err = storage.StoreHashed(hash, content)
_, err = storage.Store(key, content)
if err != nil {
t.Fatalf("StoreHashed() error = %v", err)
t.Fatalf("Store() error = %v", err)
}
stored, err := storage.Load(hash)
stored, err := storage.Load(key)
if err != nil {
t.Fatalf("Load() error = %v", err)
}