Keep cached sources and variants in one FileStore type (closes #318)
check / check (push) Canceled after 0s

imgcache.FileStore is a directory of files named by a hex key, at
<ab>/<cd>/<key>. It writes a file under a temp name, syncs it and renames
it into place. It opens, reports the size of and deletes the file for a
key. The cache keeps its sources in one FileStore and its variants in
another, so ContentStorage and VariantStorage are gone. FileStore has no
Exists: the reconciliation pass checks a variant's file with Size, and the
tests look for the file on disk. Cache.Variant reads the content type from
the file's first bytes itself. storage.go is now filestore.go, and the
unused testdata/red.avif is gone.

Model: opus-5-5
This commit was merged in pull request #335.
This commit is contained in:
2026-10-26 20:24:33 +01:00
parent 17376d885b
commit ac850823f4
12 changed files with 496 additions and 535 deletions
@@ -67,8 +67,8 @@ func TestReconciliationRemovesDamagedFiles(t *testing.T) {
sourceKey := storeEvictionTestSource(t, cache, testHostCDN, testPathCat, content)
storeEvictionTestVariant(t, cache, testVariantKeyOne, content)
sourcePath := cache.srcContent.keyToPath(sourceKey)
variantPath := cache.variants.keyToPath(testVariantKeyOne)
sourcePath := cache.srcContent.keyToPath(string(sourceKey))
variantPath := cache.variants.keyToPath(string(testVariantKeyOne))
for _, path := range []string{sourcePath, variantPath} {
err := os.Truncate(path, size)
@@ -82,11 +82,13 @@ func TestReconciliationRemovesDamagedFiles(t *testing.T) {
assertWarningNames(t, logs, sourcePath)
assertWarningNames(t, logs, variantPath)
if cache.srcContent.Exists(sourceKey) {
_, err := os.Stat(sourcePath)
if err == nil {
t.Error("the damaged source file is still on disk")
}
if cache.variants.Exists(testVariantKeyOne) {
_, err = os.Stat(variantPath)
if err == nil {
t.Error("the damaged variant file is still on disk")
}
@@ -116,16 +118,19 @@ func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) {
cache, _ := newEvictionTestCache(t, 1<<30)
_, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil))
err := cache.variants.Write(string(testVariantKeyOne), nil)
if err != nil {
t.Fatalf("failed to store an empty variant file: %v", err)
}
variantPath := cache.variants.keyToPath(string(testVariantKeyOne))
logs := reconcileAsAtStartup(t, cache)
assertWarningNames(t, logs, cache.variants.keyToPath(testVariantKeyOne))
assertWarningNames(t, logs, variantPath)
if cache.variants.Exists(testVariantKeyOne) {
_, err = os.Stat(variantPath)
if err == nil {
t.Error("the empty variant file is still on disk")
}
@@ -137,19 +142,19 @@ func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) {
}
}
// TestContentStorage_StoreReplacesFileOfAnotherSize checks that a
// store over a file of another size at the source's path, such as a
// partial file a crash left, replaces it with the content.
func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
// TestFileStore_WriteReplacesFileOfAnotherSize checks that a write over a
// file of another size at the source's path, such as a partial file a crash
// left, replaces it with the content.
func TestFileStore_WriteReplacesFileOfAnotherSize(t *testing.T) {
t.Parallel()
storage, err := NewContentStorage(t.TempDir())
storage, err := NewFileStore(t.TempDir())
if err != nil {
t.Fatalf("NewContentStorage() error = %v", err)
t.Fatalf("NewFileStore() error = %v", err)
}
content := []byte("the whole source image")
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
key := string(imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key())
path := storage.keyToPath(key)
err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
@@ -162,14 +167,14 @@ func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
t.Fatalf("writing a partial file: %v", err)
}
_, err = storage.Store(key, content)
err = storage.Write(key, content)
if err != nil {
t.Fatalf("Store() error = %v", err)
t.Fatalf("Write() error = %v", err)
}
stored, _, err := storage.LoadWithSize(key)
stored, _, err := storage.Open(key)
if err != nil {
t.Fatalf("LoadWithSize() error = %v", err)
t.Fatalf("Open() error = %v", err)
}
defer func() { _ = stored.Close() }()