Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5
This commit is contained in:
@@ -2,8 +2,6 @@ package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
@@ -67,12 +65,12 @@ func cachedSourcePath(
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
contentHash, _, err := svc.cache.LookupSource(t.Context(), req)
|
||||
if err != nil || contentHash == "" {
|
||||
t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err)
|
||||
sourceKey, _, err := svc.cache.LookupSource(t.Context(), req)
|
||||
if err != nil || sourceKey == "" {
|
||||
t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err)
|
||||
}
|
||||
|
||||
return svc.cache.srcContent.hashToPath(contentHash)
|
||||
return svc.cache.srcContent.keyToPath(sourceKey)
|
||||
}
|
||||
|
||||
// reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
|
||||
@@ -316,10 +314,10 @@ func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a
|
||||
// store over a file of another size at the content's path, such as a
|
||||
// TestContentStorage_StoreReplacesFileOfAnotherSize checks that a
|
||||
// store over a file of another size at the source's path, such as a
|
||||
// partial file a crash left, replaces it with the content.
|
||||
func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
storage, err := NewContentStorage(t.TempDir())
|
||||
@@ -328,13 +326,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
}
|
||||
|
||||
content := []byte("the whole source image")
|
||||
sum := sha256.Sum256(content)
|
||||
hash := ContentHash(hex.EncodeToString(sum[:]))
|
||||
path := storage.hashToPath(hash)
|
||||
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
|
||||
path := storage.keyToPath(key)
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
t.Fatalf("creating the content's directory: %v", err)
|
||||
t.Fatalf("creating the source's directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, content[:5], StorageFilePerm)
|
||||
@@ -342,12 +339,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
t.Fatalf("writing a partial file: %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.StoreHashed(hash, content)
|
||||
_, err = storage.Store(key, content)
|
||||
if err != nil {
|
||||
t.Fatalf("StoreHashed() error = %v", err)
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
stored, err := storage.Load(hash)
|
||||
stored, err := storage.Load(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user