Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing, so TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20. Two source URLs serving the same bytes store them twice. The tests of the removed sharing and lock go with them; the two eviction cancellation tests no longer use the hook. Model: opus-5-5
This commit is contained in:
@@ -2,8 +2,6 @@ package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
@@ -67,12 +65,12 @@ func cachedSourcePath(
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
contentHash, _, err := svc.cache.LookupSource(t.Context(), req)
|
||||
if err != nil || contentHash == "" {
|
||||
t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err)
|
||||
sourceKey, _, err := svc.cache.LookupSource(t.Context(), req)
|
||||
if err != nil || sourceKey == "" {
|
||||
t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err)
|
||||
}
|
||||
|
||||
return svc.cache.srcContent.hashToPath(contentHash)
|
||||
return svc.cache.srcContent.hashToPath(sourceKey)
|
||||
}
|
||||
|
||||
// reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
|
||||
@@ -316,10 +314,10 @@ func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a
|
||||
// store over a file of another size at the content's path, such as a
|
||||
// TestContentStorage_StoreReplacesFileOfAnotherSize checks that a
|
||||
// store over a file of another size at the source's path, such as a
|
||||
// partial file a crash left, replaces it with the content.
|
||||
func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
storage, err := NewContentStorage(t.TempDir())
|
||||
@@ -328,13 +326,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
}
|
||||
|
||||
content := []byte("the whole source image")
|
||||
sum := sha256.Sum256(content)
|
||||
hash := ContentHash(hex.EncodeToString(sum[:]))
|
||||
path := storage.hashToPath(hash)
|
||||
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
|
||||
path := storage.hashToPath(key)
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
|
||||
if err != nil {
|
||||
t.Fatalf("creating the content's directory: %v", err)
|
||||
t.Fatalf("creating the source's directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, content[:5], StorageFilePerm)
|
||||
@@ -342,12 +339,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
|
||||
t.Fatalf("writing a partial file: %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.StoreHashed(hash, content)
|
||||
_, err = storage.Store(key, content)
|
||||
if err != nil {
|
||||
t.Fatalf("StoreHashed() error = %v", err)
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
stored, err := storage.Load(hash)
|
||||
stored, err := storage.Load(key)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user