sneak/pixa
1
0
Fork 0
Code Issues 20 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
Details

Browse Source 
A source is now one source_content row and one file named by its
SourceKey, the SHA-256 of its host, path and query, made as the variant
key is. A store writes the file, then upserts the row, and removes the
file when the row cannot be written; an eviction deletes the row, then
the file, with no lock. source_metadata, the JSON files under
cache/metadata/, MetadataStorage, the content lock, the eviction
transaction and its test hook are gone, as is the reconciliation pass
that dropped a source row whose file is missing. Two source URLs
serving the same bytes store them twice.

Model: opus-5-5
This commit was merged in pull request #324.
This commit is contained in:
clawbot
2026-10-21 16:31:33 +02:00
parent 00d8ba26c5
commit a19cca2f34
24 changed files with 622 additions and 1468 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imgcache/storage_internal_test.go
+17 -184
View File
@@ -24,8 +24,9 @@ func TestContentStorage_StoreAndLoad(t *testing.T) {
}
content := []byte("hello world")
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
hash, size, err := storage.Store(bytes.NewReader(content))
size, err := storage.Store(key, content)
if err != nil {
t.Fatalf("Store() error = %v", err)
}
@@ -34,14 +35,10 @@ func TestContentStorage_StoreAndLoad(t *testing.T) {
t.Errorf("Store() size = %d, want %d", size, len(content))
}
if hash == "" {
t.Error("Store() returned empty hash")
}
// Verify file exists at expected path
hashStr := string(hash)
keyStr := string(key)
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr)
_, err = os.Stat(expectedPath)
if err != nil {
@@ -49,7 +46,7 @@ func TestContentStorage_StoreAndLoad(t *testing.T) {
}
// Load and verify content
r, err := storage.Load(hash)
r, err := storage.Load(key)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
@@ -66,33 +63,6 @@ func TestContentStorage_StoreAndLoad(t *testing.T) {
}
}
func TestContentStorage_StoreIdempotent(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
storage, err := NewContentStorage(tmpDir)
if err != nil {
t.Fatalf("NewContentStorage() error = %v", err)
}
content := []byte("same content")
hash1, _, err := storage.Store(bytes.NewReader(content))
if err != nil {
t.Fatalf("Store() first error = %v", err)
}
hash2, _, err := storage.Store(bytes.NewReader(content))
if err != nil {
t.Fatalf("Store() second error = %v", err)
}
if hash1 != hash2 {
t.Errorf("Store() hashes differ: %s vs %s", hash1, hash2)
}
}
func TestContentStorage_LoadNotFound(t *testing.T) {
t.Parallel()
@@ -103,7 +73,7 @@ func TestContentStorage_LoadNotFound(t *testing.T) {
t.Fatalf("NewContentStorage() error = %v", err)
}
_, err = storage.Load(ContentHash("nonexistent"))
_, err = storage.Load(imagerequest.SourceKey("nonexistent"))
if !errors.Is(err, ErrNotFound) {
t.Errorf("Load() error = %v, want ErrNotFound", err)
}
@@ -120,22 +90,23 @@ func TestContentStorage_Delete(t *testing.T) {
}
content := []byte("to be deleted")
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
hash, _, err := storage.Store(bytes.NewReader(content))
_, err = storage.Store(key, content)
if err != nil {
t.Fatalf("Store() error = %v", err)
}
if !storage.Exists(hash) {
if !storage.Exists(key) {
t.Error("Exists() = false, want true")
}
err = storage.Delete(hash)
err = storage.Delete(key)
if err != nil {
t.Fatalf("Delete() error = %v", err)
}
if storage.Exists(hash) {
if storage.Exists(key) {
t.Error("Exists() = true after delete, want false")
}
}
@@ -151,13 +122,13 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) {
}
// Should not error
err = storage.Delete(ContentHash("nonexistent"))
err = storage.Delete(imagerequest.SourceKey("nonexistent"))
if err != nil {
t.Errorf("Delete() error = %v, want nil", err)
}
}
func TestContentStorage_HashToPath(t *testing.T) {
func TestContentStorage_KeyToPath(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
@@ -169,15 +140,16 @@ func TestContentStorage_HashToPath(t *testing.T) {
// Test by storing and verifying the resulting path structure
content := []byte("test content for path verification")
key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()
hash, _, err := storage.Store(bytes.NewReader(content))
_, err = storage.Store(key, content)
if err != nil {
t.Fatalf("Store() error = %v", err)
}
hashStr := string(hash)
keyStr := string(key)
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr)
_, err = os.Stat(expectedPath)
if err != nil {
@@ -185,145 +157,6 @@ func TestContentStorage_HashToPath(t *testing.T) {
}
}
func TestMetadataStorage_StoreAndLoad(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
storage, err := NewMetadataStorage(tmpDir)
if err != nil {
t.Fatalf("NewMetadataStorage() error = %v", err)
}
meta := &SourceMetadata{
Host: testHostCDN,
Path: testPathCat,
ContentHash: "abc123",
StatusCode: 200,
ContentType: testContentTypeJPEG,
FetchedAt: 1704067200,
ETag: `"etag123"`,
}
pathHash := HashPath(testPathCat)
err = storage.Store(testHostCDN, pathHash, meta)
if err != nil {
t.Fatalf("Store() error = %v", err)
}
// Verify file exists at expected path
expectedPath := filepath.Join(tmpDir, testHostCDN, string(pathHash)+".json")
_, err = os.Stat(expectedPath)
if err != nil {
t.Errorf("File not at expected path %s: %v", expectedPath, err)
}
// Load and verify
loaded, err := storage.Load(testHostCDN, pathHash)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if loaded.Host != meta.Host {
t.Errorf("Host = %q, want %q", loaded.Host, meta.Host)
}
if loaded.Path != meta.Path {
t.Errorf("Path = %q, want %q", loaded.Path, meta.Path)
}
if loaded.ContentHash != meta.ContentHash {
t.Errorf("ContentHash = %q, want %q", loaded.ContentHash, meta.ContentHash)
}
if loaded.StatusCode != meta.StatusCode {
t.Errorf("StatusCode = %d, want %d", loaded.StatusCode, meta.StatusCode)
}
if loaded.ETag != meta.ETag {
t.Errorf("ETag = %q, want %q", loaded.ETag, meta.ETag)
}
}
func TestMetadataStorage_LoadNotFound(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
storage, err := NewMetadataStorage(tmpDir)
if err != nil {
t.Fatalf("NewMetadataStorage() error = %v", err)
}
_, err = storage.Load(testHostExample, PathHash("nonexistent"))
if !errors.Is(err, ErrNotFound) {
t.Errorf("Load() error = %v, want ErrNotFound", err)
}
}
func TestMetadataStorage_Delete(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
storage, err := NewMetadataStorage(tmpDir)
if err != nil {
t.Fatalf("NewMetadataStorage() error = %v", err)
}
meta := &SourceMetadata{
Host: testHostExample,
Path: "/test.jpg",
StatusCode: 200,
}
pathHash := HashPath("/test.jpg")
err = storage.Store(testHostExample, pathHash, meta)
if err != nil {
t.Fatalf("Store() error = %v", err)
}
if !storage.Exists(testHostExample, pathHash) {
t.Error("Exists() = false, want true")
}
err = storage.Delete(testHostExample, pathHash)
if err != nil {
t.Fatalf("Delete() error = %v", err)
}
if storage.Exists(testHostExample, pathHash) {
t.Error("Exists() = true after delete, want false")
}
}
func TestHashPath(t *testing.T) {
t.Parallel()
// Same input should produce same hash
hash1 := HashPath(testPathCat)
hash2 := HashPath(testPathCat)
if hash1 != hash2 {
t.Errorf("HashPath() not deterministic: %s vs %s", hash1, hash2)
}
// Different input should produce different hash
hash3 := HashPath("/photos/dog.jpg")
if hash1 == hash3 {
t.Error("HashPath() produced same hash for different inputs")
}
// Hash should be 64 hex chars (256 bits)
if len(string(hash1)) != 64 {
t.Errorf("HashPath() length = %d, want 64", len(string(hash1)))
}
}
// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
// that fails is logged at warn and the store still succeeds.
func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {