Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone. Two source URLs serving the same bytes store them twice. Reconciliation still drops a source row whose file is missing, as TestReconciliationReadsAPageAtATime counts those reads. The tests of the removed sharing, lock and hook go with them. Model: opus-5-5
This commit is contained in:
@@ -3,96 +3,14 @@ package imgcache
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
func TestContentStorage_StoreAndLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("hello world")
|
||||
|
||||
hash, size, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(content)) {
|
||||
t.Errorf("Store() size = %d, want %d", size, len(content))
|
||||
}
|
||||
|
||||
if hash == "" {
|
||||
t.Error("Store() returned empty hash")
|
||||
}
|
||||
|
||||
// Verify file exists at expected path
|
||||
hashStr := string(hash)
|
||||
|
||||
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
|
||||
|
||||
_, err = os.Stat(expectedPath)
|
||||
if err != nil {
|
||||
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
||||
}
|
||||
|
||||
// Load and verify content
|
||||
r, err := storage.Load(hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = r.Close() }()
|
||||
|
||||
loaded, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(loaded, content) {
|
||||
t.Errorf("Load() content = %q, want %q", loaded, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_StoreIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("same content")
|
||||
|
||||
hash1, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() first error = %v", err)
|
||||
}
|
||||
|
||||
hash2, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() second error = %v", err)
|
||||
}
|
||||
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("Store() hashes differ: %s vs %s", hash1, hash2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_LoadNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -103,43 +21,12 @@ func TestContentStorage_LoadNotFound(t *testing.T) {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.Load(ContentHash("nonexistent"))
|
||||
_, err = storage.Load(imagerequest.SourceKey("nonexistent"))
|
||||
if !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("Load() error = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_Delete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("to be deleted")
|
||||
|
||||
hash, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
if !storage.Exists(hash) {
|
||||
t.Error("Exists() = false, want true")
|
||||
}
|
||||
|
||||
err = storage.Delete(hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
|
||||
if storage.Exists(hash) {
|
||||
t.Error("Exists() = true after delete, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_DeleteNonexistent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -151,179 +38,12 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not error
|
||||
err = storage.Delete(ContentHash("nonexistent"))
|
||||
err = storage.Delete(imagerequest.SourceKey("nonexistent"))
|
||||
if err != nil {
|
||||
t.Errorf("Delete() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_HashToPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
// Test by storing and verifying the resulting path structure
|
||||
content := []byte("test content for path verification")
|
||||
|
||||
hash, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
hashStr := string(hash)
|
||||
|
||||
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
|
||||
|
||||
_, err = os.Stat(expectedPath)
|
||||
if err != nil {
|
||||
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataStorage_StoreAndLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewMetadataStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMetadataStorage() error = %v", err)
|
||||
}
|
||||
|
||||
meta := &SourceMetadata{
|
||||
Host: testHostCDN,
|
||||
Path: testPathCat,
|
||||
ContentHash: "abc123",
|
||||
StatusCode: 200,
|
||||
ContentType: testContentTypeJPEG,
|
||||
FetchedAt: 1704067200,
|
||||
ETag: `"etag123"`,
|
||||
}
|
||||
|
||||
pathHash := HashPath(testPathCat)
|
||||
|
||||
err = storage.Store(testHostCDN, pathHash, meta)
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists at expected path
|
||||
expectedPath := filepath.Join(tmpDir, testHostCDN, string(pathHash)+".json")
|
||||
|
||||
_, err = os.Stat(expectedPath)
|
||||
if err != nil {
|
||||
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
||||
}
|
||||
|
||||
// Load and verify
|
||||
loaded, err := storage.Load(testHostCDN, pathHash)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
if loaded.Host != meta.Host {
|
||||
t.Errorf("Host = %q, want %q", loaded.Host, meta.Host)
|
||||
}
|
||||
|
||||
if loaded.Path != meta.Path {
|
||||
t.Errorf("Path = %q, want %q", loaded.Path, meta.Path)
|
||||
}
|
||||
|
||||
if loaded.ContentHash != meta.ContentHash {
|
||||
t.Errorf("ContentHash = %q, want %q", loaded.ContentHash, meta.ContentHash)
|
||||
}
|
||||
|
||||
if loaded.StatusCode != meta.StatusCode {
|
||||
t.Errorf("StatusCode = %d, want %d", loaded.StatusCode, meta.StatusCode)
|
||||
}
|
||||
|
||||
if loaded.ETag != meta.ETag {
|
||||
t.Errorf("ETag = %q, want %q", loaded.ETag, meta.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataStorage_LoadNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewMetadataStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMetadataStorage() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.Load(testHostExample, PathHash("nonexistent"))
|
||||
if !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("Load() error = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataStorage_Delete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewMetadataStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMetadataStorage() error = %v", err)
|
||||
}
|
||||
|
||||
meta := &SourceMetadata{
|
||||
Host: testHostExample,
|
||||
Path: "/test.jpg",
|
||||
StatusCode: 200,
|
||||
}
|
||||
|
||||
pathHash := HashPath("/test.jpg")
|
||||
|
||||
err = storage.Store(testHostExample, pathHash, meta)
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
if !storage.Exists(testHostExample, pathHash) {
|
||||
t.Error("Exists() = false, want true")
|
||||
}
|
||||
|
||||
err = storage.Delete(testHostExample, pathHash)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
|
||||
if storage.Exists(testHostExample, pathHash) {
|
||||
t.Error("Exists() = true after delete, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Same input should produce same hash
|
||||
hash1 := HashPath(testPathCat)
|
||||
hash2 := HashPath(testPathCat)
|
||||
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("HashPath() not deterministic: %s vs %s", hash1, hash2)
|
||||
}
|
||||
|
||||
// Different input should produce different hash
|
||||
hash3 := HashPath("/photos/dog.jpg")
|
||||
|
||||
if hash1 == hash3 {
|
||||
t.Error("HashPath() produced same hash for different inputs")
|
||||
}
|
||||
|
||||
// Hash should be 64 hex chars (256 bits)
|
||||
if len(string(hash1)) != 64 {
|
||||
t.Errorf("HashPath() length = %d, want 64", len(string(hash1)))
|
||||
}
|
||||
}
|
||||
|
||||
// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
|
||||
// that fails is logged at warn and the store still succeeds.
|
||||
func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user