Store a source the way a variant is stored, one row and one file per source URL (closes #308)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing, so TestReconciliationReadsAPageAtATime now counts 15 rows read, not 20. Two source URLs serving the same bytes store them twice. The tests of the removed sharing and lock go with them; the two eviction cancellation tests no longer use the hook. Model: opus-5-5
This commit is contained in:
@@ -3,96 +3,14 @@ package imgcache
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
func TestContentStorage_StoreAndLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("hello world")
|
||||
|
||||
hash, size, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(content)) {
|
||||
t.Errorf("Store() size = %d, want %d", size, len(content))
|
||||
}
|
||||
|
||||
if hash == "" {
|
||||
t.Error("Store() returned empty hash")
|
||||
}
|
||||
|
||||
// Verify file exists at expected path
|
||||
hashStr := string(hash)
|
||||
|
||||
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
|
||||
|
||||
_, err = os.Stat(expectedPath)
|
||||
if err != nil {
|
||||
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
||||
}
|
||||
|
||||
// Load and verify content
|
||||
r, err := storage.Load(hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = r.Close() }()
|
||||
|
||||
loaded, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll() error = %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(loaded, content) {
|
||||
t.Errorf("Load() content = %q, want %q", loaded, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_StoreIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("same content")
|
||||
|
||||
hash1, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() first error = %v", err)
|
||||
}
|
||||
|
||||
hash2, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() second error = %v", err)
|
||||
}
|
||||
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("Store() hashes differ: %s vs %s", hash1, hash2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_LoadNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -103,43 +21,12 @@ func TestContentStorage_LoadNotFound(t *testing.T) {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.Load(ContentHash("nonexistent"))
|
||||
_, err = storage.Load(imagerequest.SourceKey("nonexistent"))
|
||||
if !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("Load() error = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_Delete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
content := []byte("to be deleted")
|
||||
|
||||
hash, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
if !storage.Exists(hash) {
|
||||
t.Error("Exists() = false, want true")
|
||||
}
|
||||
|
||||
err = storage.Delete(hash)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
|
||||
if storage.Exists(hash) {
|
||||
t.Error("Exists() = true after delete, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_DeleteNonexistent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -151,179 +38,12 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not error
|
||||
err = storage.Delete(ContentHash("nonexistent"))
|
||||
err = storage.Delete(imagerequest.SourceKey("nonexistent"))
|
||||
if err != nil {
|
||||
t.Errorf("Delete() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentStorage_HashToPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewContentStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewContentStorage() error = %v", err)
|
||||
}
|
||||
|
||||
// Test by storing and verifying the resulting path structure
|
||||
content := []byte("test content for path verification")
|
||||
|
||||
hash, _, err := storage.Store(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
hashStr := string(hash)
|
||||
|
||||
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
|
||||
|
||||
_, err = os.Stat(expectedPath)
|
||||
if err != nil {
|
||||
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataStorage_StoreAndLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewMetadataStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMetadataStorage() error = %v", err)
|
||||
}
|
||||
|
||||
meta := &SourceMetadata{
|
||||
Host: testHostCDN,
|
||||
Path: testPathCat,
|
||||
ContentHash: "abc123",
|
||||
StatusCode: 200,
|
||||
ContentType: testContentTypeJPEG,
|
||||
FetchedAt: 1704067200,
|
||||
ETag: `"etag123"`,
|
||||
}
|
||||
|
||||
pathHash := HashPath(testPathCat)
|
||||
|
||||
err = storage.Store(testHostCDN, pathHash, meta)
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists at expected path
|
||||
expectedPath := filepath.Join(tmpDir, testHostCDN, string(pathHash)+".json")
|
||||
|
||||
_, err = os.Stat(expectedPath)
|
||||
if err != nil {
|
||||
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
||||
}
|
||||
|
||||
// Load and verify
|
||||
loaded, err := storage.Load(testHostCDN, pathHash)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
if loaded.Host != meta.Host {
|
||||
t.Errorf("Host = %q, want %q", loaded.Host, meta.Host)
|
||||
}
|
||||
|
||||
if loaded.Path != meta.Path {
|
||||
t.Errorf("Path = %q, want %q", loaded.Path, meta.Path)
|
||||
}
|
||||
|
||||
if loaded.ContentHash != meta.ContentHash {
|
||||
t.Errorf("ContentHash = %q, want %q", loaded.ContentHash, meta.ContentHash)
|
||||
}
|
||||
|
||||
if loaded.StatusCode != meta.StatusCode {
|
||||
t.Errorf("StatusCode = %d, want %d", loaded.StatusCode, meta.StatusCode)
|
||||
}
|
||||
|
||||
if loaded.ETag != meta.ETag {
|
||||
t.Errorf("ETag = %q, want %q", loaded.ETag, meta.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataStorage_LoadNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewMetadataStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMetadataStorage() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = storage.Load(testHostExample, PathHash("nonexistent"))
|
||||
if !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("Load() error = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataStorage_Delete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
storage, err := NewMetadataStorage(tmpDir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMetadataStorage() error = %v", err)
|
||||
}
|
||||
|
||||
meta := &SourceMetadata{
|
||||
Host: testHostExample,
|
||||
Path: "/test.jpg",
|
||||
StatusCode: 200,
|
||||
}
|
||||
|
||||
pathHash := HashPath("/test.jpg")
|
||||
|
||||
err = storage.Store(testHostExample, pathHash, meta)
|
||||
if err != nil {
|
||||
t.Fatalf("Store() error = %v", err)
|
||||
}
|
||||
|
||||
if !storage.Exists(testHostExample, pathHash) {
|
||||
t.Error("Exists() = false, want true")
|
||||
}
|
||||
|
||||
err = storage.Delete(testHostExample, pathHash)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
|
||||
if storage.Exists(testHostExample, pathHash) {
|
||||
t.Error("Exists() = true after delete, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Same input should produce same hash
|
||||
hash1 := HashPath(testPathCat)
|
||||
hash2 := HashPath(testPathCat)
|
||||
|
||||
if hash1 != hash2 {
|
||||
t.Errorf("HashPath() not deterministic: %s vs %s", hash1, hash2)
|
||||
}
|
||||
|
||||
// Different input should produce different hash
|
||||
hash3 := HashPath("/photos/dog.jpg")
|
||||
|
||||
if hash1 == hash3 {
|
||||
t.Error("HashPath() produced same hash for different inputs")
|
||||
}
|
||||
|
||||
// Hash should be 64 hex chars (256 bits)
|
||||
if len(string(hash1)) != 64 {
|
||||
t.Errorf("HashPath() length = %d, want 64", len(string(hash1)))
|
||||
}
|
||||
}
|
||||
|
||||
// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
|
||||
// that fails is logged at warn and the store still succeeds.
|
||||
func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user