clawbot
5b17d1f555
check / check (push) Successful in 10m10s
Each upstream host's semaphore now counts the fetches holding or waiting for one of its slots, and is removed from hostSems when the last of them gives its slot back or stops waiting, so a long-running pixad no longer keeps one semaphore per host it ever fetched from. VariantStorage.Delete removes the variant's .meta file too, a missing one not being an error; DeleteWithMeta, which eviction called for that, is gone. Model: opus-5-5
511 lines
12 KiB
Go
511 lines
12 KiB
Go
package imgcache
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestContentStorage_StoreAndLoad(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewContentStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewContentStorage() error = %v", err)
|
|
}
|
|
|
|
content := []byte("hello world")
|
|
|
|
hash, size, err := storage.Store(bytes.NewReader(content))
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v", err)
|
|
}
|
|
|
|
if size != int64(len(content)) {
|
|
t.Errorf("Store() size = %d, want %d", size, len(content))
|
|
}
|
|
|
|
if hash == "" {
|
|
t.Error("Store() returned empty hash")
|
|
}
|
|
|
|
// Verify file exists at expected path
|
|
hashStr := string(hash)
|
|
|
|
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
|
|
|
|
_, err = os.Stat(expectedPath)
|
|
if err != nil {
|
|
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
|
}
|
|
|
|
// Load and verify content
|
|
r, err := storage.Load(hash)
|
|
if err != nil {
|
|
t.Fatalf("Load() error = %v", err)
|
|
}
|
|
|
|
defer func() { _ = r.Close() }()
|
|
|
|
loaded, err := io.ReadAll(r)
|
|
if err != nil {
|
|
t.Fatalf("ReadAll() error = %v", err)
|
|
}
|
|
|
|
if !bytes.Equal(loaded, content) {
|
|
t.Errorf("Load() content = %q, want %q", loaded, content)
|
|
}
|
|
}
|
|
|
|
func TestContentStorage_StoreIdempotent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewContentStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewContentStorage() error = %v", err)
|
|
}
|
|
|
|
content := []byte("same content")
|
|
|
|
hash1, _, err := storage.Store(bytes.NewReader(content))
|
|
if err != nil {
|
|
t.Fatalf("Store() first error = %v", err)
|
|
}
|
|
|
|
hash2, _, err := storage.Store(bytes.NewReader(content))
|
|
if err != nil {
|
|
t.Fatalf("Store() second error = %v", err)
|
|
}
|
|
|
|
if hash1 != hash2 {
|
|
t.Errorf("Store() hashes differ: %s vs %s", hash1, hash2)
|
|
}
|
|
}
|
|
|
|
func TestContentStorage_LoadNotFound(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewContentStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewContentStorage() error = %v", err)
|
|
}
|
|
|
|
_, err = storage.Load(ContentHash("nonexistent"))
|
|
if !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("Load() error = %v, want ErrNotFound", err)
|
|
}
|
|
}
|
|
|
|
func TestContentStorage_Delete(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewContentStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewContentStorage() error = %v", err)
|
|
}
|
|
|
|
content := []byte("to be deleted")
|
|
|
|
hash, _, err := storage.Store(bytes.NewReader(content))
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v", err)
|
|
}
|
|
|
|
if !storage.Exists(hash) {
|
|
t.Error("Exists() = false, want true")
|
|
}
|
|
|
|
err = storage.Delete(hash)
|
|
if err != nil {
|
|
t.Fatalf("Delete() error = %v", err)
|
|
}
|
|
|
|
if storage.Exists(hash) {
|
|
t.Error("Exists() = true after delete, want false")
|
|
}
|
|
}
|
|
|
|
func TestContentStorage_DeleteNonexistent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewContentStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewContentStorage() error = %v", err)
|
|
}
|
|
|
|
// Should not error
|
|
err = storage.Delete(ContentHash("nonexistent"))
|
|
if err != nil {
|
|
t.Errorf("Delete() error = %v, want nil", err)
|
|
}
|
|
}
|
|
|
|
func TestContentStorage_HashToPath(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewContentStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewContentStorage() error = %v", err)
|
|
}
|
|
|
|
// Test by storing and verifying the resulting path structure
|
|
content := []byte("test content for path verification")
|
|
|
|
hash, _, err := storage.Store(bytes.NewReader(content))
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v", err)
|
|
}
|
|
|
|
hashStr := string(hash)
|
|
|
|
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
|
|
|
|
_, err = os.Stat(expectedPath)
|
|
if err != nil {
|
|
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
|
}
|
|
}
|
|
|
|
func TestMetadataStorage_StoreAndLoad(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewMetadataStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewMetadataStorage() error = %v", err)
|
|
}
|
|
|
|
meta := &SourceMetadata{
|
|
Host: testHostCDN,
|
|
Path: testPathCat,
|
|
ContentHash: "abc123",
|
|
StatusCode: 200,
|
|
ContentType: testContentTypeJPEG,
|
|
FetchedAt: 1704067200,
|
|
ETag: `"etag123"`,
|
|
}
|
|
|
|
pathHash := HashPath(testPathCat)
|
|
|
|
err = storage.Store(testHostCDN, pathHash, meta)
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v", err)
|
|
}
|
|
|
|
// Verify file exists at expected path
|
|
expectedPath := filepath.Join(tmpDir, testHostCDN, string(pathHash)+".json")
|
|
|
|
_, err = os.Stat(expectedPath)
|
|
if err != nil {
|
|
t.Errorf("File not at expected path %s: %v", expectedPath, err)
|
|
}
|
|
|
|
// Load and verify
|
|
loaded, err := storage.Load(testHostCDN, pathHash)
|
|
if err != nil {
|
|
t.Fatalf("Load() error = %v", err)
|
|
}
|
|
|
|
if loaded.Host != meta.Host {
|
|
t.Errorf("Host = %q, want %q", loaded.Host, meta.Host)
|
|
}
|
|
|
|
if loaded.Path != meta.Path {
|
|
t.Errorf("Path = %q, want %q", loaded.Path, meta.Path)
|
|
}
|
|
|
|
if loaded.ContentHash != meta.ContentHash {
|
|
t.Errorf("ContentHash = %q, want %q", loaded.ContentHash, meta.ContentHash)
|
|
}
|
|
|
|
if loaded.StatusCode != meta.StatusCode {
|
|
t.Errorf("StatusCode = %d, want %d", loaded.StatusCode, meta.StatusCode)
|
|
}
|
|
|
|
if loaded.ETag != meta.ETag {
|
|
t.Errorf("ETag = %q, want %q", loaded.ETag, meta.ETag)
|
|
}
|
|
}
|
|
|
|
func TestMetadataStorage_LoadNotFound(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewMetadataStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewMetadataStorage() error = %v", err)
|
|
}
|
|
|
|
_, err = storage.Load(testHostExample, PathHash("nonexistent"))
|
|
if !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("Load() error = %v, want ErrNotFound", err)
|
|
}
|
|
}
|
|
|
|
func TestMetadataStorage_Delete(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
storage, err := NewMetadataStorage(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("NewMetadataStorage() error = %v", err)
|
|
}
|
|
|
|
meta := &SourceMetadata{
|
|
Host: testHostExample,
|
|
Path: "/test.jpg",
|
|
StatusCode: 200,
|
|
}
|
|
|
|
pathHash := HashPath("/test.jpg")
|
|
|
|
err = storage.Store(testHostExample, pathHash, meta)
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v", err)
|
|
}
|
|
|
|
if !storage.Exists(testHostExample, pathHash) {
|
|
t.Error("Exists() = false, want true")
|
|
}
|
|
|
|
err = storage.Delete(testHostExample, pathHash)
|
|
if err != nil {
|
|
t.Fatalf("Delete() error = %v", err)
|
|
}
|
|
|
|
if storage.Exists(testHostExample, pathHash) {
|
|
t.Error("Exists() = true after delete, want false")
|
|
}
|
|
}
|
|
|
|
func TestHashPath(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Same input should produce same hash
|
|
hash1 := HashPath(testPathCat)
|
|
hash2 := HashPath(testPathCat)
|
|
|
|
if hash1 != hash2 {
|
|
t.Errorf("HashPath() not deterministic: %s vs %s", hash1, hash2)
|
|
}
|
|
|
|
// Different input should produce different hash
|
|
hash3 := HashPath("/photos/dog.jpg")
|
|
|
|
if hash1 == hash3 {
|
|
t.Error("HashPath() produced same hash for different inputs")
|
|
}
|
|
|
|
// Hash should be 64 hex chars (256 bits)
|
|
if len(string(hash1)) != 64 {
|
|
t.Errorf("HashPath() length = %d, want 64", len(string(hash1)))
|
|
}
|
|
}
|
|
|
|
func TestCacheKey(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
req1 := &ImageRequest{
|
|
SourceHost: testHostCDN,
|
|
SourcePath: testPathCat,
|
|
SourceQuery: "",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
req2 := &ImageRequest{
|
|
SourceHost: testHostCDN,
|
|
SourcePath: testPathCat,
|
|
SourceQuery: "",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
// Same request should produce same key
|
|
key1 := CacheKey(req1)
|
|
key2 := CacheKey(req2)
|
|
|
|
if key1 != key2 {
|
|
t.Errorf("CacheKey() not deterministic: %s vs %s", key1, key2)
|
|
}
|
|
|
|
// Key should be 64 hex chars
|
|
if len(string(key1)) != 64 {
|
|
t.Errorf("CacheKey() length = %d, want 64", len(string(key1)))
|
|
}
|
|
|
|
// Different size should produce different key
|
|
req3 := &ImageRequest{
|
|
SourceHost: testHostCDN,
|
|
SourcePath: testPathCat,
|
|
SourceQuery: "",
|
|
Size: Size{Width: 400, Height: 300}, // Different size
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
key3 := CacheKey(req3)
|
|
if key1 == key3 {
|
|
t.Error("CacheKey() produced same key for different sizes")
|
|
}
|
|
|
|
// Different format should produce different key
|
|
req4 := &ImageRequest{
|
|
SourceHost: testHostCDN,
|
|
SourcePath: testPathCat,
|
|
SourceQuery: "",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatPNG, // Different format
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
key4 := CacheKey(req4)
|
|
if key1 == key4 {
|
|
t.Error("CacheKey() produced same key for different formats")
|
|
}
|
|
|
|
// Different quality should produce different key
|
|
req5 := &ImageRequest{
|
|
SourceHost: testHostCDN,
|
|
SourcePath: testPathCat,
|
|
SourceQuery: "",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 50, // Different quality
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
key5 := CacheKey(req5)
|
|
if key1 == key5 {
|
|
t.Error("CacheKey() produced same key for different quality")
|
|
}
|
|
}
|
|
|
|
// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write
|
|
// that fails is logged at warn and the store still succeeds.
|
|
func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var logBuf bytes.Buffer
|
|
|
|
storage, err := NewVariantStorage(
|
|
t.TempDir(), slog.New(slog.NewJSONHandler(&logBuf, nil)))
|
|
if err != nil {
|
|
t.Fatalf("NewVariantStorage() error = %v", err)
|
|
}
|
|
|
|
key := CacheKey(&ImageRequest{SourceHost: testHostCDN, SourcePath: testPathCat})
|
|
|
|
// A directory where the .meta file goes makes the .meta write fail.
|
|
err = os.MkdirAll(storage.keyToPath(key)+".meta", StorageDirPerm)
|
|
if err != nil {
|
|
t.Fatalf("failed to create directory: %v", err)
|
|
}
|
|
|
|
_, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp")
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v, want nil", err)
|
|
}
|
|
|
|
want := `"level":"WARN","msg":"failed to write variant metadata sidecar"`
|
|
if !strings.Contains(logBuf.String(), want) {
|
|
t.Errorf("log missing %s; got %q", want, logBuf.String())
|
|
}
|
|
}
|
|
|
|
// storeTestVariant stores one variant, with its .meta file, in a new
|
|
// VariantStorage and returns the storage and the variant's key.
|
|
func storeTestVariant(t *testing.T) (*VariantStorage, VariantKey) {
|
|
t.Helper()
|
|
|
|
storage, err := NewVariantStorage(t.TempDir(), slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatalf("NewVariantStorage() error = %v", err)
|
|
}
|
|
|
|
key := CacheKey(&ImageRequest{SourceHost: testHostCDN, SourcePath: testPathCat})
|
|
|
|
_, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp")
|
|
if err != nil {
|
|
t.Fatalf("Store() error = %v", err)
|
|
}
|
|
|
|
return storage, key
|
|
}
|
|
|
|
// TestVariantStorage_DeleteRemovesMeta verifies that Delete removes the
|
|
// variant's .meta file along with the variant file.
|
|
func TestVariantStorage_DeleteRemovesMeta(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
storage, key := storeTestVariant(t)
|
|
metaPath := storage.keyToPath(key) + ".meta"
|
|
|
|
_, err := os.Stat(metaPath)
|
|
if err != nil {
|
|
t.Fatalf("Store() wrote no .meta file: %v", err)
|
|
}
|
|
|
|
err = storage.Delete(key)
|
|
if err != nil {
|
|
t.Fatalf("Delete() error = %v", err)
|
|
}
|
|
|
|
if storage.Exists(key) {
|
|
t.Error("Exists() = true after delete, want false")
|
|
}
|
|
|
|
_, err = os.Stat(metaPath)
|
|
if !os.IsNotExist(err) {
|
|
t.Errorf(".meta file left after Delete() (stat err=%v)", err)
|
|
}
|
|
}
|
|
|
|
// TestVariantStorage_DeleteWithoutMeta verifies that Delete succeeds for a
|
|
// variant whose .meta file is missing.
|
|
func TestVariantStorage_DeleteWithoutMeta(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
storage, key := storeTestVariant(t)
|
|
|
|
err := os.Remove(storage.keyToPath(key) + ".meta")
|
|
if err != nil {
|
|
t.Fatalf("removing .meta file: %v", err)
|
|
}
|
|
|
|
err = storage.Delete(key)
|
|
if err != nil {
|
|
t.Fatalf("Delete() error = %v, want nil", err)
|
|
}
|
|
|
|
if storage.Exists(key) {
|
|
t.Error("Exists() = true after delete, want false")
|
|
}
|
|
}
|