All checks were successful
check / check (push) Successful in 57s
Move HTTPFetcher, Config (was FetcherConfig), SSRF-safe dialer, rate limiting, content-type validation, and related error vars from internal/imgcache/fetcher.go into new internal/httpfetcher/ package. The Fetcher interface and FetchResult type also move to httpfetcher to avoid circular imports (imgcache imports httpfetcher, not the other way around). Renames to avoid stuttering: NewHTTPFetcher -> httpfetcher.New FetcherConfig -> httpfetcher.Config NewMockFetcher -> httpfetcher.NewMock The ServiceConfig.FetcherConfig field is retained (it describes what kind of config it holds, not a stutter). Pure refactor - no behavior changes. Unit tests for the httpfetcher package are included. refs #39
491 lines
12 KiB
Go
491 lines
12 KiB
Go
package imgcache
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
"sneak.berlin/go/pixa/internal/httpfetcher"
|
|
)
|
|
|
|
func setupTestDB(t *testing.T) *sql.DB {
|
|
t.Helper()
|
|
|
|
db, err := sql.Open("sqlite", ":memory:")
|
|
if err != nil {
|
|
t.Fatalf("failed to open test db: %v", err)
|
|
}
|
|
|
|
// Create schema
|
|
schema := `
|
|
CREATE TABLE source_content (
|
|
content_hash TEXT PRIMARY KEY,
|
|
content_type TEXT NOT NULL,
|
|
size_bytes INTEGER NOT NULL,
|
|
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
CREATE TABLE source_metadata (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
source_host TEXT NOT NULL,
|
|
source_path TEXT NOT NULL,
|
|
source_query TEXT NOT NULL DEFAULT '',
|
|
path_hash TEXT NOT NULL,
|
|
content_hash TEXT,
|
|
status_code INTEGER NOT NULL,
|
|
content_type TEXT,
|
|
response_headers TEXT,
|
|
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
expires_at DATETIME,
|
|
etag TEXT,
|
|
last_modified TEXT,
|
|
UNIQUE(source_host, source_path, source_query)
|
|
);
|
|
CREATE TABLE output_content (
|
|
content_hash TEXT PRIMARY KEY,
|
|
content_type TEXT NOT NULL,
|
|
size_bytes INTEGER NOT NULL,
|
|
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
CREATE TABLE request_cache (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
cache_key TEXT NOT NULL UNIQUE,
|
|
source_metadata_id INTEGER NOT NULL,
|
|
output_hash TEXT NOT NULL,
|
|
width INTEGER NOT NULL,
|
|
height INTEGER NOT NULL,
|
|
format TEXT NOT NULL,
|
|
quality INTEGER NOT NULL DEFAULT 85,
|
|
fit_mode TEXT NOT NULL DEFAULT 'cover',
|
|
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
access_count INTEGER NOT NULL DEFAULT 1
|
|
);
|
|
CREATE TABLE negative_cache (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
source_host TEXT NOT NULL,
|
|
source_path TEXT NOT NULL,
|
|
source_query TEXT NOT NULL DEFAULT '',
|
|
status_code INTEGER NOT NULL,
|
|
error_message TEXT,
|
|
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
expires_at DATETIME NOT NULL,
|
|
UNIQUE(source_host, source_path, source_query)
|
|
);
|
|
CREATE TABLE cache_stats (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
miss_count INTEGER NOT NULL DEFAULT 0,
|
|
upstream_fetch_count INTEGER NOT NULL DEFAULT 0,
|
|
upstream_fetch_bytes INTEGER NOT NULL DEFAULT 0,
|
|
transform_count INTEGER NOT NULL DEFAULT 0,
|
|
last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
INSERT INTO cache_stats (id) VALUES (1);
|
|
`
|
|
|
|
if _, err := db.Exec(schema); err != nil {
|
|
t.Fatalf("failed to create schema: %v", err)
|
|
}
|
|
|
|
return db
|
|
}
|
|
|
|
func setupTestCache(t *testing.T) (*Cache, string) {
|
|
t.Helper()
|
|
|
|
tmpDir := t.TempDir()
|
|
db := setupTestDB(t)
|
|
|
|
cache, err := NewCache(db, CacheConfig{
|
|
StateDir: tmpDir,
|
|
CacheTTL: time.Hour,
|
|
NegativeTTL: 5 * time.Minute,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create cache: %v", err)
|
|
}
|
|
|
|
return cache, tmpDir
|
|
}
|
|
|
|
func TestCache_LookupMiss(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/cat.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
result, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() error = %v", err)
|
|
}
|
|
|
|
if result.Hit {
|
|
t.Error("Lookup() hit = true, want false for cache miss")
|
|
}
|
|
|
|
if result.CacheStatus != CacheMiss {
|
|
t.Errorf("Lookup() status = %v, want %v", result.CacheStatus, CacheMiss)
|
|
}
|
|
}
|
|
|
|
func TestCache_StoreAndLookup(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/cat.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
// Store source content
|
|
sourceContent := []byte("fake jpeg data")
|
|
fetchResult := &httpfetcher.FetchResult{
|
|
ContentType: "image/jpeg",
|
|
Headers: map[string][]string{"Content-Type": {"image/jpeg"}},
|
|
}
|
|
|
|
contentHash, err := cache.StoreSource(ctx, req, bytes.NewReader(sourceContent), fetchResult)
|
|
if err != nil {
|
|
t.Fatalf("StoreSource() error = %v", err)
|
|
}
|
|
|
|
if contentHash == "" {
|
|
t.Error("StoreSource() returned empty hash")
|
|
}
|
|
|
|
// Store variant
|
|
cacheKey := CacheKey(req)
|
|
outputContent := []byte("fake webp data")
|
|
err = cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
|
|
if err != nil {
|
|
t.Fatalf("StoreVariant() error = %v", err)
|
|
}
|
|
|
|
// Now lookup should hit
|
|
result, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() error = %v", err)
|
|
}
|
|
|
|
if !result.Hit {
|
|
t.Error("Lookup() hit = false, want true after store")
|
|
}
|
|
|
|
if result.CacheStatus != CacheHit {
|
|
t.Errorf("Lookup() status = %v, want %v", result.CacheStatus, CacheHit)
|
|
}
|
|
|
|
if result.CacheKey == "" {
|
|
t.Error("Lookup() returned empty cache key")
|
|
}
|
|
}
|
|
|
|
func TestCache_NegativeCache(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/notfound.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
}
|
|
|
|
// Store negative cache entry
|
|
err := cache.StoreNegative(ctx, req, 404, "not found")
|
|
if err != nil {
|
|
t.Fatalf("StoreNegative() error = %v", err)
|
|
}
|
|
|
|
// Lookup should return miss (negative cache is checked at service level)
|
|
result, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() error = %v", err)
|
|
}
|
|
|
|
if result.Hit {
|
|
t.Error("Lookup() hit = true, want false for negative cache entry")
|
|
}
|
|
}
|
|
|
|
func TestCache_NegativeCacheExpiry(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
db := setupTestDB(t)
|
|
|
|
// Very short negative TTL for testing
|
|
cache, err := NewCache(db, CacheConfig{
|
|
StateDir: tmpDir,
|
|
CacheTTL: time.Hour,
|
|
NegativeTTL: 1 * time.Millisecond,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create cache: %v", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/expired.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
}
|
|
|
|
// Store negative cache entry
|
|
err = cache.StoreNegative(ctx, req, 404, "not found")
|
|
if err != nil {
|
|
t.Fatalf("StoreNegative() error = %v", err)
|
|
}
|
|
|
|
// Wait for expiry
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
// Lookup should return miss after expiry
|
|
result, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() error = %v, want nil after expiry", err)
|
|
}
|
|
|
|
if result.Hit {
|
|
t.Error("Lookup() hit = true, want false after negative cache expiry")
|
|
}
|
|
}
|
|
|
|
func TestCache_VariantLookup(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/variant.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
// Store variant
|
|
cacheKey := CacheKey(req)
|
|
outputContent := []byte("output data")
|
|
err := cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
|
|
if err != nil {
|
|
t.Fatalf("StoreVariant() error = %v", err)
|
|
}
|
|
|
|
// First lookup
|
|
result1, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() first error = %v", err)
|
|
}
|
|
|
|
if !result1.Hit {
|
|
t.Error("Lookup() first hit = false")
|
|
}
|
|
|
|
// Second lookup should also hit (from disk)
|
|
result2, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() second error = %v", err)
|
|
}
|
|
|
|
if !result2.Hit {
|
|
t.Error("Lookup() second hit = false")
|
|
}
|
|
|
|
if result1.CacheKey != result2.CacheKey {
|
|
t.Error("Lookup() returned different cache keys")
|
|
}
|
|
}
|
|
|
|
func TestCache_GetVariant_ReturnsContentType(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/variantct.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
// Store variant
|
|
cacheKey := CacheKey(req)
|
|
outputContent := []byte("output webp data")
|
|
err := cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
|
|
if err != nil {
|
|
t.Fatalf("StoreVariant() error = %v", err)
|
|
}
|
|
|
|
// Lookup
|
|
result, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() error = %v", err)
|
|
}
|
|
|
|
if !result.Hit {
|
|
t.Fatal("Lookup() hit = false, want true")
|
|
}
|
|
|
|
// GetVariant should return content type
|
|
reader, size, contentType, err := cache.GetVariant(result.CacheKey)
|
|
if err != nil {
|
|
t.Fatalf("GetVariant() error = %v", err)
|
|
}
|
|
defer reader.Close()
|
|
|
|
if contentType != "image/webp" {
|
|
t.Errorf("GetVariant() ContentType = %q, want %q", contentType, "image/webp")
|
|
}
|
|
|
|
if size != int64(len(outputContent)) {
|
|
t.Errorf("GetVariant() size = %d, want %d", size, len(outputContent))
|
|
}
|
|
}
|
|
|
|
func TestCache_GetVariant(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
req := &ImageRequest{
|
|
SourceHost: "cdn.example.com",
|
|
SourcePath: "/photos/output.jpg",
|
|
Size: Size{Width: 800, Height: 600},
|
|
Format: FormatWebP,
|
|
Quality: 85,
|
|
FitMode: FitCover,
|
|
}
|
|
|
|
// Store variant
|
|
cacheKey := CacheKey(req)
|
|
outputContent := []byte("the actual output content")
|
|
err := cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
|
|
if err != nil {
|
|
t.Fatalf("StoreVariant() error = %v", err)
|
|
}
|
|
|
|
// Lookup to get cache key
|
|
result, err := cache.Lookup(ctx, req)
|
|
if err != nil {
|
|
t.Fatalf("Lookup() error = %v", err)
|
|
}
|
|
|
|
// Get variant content
|
|
reader, _, _, err := cache.GetVariant(result.CacheKey)
|
|
if err != nil {
|
|
t.Fatalf("GetVariant() error = %v", err)
|
|
}
|
|
defer reader.Close()
|
|
|
|
buf := make([]byte, 100)
|
|
n, _ := reader.Read(buf)
|
|
|
|
if !bytes.Equal(buf[:n], outputContent) {
|
|
t.Errorf("GetVariant() content = %q, want %q", buf[:n], outputContent)
|
|
}
|
|
}
|
|
|
|
func TestCache_Stats(t *testing.T) {
|
|
cache, _ := setupTestCache(t)
|
|
ctx := context.Background()
|
|
|
|
// Increment some stats
|
|
cache.IncrementStats(ctx, true, 0)
|
|
cache.IncrementStats(ctx, true, 0)
|
|
cache.IncrementStats(ctx, false, 1024)
|
|
|
|
stats, err := cache.Stats(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Stats() error = %v", err)
|
|
}
|
|
|
|
if stats.HitCount != 2 {
|
|
t.Errorf("Stats() HitCount = %d, want 2", stats.HitCount)
|
|
}
|
|
|
|
if stats.MissCount != 1 {
|
|
t.Errorf("Stats() MissCount = %d, want 1", stats.MissCount)
|
|
}
|
|
}
|
|
|
|
func TestCache_CleanExpired(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
db := setupTestDB(t)
|
|
|
|
cache, _ := NewCache(db, CacheConfig{
|
|
StateDir: tmpDir,
|
|
NegativeTTL: 1 * time.Millisecond,
|
|
})
|
|
|
|
ctx := context.Background()
|
|
|
|
// Insert expired negative cache entry directly
|
|
_, err := db.ExecContext(ctx, `
|
|
INSERT INTO negative_cache (source_host, source_path, source_query, status_code, expires_at)
|
|
VALUES ('example.com', '/old.jpg', '', 404, datetime('now', '-1 hour'))
|
|
`)
|
|
if err != nil {
|
|
t.Fatalf("failed to insert test data: %v", err)
|
|
}
|
|
|
|
// Verify it exists
|
|
var count int
|
|
db.QueryRowContext(ctx, `SELECT COUNT(*) FROM negative_cache`).Scan(&count)
|
|
if count != 1 {
|
|
t.Fatalf("expected 1 negative cache entry, got %d", count)
|
|
}
|
|
|
|
// Clean expired
|
|
err = cache.CleanExpired(ctx)
|
|
if err != nil {
|
|
t.Fatalf("CleanExpired() error = %v", err)
|
|
}
|
|
|
|
// Verify it's gone
|
|
db.QueryRowContext(ctx, `SELECT COUNT(*) FROM negative_cache`).Scan(&count)
|
|
if count != 0 {
|
|
t.Errorf("expected 0 negative cache entries after clean, got %d", count)
|
|
}
|
|
}
|
|
|
|
func TestCache_StorageDirectoriesCreated(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
db := setupTestDB(t)
|
|
|
|
_, err := NewCache(db, CacheConfig{
|
|
StateDir: tmpDir,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewCache() error = %v", err)
|
|
}
|
|
|
|
// Verify directories were created
|
|
dirs := []string{
|
|
"cache/sources",
|
|
"cache/variants",
|
|
"cache/metadata",
|
|
}
|
|
|
|
for _, dir := range dirs {
|
|
path := tmpDir + "/" + dir
|
|
if _, err := os.Stat(path); os.IsNotExist(err) {
|
|
t.Errorf("directory %s was not created", dir)
|
|
}
|
|
}
|
|
}
|