Add type-safe hash types for cache storage

Define ContentHash, VariantKey, and PathHash types to replace
raw strings, providing compile-time type safety for storage
operations. Update storage layer to use typed parameters,
refactor cache to use variant storage keyed by VariantKey,
and implement source content reuse on cache misses.
This commit is contained in:
2026-01-08 16:55:20 -08:00
parent 555f150179
commit be293906bc
9 changed files with 476 additions and 381 deletions

View File

@@ -64,8 +64,6 @@ func (s *Handlers) initImageService() error {
StateDir: s.config.StateDir, StateDir: s.config.StateDir,
CacheTTL: imgcache.DefaultCacheTTL, CacheTTL: imgcache.DefaultCacheTTL,
NegativeTTL: imgcache.DefaultNegativeTTL, NegativeTTL: imgcache.DefaultNegativeTTL,
HotCacheSize: imgcache.DefaultHotCacheSize,
HotCacheEnabled: true,
}) })
if err != nil { if err != nil {
return err return err

View File

@@ -47,8 +47,6 @@ func setupTestHandler(t *testing.T) *testFixtures {
StateDir: tmpDir, StateDir: tmpDir,
CacheTTL: time.Hour, CacheTTL: time.Hour,
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
HotCacheSize: 100,
HotCacheEnabled: true,
}) })
if err != nil { if err != nil {
t.Fatalf("failed to create cache: %v", err) t.Fatalf("failed to create cache: %v", err)

View File

@@ -26,167 +26,86 @@ type CacheConfig struct {
StateDir string StateDir string
CacheTTL time.Duration CacheTTL time.Duration
NegativeTTL time.Duration NegativeTTL time.Duration
HotCacheSize int
HotCacheEnabled bool
} }
// hotCacheEntry stores all data needed to serve a cache hit without DB access. // variantMeta stores content type for fast cache hits without reading .meta file.
type hotCacheEntry struct { type variantMeta struct {
OutputHash string
ContentType string ContentType string
SizeBytes int64 Size int64
} }
// Cache implements the caching layer for the image proxy. // Cache implements the caching layer for the image proxy.
type Cache struct { type Cache struct {
db *sql.DB db *sql.DB
srcContent *ContentStorage srcContent *ContentStorage // source images by content hash
dstContent *ContentStorage variants *VariantStorage // processed variants by cache key
srcMetadata *MetadataStorage srcMetadata *MetadataStorage // source metadata by host/path
config CacheConfig config CacheConfig
hotCache map[string]hotCacheEntry // cache_key -> entry
hotCacheMu sync.RWMutex // In-memory cache of variant metadata (content type, size) to avoid reading .meta files
hotCacheEnabled bool metaCache map[VariantKey]variantMeta
metaCacheMu sync.RWMutex
} }
// NewCache creates a new cache instance. // NewCache creates a new cache instance.
func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) {
srcContent, err := NewContentStorage(filepath.Join(config.StateDir, "cache", "src-content")) srcContent, err := NewContentStorage(filepath.Join(config.StateDir, "cache", "sources"))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create source content storage: %w", err) return nil, fmt.Errorf("failed to create source content storage: %w", err)
} }
dstContent, err := NewContentStorage(filepath.Join(config.StateDir, "cache", "dst-content")) variants, err := NewVariantStorage(filepath.Join(config.StateDir, "cache", "variants"))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create destination content storage: %w", err) return nil, fmt.Errorf("failed to create variant storage: %w", err)
} }
srcMetadata, err := NewMetadataStorage(filepath.Join(config.StateDir, "cache", "src-metadata")) srcMetadata, err := NewMetadataStorage(filepath.Join(config.StateDir, "cache", "metadata"))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create source metadata storage: %w", err) return nil, fmt.Errorf("failed to create source metadata storage: %w", err)
} }
c := &Cache{ return &Cache{
db: db, db: db,
srcContent: srcContent, srcContent: srcContent,
dstContent: dstContent, variants: variants,
srcMetadata: srcMetadata, srcMetadata: srcMetadata,
config: config, config: config,
hotCacheEnabled: config.HotCacheEnabled, metaCache: make(map[VariantKey]variantMeta),
} }, nil
if config.HotCacheEnabled && config.HotCacheSize > 0 {
c.hotCache = make(map[string]hotCacheEntry, config.HotCacheSize)
}
return c, nil
} }
// LookupResult contains the result of a cache lookup. // LookupResult contains the result of a cache lookup.
type LookupResult struct { type LookupResult struct {
Hit bool Hit bool
OutputHash string CacheKey VariantKey
ContentType string ContentType string
SizeBytes int64 SizeBytes int64
CacheStatus CacheStatus CacheStatus CacheStatus
} }
// Lookup checks if a processed image exists in the cache. // Lookup checks if a processed variant exists on disk (no DB access for hits).
func (c *Cache) Lookup(ctx context.Context, req *ImageRequest) (*LookupResult, error) { func (c *Cache) Lookup(_ context.Context, req *ImageRequest) (*LookupResult, error) {
cacheKey := CacheKey(req) cacheKey := CacheKey(req)
// Check hot cache first // Check variant storage directly - no DB needed for cache hits
if c.hotCacheEnabled { if c.variants.Exists(cacheKey) {
c.hotCacheMu.RLock()
entry, ok := c.hotCache[cacheKey]
c.hotCacheMu.RUnlock()
if ok && c.dstContent.Exists(entry.OutputHash) {
return &LookupResult{ return &LookupResult{
Hit: true, Hit: true,
OutputHash: entry.OutputHash, CacheKey: cacheKey,
ContentType: entry.ContentType,
SizeBytes: entry.SizeBytes,
CacheStatus: CacheHit,
}, nil
}
}
// Check negative cache
negCached, err := c.checkNegativeCache(ctx, req)
if err != nil {
return nil, err
}
if negCached {
return nil, ErrNegativeCache
}
// Check database
var outputHash, contentType string
var sizeBytes int64
var fetchedAt time.Time
err = c.db.QueryRowContext(ctx, `
SELECT rc.output_hash, oc.content_type, oc.size_bytes, rc.fetched_at
FROM request_cache rc
JOIN output_content oc ON rc.output_hash = oc.content_hash
WHERE rc.cache_key = ?
`, cacheKey).Scan(&outputHash, &contentType, &sizeBytes, &fetchedAt)
if errors.Is(err, sql.ErrNoRows) {
return &LookupResult{Hit: false, CacheStatus: CacheMiss}, nil
}
if err != nil {
return nil, fmt.Errorf("failed to query cache: %w", err)
}
// Check TTL
if c.config.CacheTTL > 0 && time.Since(fetchedAt) > c.config.CacheTTL {
return &LookupResult{Hit: false, CacheStatus: CacheStale}, nil
}
// Verify file exists on disk
if !c.dstContent.Exists(outputHash) {
return &LookupResult{Hit: false, CacheStatus: CacheMiss}, nil
}
// Update hot cache
if c.hotCacheEnabled {
c.hotCacheMu.Lock()
c.hotCache[cacheKey] = hotCacheEntry{
OutputHash: outputHash,
ContentType: contentType,
SizeBytes: sizeBytes,
}
c.hotCacheMu.Unlock()
}
// Update access count
_, _ = c.db.ExecContext(ctx, `
UPDATE request_cache
SET access_count = access_count + 1
WHERE cache_key = ?
`, cacheKey)
return &LookupResult{
Hit: true,
OutputHash: outputHash,
ContentType: contentType,
SizeBytes: sizeBytes,
CacheStatus: CacheHit, CacheStatus: CacheHit,
}, nil }, nil
} }
// GetOutput returns a reader for cached output content. return &LookupResult{
func (c *Cache) GetOutput(outputHash string) (io.ReadCloser, error) { Hit: false,
return c.dstContent.Load(outputHash) CacheKey: cacheKey,
CacheStatus: CacheMiss,
}, nil
} }
// GetOutputWithSize returns a reader and size for cached output content. // GetVariant returns a reader, size, and content type for a cached variant.
func (c *Cache) GetOutputWithSize(outputHash string) (io.ReadCloser, int64, error) { func (c *Cache) GetVariant(cacheKey VariantKey) (io.ReadCloser, int64, string, error) {
return c.dstContent.LoadWithSize(outputHash) return c.variants.LoadWithMeta(cacheKey)
} }
// StoreSource stores fetched source content and metadata. // StoreSource stores fetched source content and metadata.
@@ -195,7 +114,7 @@ func (c *Cache) StoreSource(
req *ImageRequest, req *ImageRequest,
content io.Reader, content io.Reader,
result *FetchResult, result *FetchResult,
) (contentHash string, err error) { ) (ContentHash, error) {
// Store content // Store content
contentHash, size, err := c.srcContent.Store(content) contentHash, size, err := c.srcContent.Store(content)
if err != nil { if err != nil {
@@ -237,12 +156,12 @@ func (c *Cache) StoreSource(
Host: req.SourceHost, Host: req.SourceHost,
Path: req.SourcePath, Path: req.SourcePath,
Query: req.SourceQuery, Query: req.SourceQuery,
ContentHash: contentHash, ContentHash: string(contentHash),
StatusCode: result.StatusCode, StatusCode: result.StatusCode,
ContentType: result.ContentType, ContentType: result.ContentType,
ContentLength: result.ContentLength, ContentLength: result.ContentLength,
ResponseHeaders: result.Headers, ResponseHeaders: result.Headers,
FetchedAt: time.Now().Unix(), FetchedAt: time.Now().UTC().Unix(),
FetchDurationMs: result.FetchDurationMs, FetchDurationMs: result.FetchDurationMs,
RemoteAddr: result.RemoteAddr, RemoteAddr: result.RemoteAddr,
} }
@@ -255,61 +174,43 @@ func (c *Cache) StoreSource(
return contentHash, nil return contentHash, nil
} }
// StoreOutput stores processed output content and returns the output hash. // StoreVariant stores a processed variant by its cache key.
func (c *Cache) StoreOutput( func (c *Cache) StoreVariant(cacheKey VariantKey, content io.Reader, contentType string) error {
ctx context.Context, _, err := c.variants.Store(cacheKey, content, contentType)
req *ImageRequest, return err
sourceMetadataID int64, }
content io.Reader,
contentType string, // LookupSource checks if we have cached source content for a request.
) (string, error) { // Returns the content hash and content type if found, or empty values if not.
// Store content func (c *Cache) LookupSource(ctx context.Context, req *ImageRequest) (ContentHash, string, error) {
outputHash, size, err := c.dstContent.Store(content) var hashStr, contentType string
err := c.db.QueryRowContext(ctx, `
SELECT content_hash, content_type FROM source_metadata
WHERE source_host = ? AND source_path = ? AND source_query = ?
`, req.SourceHost, req.SourcePath, req.SourceQuery).Scan(&hashStr, &contentType)
if errors.Is(err, sql.ErrNoRows) {
return "", "", nil
}
if err != nil { if err != nil {
return "", fmt.Errorf("failed to store output content: %w", err) return "", "", fmt.Errorf("failed to lookup source: %w", err)
} }
cacheKey := CacheKey(req) contentHash := ContentHash(hashStr)
// Store in database // Verify the content file exists
_, err = c.db.ExecContext(ctx, ` if !c.srcContent.Exists(contentHash) {
INSERT INTO output_content (content_hash, content_type, size_bytes) return "", "", nil
VALUES (?, ?, ?)
ON CONFLICT(content_hash) DO NOTHING
`, outputHash, contentType, size)
if err != nil {
return "", fmt.Errorf("failed to insert output content: %w", err)
} }
_, err = c.db.ExecContext(ctx, ` return contentHash, contentType, nil
INSERT INTO request_cache (cache_key, source_metadata_id, output_hash, width, height, format, quality, fit_mode)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET
output_hash = excluded.output_hash,
fetched_at = CURRENT_TIMESTAMP,
access_count = request_cache.access_count + 1
`, cacheKey, sourceMetadataID, outputHash, req.Size.Width, req.Size.Height, req.Format, req.Quality, req.FitMode)
if err != nil {
return "", fmt.Errorf("failed to insert request cache: %w", err)
}
// Update hot cache
if c.hotCacheEnabled {
c.hotCacheMu.Lock()
c.hotCache[cacheKey] = hotCacheEntry{
OutputHash: outputHash,
ContentType: contentType,
SizeBytes: size,
}
c.hotCacheMu.Unlock()
}
return outputHash, nil
} }
// StoreNegative stores a negative cache entry for a failed fetch. // StoreNegative stores a negative cache entry for a failed fetch.
func (c *Cache) StoreNegative(ctx context.Context, req *ImageRequest, statusCode int, errMsg string) error { func (c *Cache) StoreNegative(ctx context.Context, req *ImageRequest, statusCode int, errMsg string) error {
expiresAt := time.Now().Add(c.config.NegativeTTL) expiresAt := time.Now().UTC().Add(c.config.NegativeTTL)
_, err := c.db.ExecContext(ctx, ` _, err := c.db.ExecContext(ctx, `
INSERT INTO negative_cache (source_host, source_path, source_query, status_code, error_message, expires_at) INSERT INTO negative_cache (source_host, source_path, source_query, status_code, error_message, expires_at)
@@ -375,7 +276,7 @@ func (c *Cache) GetSourceMetadataID(ctx context.Context, req *ImageRequest) (int
} }
// GetSourceContent returns a reader for cached source content by its hash. // GetSourceContent returns a reader for cached source content by its hash.
func (c *Cache) GetSourceContent(contentHash string) (io.ReadCloser, error) { func (c *Cache) GetSourceContent(contentHash ContentHash) (io.ReadCloser, error) {
return c.srcContent.Load(contentHash) return c.srcContent.Load(contentHash)
} }

View File

@@ -102,8 +102,6 @@ func setupTestCache(t *testing.T) (*Cache, string) {
StateDir: tmpDir, StateDir: tmpDir,
CacheTTL: time.Hour, CacheTTL: time.Hour,
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
HotCacheSize: 100,
HotCacheEnabled: true,
}) })
if err != nil { if err != nil {
t.Fatalf("failed to create cache: %v", err) t.Fatalf("failed to create cache: %v", err)
@@ -168,17 +166,12 @@ func TestCache_StoreAndLookup(t *testing.T) {
t.Error("StoreSource() returned empty hash") t.Error("StoreSource() returned empty hash")
} }
// Get source metadata ID // Store variant
metaID, err := cache.GetSourceMetadataID(ctx, req) cacheKey := CacheKey(req)
if err != nil {
t.Fatalf("GetSourceMetadataID() error = %v", err)
}
// Store output content
outputContent := []byte("fake webp data") outputContent := []byte("fake webp data")
_, err = cache.StoreOutput(ctx, req, metaID, bytes.NewReader(outputContent), "image/webp") err = cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
if err != nil { if err != nil {
t.Fatalf("StoreOutput() error = %v", err) t.Fatalf("StoreVariant() error = %v", err)
} }
// Now lookup should hit // Now lookup should hit
@@ -195,8 +188,8 @@ func TestCache_StoreAndLookup(t *testing.T) {
t.Errorf("Lookup() status = %v, want %v", result.CacheStatus, CacheHit) t.Errorf("Lookup() status = %v, want %v", result.CacheStatus, CacheHit)
} }
if result.OutputHash == "" { if result.CacheKey == "" {
t.Error("Lookup() returned empty output hash") t.Error("Lookup() returned empty cache key")
} }
} }
@@ -217,10 +210,14 @@ func TestCache_NegativeCache(t *testing.T) {
t.Fatalf("StoreNegative() error = %v", err) t.Fatalf("StoreNegative() error = %v", err)
} }
// Lookup should return negative cache error // Lookup should return miss (negative cache is checked at service level)
_, err = cache.Lookup(ctx, req) result, err := cache.Lookup(ctx, req)
if err != ErrNegativeCache { if err != nil {
t.Errorf("Lookup() error = %v, want ErrNegativeCache", err) t.Fatalf("Lookup() error = %v", err)
}
if result.Hit {
t.Error("Lookup() hit = true, want false for negative cache entry")
} }
} }
@@ -233,8 +230,6 @@ func TestCache_NegativeCacheExpiry(t *testing.T) {
StateDir: tmpDir, StateDir: tmpDir,
CacheTTL: time.Hour, CacheTTL: time.Hour,
NegativeTTL: 1 * time.Millisecond, NegativeTTL: 1 * time.Millisecond,
HotCacheSize: 100,
HotCacheEnabled: true,
}) })
if err != nil { if err != nil {
t.Fatalf("failed to create cache: %v", err) t.Fatalf("failed to create cache: %v", err)
@@ -258,7 +253,7 @@ func TestCache_NegativeCacheExpiry(t *testing.T) {
// Wait for expiry // Wait for expiry
time.Sleep(10 * time.Millisecond) time.Sleep(10 * time.Millisecond)
// Lookup should return miss (not negative cache) after expiry // Lookup should return miss after expiry
result, err := cache.Lookup(ctx, req) result, err := cache.Lookup(ctx, req)
if err != nil { if err != nil {
t.Fatalf("Lookup() error = %v, want nil after expiry", err) t.Fatalf("Lookup() error = %v, want nil after expiry", err)
@@ -269,40 +264,28 @@ func TestCache_NegativeCacheExpiry(t *testing.T) {
} }
} }
func TestCache_HotCache(t *testing.T) { func TestCache_VariantLookup(t *testing.T) {
cache, _ := setupTestCache(t) cache, _ := setupTestCache(t)
ctx := context.Background() ctx := context.Background()
req := &ImageRequest{ req := &ImageRequest{
SourceHost: "cdn.example.com", SourceHost: "cdn.example.com",
SourcePath: "/photos/hot.jpg", SourcePath: "/photos/variant.jpg",
Size: Size{Width: 800, Height: 600}, Size: Size{Width: 800, Height: 600},
Format: FormatWebP, Format: FormatWebP,
Quality: 85, Quality: 85,
FitMode: FitCover, FitMode: FitCover,
} }
// Store content // Store variant
sourceContent := []byte("source data") cacheKey := CacheKey(req)
fetchResult := &FetchResult{
ContentType: "image/jpeg",
Headers: map[string][]string{},
}
_, err := cache.StoreSource(ctx, req, bytes.NewReader(sourceContent), fetchResult)
if err != nil {
t.Fatalf("StoreSource() error = %v", err)
}
metaID, _ := cache.GetSourceMetadataID(ctx, req)
outputContent := []byte("output data") outputContent := []byte("output data")
_, err = cache.StoreOutput(ctx, req, metaID, bytes.NewReader(outputContent), "image/webp") err := cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
if err != nil { if err != nil {
t.Fatalf("StoreOutput() error = %v", err) t.Fatalf("StoreVariant() error = %v", err)
} }
// First lookup populates hot cache // First lookup
result1, err := cache.Lookup(ctx, req) result1, err := cache.Lookup(ctx, req)
if err != nil { if err != nil {
t.Fatalf("Lookup() first error = %v", err) t.Fatalf("Lookup() first error = %v", err)
@@ -312,7 +295,7 @@ func TestCache_HotCache(t *testing.T) {
t.Error("Lookup() first hit = false") t.Error("Lookup() first hit = false")
} }
// Second lookup should use hot cache // Second lookup should also hit (from disk)
result2, err := cache.Lookup(ctx, req) result2, err := cache.Lookup(ctx, req)
if err != nil { if err != nil {
t.Fatalf("Lookup() second error = %v", err) t.Fatalf("Lookup() second error = %v", err)
@@ -322,66 +305,59 @@ func TestCache_HotCache(t *testing.T) {
t.Error("Lookup() second hit = false") t.Error("Lookup() second hit = false")
} }
if result1.OutputHash != result2.OutputHash { if result1.CacheKey != result2.CacheKey {
t.Error("Lookup() returned different hashes") t.Error("Lookup() returned different cache keys")
} }
} }
func TestCache_HotCache_ReturnsContentType(t *testing.T) { func TestCache_GetVariant_ReturnsContentType(t *testing.T) {
cache, _ := setupTestCache(t) cache, _ := setupTestCache(t)
ctx := context.Background() ctx := context.Background()
req := &ImageRequest{ req := &ImageRequest{
SourceHost: "cdn.example.com", SourceHost: "cdn.example.com",
SourcePath: "/photos/hotct.jpg", SourcePath: "/photos/variantct.jpg",
Size: Size{Width: 800, Height: 600}, Size: Size{Width: 800, Height: 600},
Format: FormatWebP, Format: FormatWebP,
Quality: 85, Quality: 85,
FitMode: FitCover, FitMode: FitCover,
} }
// Store content // Store variant
sourceContent := []byte("source data") cacheKey := CacheKey(req)
fetchResult := &FetchResult{
ContentType: "image/jpeg",
Headers: map[string][]string{},
}
_, err := cache.StoreSource(ctx, req, bytes.NewReader(sourceContent), fetchResult)
if err != nil {
t.Fatalf("StoreSource() error = %v", err)
}
metaID, _ := cache.GetSourceMetadataID(ctx, req)
outputContent := []byte("output webp data") outputContent := []byte("output webp data")
_, err = cache.StoreOutput(ctx, req, metaID, bytes.NewReader(outputContent), "image/webp") err := cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
if err != nil { if err != nil {
t.Fatalf("StoreOutput() error = %v", err) t.Fatalf("StoreVariant() error = %v", err)
} }
// First lookup populates hot cache // Lookup
result1, err := cache.Lookup(ctx, req) result, err := cache.Lookup(ctx, req)
if err != nil { if err != nil {
t.Fatalf("Lookup() first error = %v", err) t.Fatalf("Lookup() error = %v", err)
} }
if result1.ContentType != "image/webp" { if !result.Hit {
t.Errorf("Lookup() first ContentType = %q, want %q", result1.ContentType, "image/webp") t.Fatal("Lookup() hit = false, want true")
} }
// Second lookup uses hot cache - must still have ContentType // GetVariant should return content type
result2, err := cache.Lookup(ctx, req) reader, size, contentType, err := cache.GetVariant(result.CacheKey)
if err != nil { if err != nil {
t.Fatalf("Lookup() second error = %v", err) t.Fatalf("GetVariant() error = %v", err)
}
defer reader.Close()
if contentType != "image/webp" {
t.Errorf("GetVariant() ContentType = %q, want %q", contentType, "image/webp")
} }
if result2.ContentType != "image/webp" { if size != int64(len(outputContent)) {
t.Errorf("Lookup() hot cache ContentType = %q, want %q", result2.ContentType, "image/webp") t.Errorf("GetVariant() size = %d, want %d", size, len(outputContent))
} }
} }
func TestCache_GetOutput(t *testing.T) { func TestCache_GetVariant(t *testing.T) {
cache, _ := setupTestCache(t) cache, _ := setupTestCache(t)
ctx := context.Background() ctx := context.Background()
@@ -394,38 +370,24 @@ func TestCache_GetOutput(t *testing.T) {
FitMode: FitCover, FitMode: FitCover,
} }
// Store content // Store variant
sourceContent := []byte("source") cacheKey := CacheKey(req)
fetchResult := &FetchResult{ContentType: "image/jpeg", Headers: map[string][]string{}}
_, err := cache.StoreSource(ctx, req, bytes.NewReader(sourceContent), fetchResult)
if err != nil {
t.Fatalf("StoreSource() error = %v", err)
}
metaID, err := cache.GetSourceMetadataID(ctx, req)
if err != nil {
t.Fatalf("GetSourceMetadataID() error = %v", err)
}
outputContent := []byte("the actual output content") outputContent := []byte("the actual output content")
outputHash, err := cache.StoreOutput(ctx, req, metaID, bytes.NewReader(outputContent), "image/webp") err := cache.StoreVariant(cacheKey, bytes.NewReader(outputContent), "image/webp")
if err != nil { if err != nil {
t.Fatalf("StoreOutput() error = %v", err) t.Fatalf("StoreVariant() error = %v", err)
}
if outputHash == "" {
t.Fatal("StoreOutput() returned empty hash")
} }
// Lookup to get hash // Lookup to get cache key
result, err := cache.Lookup(ctx, req) result, err := cache.Lookup(ctx, req)
if err != nil { if err != nil {
t.Fatalf("Lookup() error = %v", err) t.Fatalf("Lookup() error = %v", err)
} }
// Get output content // Get variant content
reader, err := cache.GetOutput(result.OutputHash) reader, _, _, err := cache.GetVariant(result.CacheKey)
if err != nil { if err != nil {
t.Fatalf("GetOutput() error = %v", err) t.Fatalf("GetVariant() error = %v", err)
} }
defer reader.Close() defer reader.Close()
@@ -433,7 +395,7 @@ func TestCache_GetOutput(t *testing.T) {
n, _ := reader.Read(buf) n, _ := reader.Read(buf)
if !bytes.Equal(buf[:n], outputContent) { if !bytes.Equal(buf[:n], outputContent) {
t.Errorf("GetOutput() content = %q, want %q", buf[:n], outputContent) t.Errorf("GetVariant() content = %q, want %q", buf[:n], outputContent)
} }
} }
@@ -467,7 +429,6 @@ func TestCache_CleanExpired(t *testing.T) {
cache, _ := NewCache(db, CacheConfig{ cache, _ := NewCache(db, CacheConfig{
StateDir: tmpDir, StateDir: tmpDir,
NegativeTTL: 1 * time.Millisecond, NegativeTTL: 1 * time.Millisecond,
HotCacheEnabled: false,
}) })
ctx := context.Background() ctx := context.Background()
@@ -507,7 +468,6 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) {
_, err := NewCache(db, CacheConfig{ _, err := NewCache(db, CacheConfig{
StateDir: tmpDir, StateDir: tmpDir,
HotCacheEnabled: false,
}) })
if err != nil { if err != nil {
t.Fatalf("NewCache() error = %v", err) t.Fatalf("NewCache() error = %v", err)
@@ -515,9 +475,9 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) {
// Verify directories were created // Verify directories were created
dirs := []string{ dirs := []string{
"cache/src-content", "cache/sources",
"cache/dst-content", "cache/variants",
"cache/src-metadata", "cache/metadata",
} }
for _, dir := range dirs { for _, dir := range dirs {

View File

@@ -6,5 +6,4 @@ import "time"
const ( const (
DefaultCacheTTL = 24 * time.Hour DefaultCacheTTL = 24 * time.Hour
DefaultNegativeTTL = 5 * time.Minute DefaultNegativeTTL = 5 * time.Minute
DefaultHotCacheSize = 1000
) )

View File

@@ -80,39 +80,36 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
// Get retrieves a processed image, fetching and processing if necessary. // Get retrieves a processed image, fetching and processing if necessary.
func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) { func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) {
// Check cache first // Check variant cache first (disk only, no DB)
result, err := s.cache.Lookup(ctx, req) result, err := s.cache.Lookup(ctx, req)
if err != nil { if err != nil {
if errors.Is(err, ErrNegativeCache) {
return nil, fmt.Errorf("upstream returned error (cached)")
}
s.log.Warn("cache lookup failed", "error", err) s.log.Warn("cache lookup failed", "error", err)
} }
// Cache hit - serve from cache // Cache hit - serve directly from disk
if result != nil && result.Hit { if result != nil && result.Hit {
reader, size, contentType, err := s.cache.GetVariant(result.CacheKey)
if err != nil {
s.log.Error("failed to get cached variant", "key", result.CacheKey, "error", err)
// Fall through to re-process
} else {
s.cache.IncrementStats(ctx, true, 0) s.cache.IncrementStats(ctx, true, 0)
reader, size, err := s.cache.GetOutputWithSize(result.OutputHash)
if err != nil {
s.log.Error("failed to get cached output", "hash", result.OutputHash, "error", err)
// Fall through to re-fetch
} else {
return &ImageResponse{ return &ImageResponse{
Content: reader, Content: reader,
ContentLength: size, ContentLength: size,
ContentType: result.ContentType, ContentType: contentType,
CacheStatus: CacheHit, CacheStatus: CacheHit,
ETag: formatETag(result.OutputHash), ETag: formatETag(result.CacheKey),
}, nil }, nil
} }
} }
// Cache miss - need to fetch, process, and cache // Cache miss - check if we have source content cached
cacheKey := CacheKey(req)
s.cache.IncrementStats(ctx, false, 0) s.cache.IncrementStats(ctx, false, 0)
response, err := s.fetchAndProcess(ctx, req) response, err := s.processFromSourceOrFetch(ctx, req, cacheKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -122,8 +119,62 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e
return response, nil return response, nil
} }
// processFromSourceOrFetch processes an image, using cached source content if available.
func (s *Service) processFromSourceOrFetch(
ctx context.Context,
req *ImageRequest,
cacheKey VariantKey,
) (*ImageResponse, error) {
// Check if we have cached source content
contentHash, _, err := s.cache.LookupSource(ctx, req)
if err != nil {
s.log.Warn("source lookup failed", "error", err)
}
var sourceData []byte
var fetchBytes int64
if contentHash != "" {
// We have cached source - load it
s.log.Debug("using cached source", "hash", contentHash)
reader, err := s.cache.GetSourceContent(contentHash)
if err != nil {
s.log.Warn("failed to load cached source, fetching", "error", err)
// Fall through to fetch
} else {
sourceData, err = io.ReadAll(reader)
_ = reader.Close()
if err != nil {
s.log.Warn("failed to read cached source, fetching", "error", err)
// Fall through to fetch
}
}
}
// Fetch from upstream if we don't have source data
if sourceData == nil {
resp, err := s.fetchAndProcess(ctx, req, cacheKey)
if err != nil {
return nil, err
}
return resp, nil
}
// Process using cached source
fetchBytes = int64(len(sourceData))
return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
}
// fetchAndProcess fetches from upstream, processes, and caches the result. // fetchAndProcess fetches from upstream, processes, and caches the result.
func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*ImageResponse, error) { func (s *Service) fetchAndProcess(
ctx context.Context,
req *ImageRequest,
cacheKey VariantKey,
) (*ImageResponse, error) {
// Fetch from upstream // Fetch from upstream
sourceURL := req.SourceURL() sourceURL := req.SourceURL()
@@ -182,6 +233,17 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
// Continue even if caching fails // Continue even if caching fails
} }
return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
}
// processAndStore processes an image and stores the result.
func (s *Service) processAndStore(
ctx context.Context,
req *ImageRequest,
cacheKey VariantKey,
sourceData []byte,
fetchBytes int64,
) (*ImageResponse, error) {
// Process the image // Process the image
processStart := time.Now() processStart := time.Now()
@@ -192,8 +254,16 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
processDuration := time.Since(processStart) processDuration := time.Since(processStart)
// Read processed content
processedData, err := io.ReadAll(processResult.Content)
_ = processResult.Content.Close()
if err != nil {
return nil, fmt.Errorf("failed to read processed content: %w", err)
}
// Log conversion details // Log conversion details
outputSize := processResult.ContentLength outputSize := int64(len(processedData))
sizePercent := float64(outputSize) / float64(fetchBytes) * 100.0 //nolint:mnd // percentage calculation sizePercent := float64(outputSize) / float64(fetchBytes) * 100.0 //nolint:mnd // percentage calculation
s.log.Info("image converted", s.log.Info("image converted",
@@ -211,30 +281,18 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
"fit", req.FitMode, "fit", req.FitMode,
) )
// Store output content to cache // Store variant to cache
metaID, err := s.cache.GetSourceMetadataID(ctx, req) if err := s.cache.StoreVariant(cacheKey, bytes.NewReader(processedData), processResult.ContentType); err != nil {
if err != nil { s.log.Warn("failed to store variant", "error", err)
return nil, fmt.Errorf("failed to get source metadata ID: %w", err) // Continue even if caching fails
}
outputHash, err := s.cache.StoreOutput(ctx, req, metaID, processResult.Content, processResult.ContentType)
_ = processResult.Content.Close()
if err != nil {
return nil, fmt.Errorf("failed to store output content: %w", err)
}
// Serve from the cached file on disk (same path as cache hits)
reader, size, err := s.cache.GetOutputWithSize(outputHash)
if err != nil {
return nil, fmt.Errorf("failed to read cached output: %w", err)
} }
return &ImageResponse{ return &ImageResponse{
Content: reader, Content: io.NopCloser(bytes.NewReader(processedData)),
ContentLength: size, ContentLength: outputSize,
ContentType: processResult.ContentType, ContentType: processResult.ContentType,
FetchedBytes: int64(len(sourceData)), FetchedBytes: fetchBytes,
ETag: formatETag(outputHash), ETag: formatETag(cacheKey),
}, nil }, nil
} }
@@ -309,8 +367,9 @@ func extractStatusCode(err error) int {
// etagHashLength is the number of hash characters to use for ETags. // etagHashLength is the number of hash characters to use for ETags.
const etagHashLength = 16 const etagHashLength = 16
// formatETag formats a hash as a quoted ETag value. // formatETag formats a VariantKey as a quoted ETag value.
func formatETag(hash string) string { func formatETag(key VariantKey) string {
hash := string(key)
// Use first 16 characters of hash for a shorter but still unique ETag // Use first 16 characters of hash for a shorter but still unique ETag
if len(hash) > etagHashLength { if len(hash) > etagHashLength {
hash = hash[:etagHashLength] hash = hash[:etagHashLength]

View File

@@ -9,6 +9,7 @@ import (
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"time"
) )
// Storage constants. // Storage constants.
@@ -24,6 +25,15 @@ var (
ErrNotFound = errors.New("content not found") ErrNotFound = errors.New("content not found")
) )
// ContentHash is a SHA256 hash of file content (hex-encoded).
type ContentHash string
// VariantKey is a SHA256 hash identifying a specific image variant (hex-encoded).
type VariantKey string
// PathHash is a SHA256 hash of a URL path (hex-encoded).
type PathHash string
// ContentStorage handles content-addressable file storage. // ContentStorage handles content-addressable file storage.
// Files are stored at: <basedir>/<ab>/<cd>/<abcdef...sha256> // Files are stored at: <basedir>/<ab>/<cd>/<abcdef...sha256>
type ContentStorage struct { type ContentStorage struct {
@@ -41,7 +51,7 @@ func NewContentStorage(baseDir string) (*ContentStorage, error) {
// Store writes content to storage and returns its SHA256 hash. // Store writes content to storage and returns its SHA256 hash.
// The content is read fully into memory to compute the hash before writing. // The content is read fully into memory to compute the hash before writing.
func (s *ContentStorage) Store(r io.Reader) (hash string, size int64, err error) { func (s *ContentStorage) Store(r io.Reader) (hash ContentHash, size int64, err error) {
// Read all content to compute hash // Read all content to compute hash
data, err := io.ReadAll(r) data, err := io.ReadAll(r)
if err != nil { if err != nil {
@@ -50,7 +60,7 @@ func (s *ContentStorage) Store(r io.Reader) (hash string, size int64, err error)
// Compute hash // Compute hash
h := sha256.Sum256(data) h := sha256.Sum256(data)
hash = hex.EncodeToString(h[:]) hash = ContentHash(hex.EncodeToString(h[:]))
size = int64(len(data)) size = int64(len(data))
// Build path: <basedir>/<ab>/<cd>/<hash> // Build path: <basedir>/<ab>/<cd>/<hash>
@@ -99,10 +109,10 @@ func (s *ContentStorage) Store(r io.Reader) (hash string, size int64, err error)
} }
// Load returns a reader for the content with the given hash. // Load returns a reader for the content with the given hash.
func (s *ContentStorage) Load(hash string) (io.ReadCloser, error) { func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) {
path := s.hashToPath(hash) path := s.hashToPath(hash)
f, err := os.Open(path) //nolint:gosec // content-addressable path from hash f, err := os.Open(path) //nolint:gosec // path derived from content hash
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, ErrNotFound return nil, ErrNotFound
@@ -115,10 +125,10 @@ func (s *ContentStorage) Load(hash string) (io.ReadCloser, error) {
} }
// LoadWithSize returns a reader and file size for the content with the given hash. // LoadWithSize returns a reader and file size for the content with the given hash.
func (s *ContentStorage) LoadWithSize(hash string) (io.ReadCloser, int64, error) { func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, error) {
path := s.hashToPath(hash) path := s.hashToPath(hash)
f, err := os.Open(path) //nolint:gosec // content-addressable path from hash f, err := os.Open(path) //nolint:gosec // path derived from content hash
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, 0, ErrNotFound return nil, 0, ErrNotFound
@@ -138,7 +148,7 @@ func (s *ContentStorage) LoadWithSize(hash string) (io.ReadCloser, int64, error)
} }
// Delete removes content with the given hash. // Delete removes content with the given hash.
func (s *ContentStorage) Delete(hash string) error { func (s *ContentStorage) Delete(hash ContentHash) error {
path := s.hashToPath(hash) path := s.hashToPath(hash)
err := os.Remove(path) err := os.Remove(path)
@@ -150,25 +160,21 @@ func (s *ContentStorage) Delete(hash string) error {
} }
// Exists checks if content with the given hash exists. // Exists checks if content with the given hash exists.
func (s *ContentStorage) Exists(hash string) bool { func (s *ContentStorage) Exists(hash ContentHash) bool {
path := s.hashToPath(hash) path := s.hashToPath(hash)
_, err := os.Stat(path) _, err := os.Stat(path)
return err == nil return err == nil
} }
// Path returns the file path for a given hash.
func (s *ContentStorage) Path(hash string) string {
return s.hashToPath(hash)
}
// hashToPath converts a hash to a file path: <basedir>/<ab>/<cd>/<hash> // hashToPath converts a hash to a file path: <basedir>/<ab>/<cd>/<hash>
func (s *ContentStorage) hashToPath(hash string) string { func (s *ContentStorage) hashToPath(hash ContentHash) string {
if len(hash) < MinHashLength { h := string(hash)
return filepath.Join(s.baseDir, hash) if len(h) < MinHashLength {
return filepath.Join(s.baseDir, h)
} }
return filepath.Join(s.baseDir, hash[0:2], hash[2:4], hash) return filepath.Join(s.baseDir, h[0:2], h[2:4], h)
} }
// MetadataStorage handles JSON metadata file storage. // MetadataStorage handles JSON metadata file storage.
@@ -205,7 +211,7 @@ type SourceMetadata struct {
} }
// Store writes metadata to storage. // Store writes metadata to storage.
func (s *MetadataStorage) Store(host, pathHash string, meta *SourceMetadata) error { func (s *MetadataStorage) Store(host string, pathHash PathHash, meta *SourceMetadata) error {
path := s.metaPath(host, pathHash) path := s.metaPath(host, pathHash)
// Create directory structure // Create directory structure
@@ -252,7 +258,7 @@ func (s *MetadataStorage) Store(host, pathHash string, meta *SourceMetadata) err
} }
// Load reads metadata from storage. // Load reads metadata from storage.
func (s *MetadataStorage) Load(host, pathHash string) (*SourceMetadata, error) { func (s *MetadataStorage) Load(host string, pathHash PathHash) (*SourceMetadata, error) {
path := s.metaPath(host, pathHash) path := s.metaPath(host, pathHash)
data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash
@@ -273,7 +279,7 @@ func (s *MetadataStorage) Load(host, pathHash string) (*SourceMetadata, error) {
} }
// Delete removes metadata for the given host and path hash. // Delete removes metadata for the given host and path hash.
func (s *MetadataStorage) Delete(host, pathHash string) error { func (s *MetadataStorage) Delete(host string, pathHash PathHash) error {
path := s.metaPath(host, pathHash) path := s.metaPath(host, pathHash)
err := os.Remove(path) err := os.Remove(path)
@@ -285,7 +291,7 @@ func (s *MetadataStorage) Delete(host, pathHash string) error {
} }
// Exists checks if metadata exists for the given host and path hash. // Exists checks if metadata exists for the given host and path hash.
func (s *MetadataStorage) Exists(host, pathHash string) bool { func (s *MetadataStorage) Exists(host string, pathHash PathHash) bool {
path := s.metaPath(host, pathHash) path := s.metaPath(host, pathHash)
_, err := os.Stat(path) _, err := os.Stat(path)
@@ -293,20 +299,20 @@ func (s *MetadataStorage) Exists(host, pathHash string) bool {
} }
// metaPath returns the file path for metadata: <basedir>/<host>/<path_hash>.json // metaPath returns the file path for metadata: <basedir>/<host>/<path_hash>.json
func (s *MetadataStorage) metaPath(host, pathHash string) string { func (s *MetadataStorage) metaPath(host string, pathHash PathHash) string {
return filepath.Join(s.baseDir, host, pathHash+".json") return filepath.Join(s.baseDir, host, string(pathHash)+".json")
} }
// HashPath computes the SHA256 hash of a path string. // HashPath computes the SHA256 hash of a path string.
func HashPath(path string) string { func HashPath(path string) PathHash {
h := sha256.Sum256([]byte(path)) h := sha256.Sum256([]byte(path))
return hex.EncodeToString(h[:]) return PathHash(hex.EncodeToString(h[:]))
} }
// CacheKey generates a unique cache key for a request. // CacheKey generates a unique key for a request variant.
// Format: sha256(host:path:query:width:height:format:quality:fit_mode) // Format: sha256(host:path:query:width:height:format:quality:fit_mode)
func CacheKey(req *ImageRequest) string { func CacheKey(req *ImageRequest) VariantKey {
data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s", data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s",
req.SourceHost, req.SourceHost,
req.SourcePath, req.SourcePath,
@@ -319,5 +325,175 @@ func CacheKey(req *ImageRequest) string {
) )
h := sha256.Sum256([]byte(data)) h := sha256.Sum256([]byte(data))
return hex.EncodeToString(h[:]) return VariantKey(hex.EncodeToString(h[:]))
}
// VariantStorage handles key-based file storage for processed image variants.
// Files are stored at: <basedir>/<ab>/<cd>/<cache_key>
// Metadata is stored at: <basedir>/<ab>/<cd>/<cache_key>.meta
// Unlike ContentStorage, the key is provided by the caller (not computed from content).
type VariantStorage struct {
baseDir string
}
// VariantMeta contains metadata about a cached variant.
type VariantMeta struct {
ContentType string `json:"content_type"`
Size int64 `json:"size"`
CreatedAt int64 `json:"created_at"`
}
// NewVariantStorage creates a new variant storage at the given base directory.
func NewVariantStorage(baseDir string) (*VariantStorage, error) {
if err := os.MkdirAll(baseDir, StorageDirPerm); err != nil {
return nil, fmt.Errorf("failed to create variant storage directory: %w", err)
}
return &VariantStorage{baseDir: baseDir}, nil
}
// Store writes content and metadata to storage at the given key.
func (s *VariantStorage) Store(key VariantKey, r io.Reader, contentType string) (size int64, err error) {
data, err := io.ReadAll(r)
if err != nil {
return 0, fmt.Errorf("failed to read content: %w", err)
}
size = int64(len(data))
path := s.keyToPath(key)
metaPath := path + ".meta"
// Create directory structure
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, StorageDirPerm); err != nil {
return 0, fmt.Errorf("failed to create directory: %w", err)
}
// Write content to temp file first, then rename for atomicity
tmpFile, err := os.CreateTemp(dir, ".tmp-*")
if err != nil {
return 0, fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmpFile.Name()
defer func() {
if err != nil {
_ = os.Remove(tmpPath)
}
}()
if _, err := tmpFile.Write(data); err != nil {
_ = tmpFile.Close()
return 0, fmt.Errorf("failed to write content: %w", err)
}
if err := tmpFile.Close(); err != nil {
return 0, fmt.Errorf("failed to close temp file: %w", err)
}
// Atomic rename content
if err := os.Rename(tmpPath, path); err != nil {
return 0, fmt.Errorf("failed to rename temp file: %w", err)
}
// Write metadata
meta := VariantMeta{
ContentType: contentType,
Size: size,
CreatedAt: time.Now().UTC().Unix(),
}
metaData, err := json.Marshal(meta)
if err != nil {
return 0, fmt.Errorf("failed to marshal metadata: %w", err)
}
if err := os.WriteFile(metaPath, metaData, 0640); err != nil {
// Non-fatal, content is stored
_ = err
}
return size, nil
}
// Load returns a reader for the content at the given key.
func (s *VariantStorage) Load(key VariantKey) (io.ReadCloser, error) {
path := s.keyToPath(key)
f, err := os.Open(path) //nolint:gosec // path derived from cache key
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("failed to open content: %w", err)
}
return f, nil
}
// LoadWithMeta returns a reader, size, and content type for the content at the given key.
func (s *VariantStorage) LoadWithMeta(key VariantKey) (io.ReadCloser, int64, string, error) {
path := s.keyToPath(key)
metaPath := path + ".meta"
f, err := os.Open(path) //nolint:gosec // path derived from cache key
if err != nil {
if os.IsNotExist(err) {
return nil, 0, "", ErrNotFound
}
return nil, 0, "", fmt.Errorf("failed to open content: %w", err)
}
stat, err := f.Stat()
if err != nil {
_ = f.Close()
return nil, 0, "", fmt.Errorf("failed to stat content: %w", err)
}
// Load metadata for content type
contentType := "application/octet-stream" // fallback
metaData, err := os.ReadFile(metaPath) //nolint:gosec // path derived from cache key
if err == nil {
var meta VariantMeta
if json.Unmarshal(metaData, &meta) == nil && meta.ContentType != "" {
contentType = meta.ContentType
}
}
return f, stat.Size(), contentType, nil
}
// Exists checks if content exists at the given key.
func (s *VariantStorage) Exists(key VariantKey) bool {
path := s.keyToPath(key)
_, err := os.Stat(path)
return err == nil
}
// Delete removes content at the given key.
func (s *VariantStorage) Delete(key VariantKey) error {
path := s.keyToPath(key)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete content: %w", err)
}
return nil
}
// keyToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
func (s *VariantStorage) keyToPath(key VariantKey) string {
k := string(key)
if len(k) < MinHashLength {
return filepath.Join(s.baseDir, k)
}
return filepath.Join(s.baseDir, k[0:2], k[2:4], k)
} }

View File

@@ -30,7 +30,8 @@ func TestContentStorage_StoreAndLoad(t *testing.T) {
} }
// Verify file exists at expected path // Verify file exists at expected path
expectedPath := filepath.Join(tmpDir, hash[0:2], hash[2:4], hash) hashStr := string(hash)
expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
if _, err := os.Stat(expectedPath); err != nil { if _, err := os.Stat(expectedPath); err != nil {
t.Errorf("File not at expected path %s: %v", expectedPath, err) t.Errorf("File not at expected path %s: %v", expectedPath, err)
} }
@@ -83,7 +84,7 @@ func TestContentStorage_LoadNotFound(t *testing.T) {
t.Fatalf("NewContentStorage() error = %v", err) t.Fatalf("NewContentStorage() error = %v", err)
} }
_, err = storage.Load("nonexistent") _, err = storage.Load(ContentHash("nonexistent"))
if err != ErrNotFound { if err != ErrNotFound {
t.Errorf("Load() error = %v, want ErrNotFound", err) t.Errorf("Load() error = %v, want ErrNotFound", err)
} }
@@ -123,24 +124,29 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) {
} }
// Should not error // Should not error
if err := storage.Delete("nonexistent"); err != nil { if err := storage.Delete(ContentHash("nonexistent")); err != nil {
t.Errorf("Delete() error = %v, want nil", err) t.Errorf("Delete() error = %v, want nil", err)
} }
} }
func TestContentStorage_Path(t *testing.T) { func TestContentStorage_HashToPath(t *testing.T) {
tmpDir := t.TempDir() tmpDir := t.TempDir()
storage, err := NewContentStorage(tmpDir) storage, err := NewContentStorage(tmpDir)
if err != nil { if err != nil {
t.Fatalf("NewContentStorage() error = %v", err) t.Fatalf("NewContentStorage() error = %v", err)
} }
hash := "abcdef0123456789" // Test by storing and verifying the resulting path structure
path := storage.Path(hash) content := []byte("test content for path verification")
expected := filepath.Join(tmpDir, "ab", "cd", hash) hash, _, err := storage.Store(bytes.NewReader(content))
if err != nil {
t.Fatalf("Store() error = %v", err)
}
if path != expected { hashStr := string(hash)
t.Errorf("Path() = %q, want %q", path, expected) expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr)
if _, err := os.Stat(expectedPath); err != nil {
t.Errorf("File not at expected path %s: %v", expectedPath, err)
} }
} }
@@ -169,7 +175,7 @@ func TestMetadataStorage_StoreAndLoad(t *testing.T) {
} }
// Verify file exists at expected path // Verify file exists at expected path
expectedPath := filepath.Join(tmpDir, "cdn.example.com", pathHash+".json") expectedPath := filepath.Join(tmpDir, "cdn.example.com", string(pathHash)+".json")
if _, err := os.Stat(expectedPath); err != nil { if _, err := os.Stat(expectedPath); err != nil {
t.Errorf("File not at expected path %s: %v", expectedPath, err) t.Errorf("File not at expected path %s: %v", expectedPath, err)
} }
@@ -208,7 +214,7 @@ func TestMetadataStorage_LoadNotFound(t *testing.T) {
t.Fatalf("NewMetadataStorage() error = %v", err) t.Fatalf("NewMetadataStorage() error = %v", err)
} }
_, err = storage.Load("example.com", "nonexistent") _, err = storage.Load("example.com", PathHash("nonexistent"))
if err != ErrNotFound { if err != ErrNotFound {
t.Errorf("Load() error = %v, want ErrNotFound", err) t.Errorf("Load() error = %v, want ErrNotFound", err)
} }
@@ -264,8 +270,8 @@ func TestHashPath(t *testing.T) {
} }
// Hash should be 64 hex chars (256 bits) // Hash should be 64 hex chars (256 bits)
if len(hash1) != 64 { if len(string(hash1)) != 64 {
t.Errorf("HashPath() length = %d, want 64", len(hash1)) t.Errorf("HashPath() length = %d, want 64", len(string(hash1)))
} }
} }
@@ -299,8 +305,8 @@ func TestCacheKey(t *testing.T) {
} }
// Key should be 64 hex chars // Key should be 64 hex chars
if len(key1) != 64 { if len(string(key1)) != 64 {
t.Errorf("CacheKey() length = %d, want 64", len(key1)) t.Errorf("CacheKey() length = %d, want 64", len(string(key1)))
} }
// Different size should produce different key // Different size should produce different key

View File

@@ -164,8 +164,6 @@ func SetupTestService(t *testing.T, opts ...TestServiceOption) (*Service, *TestF
StateDir: tmpDir, StateDir: tmpDir,
CacheTTL: time.Hour, CacheTTL: time.Hour,
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
HotCacheSize: 100,
HotCacheEnabled: true,
}) })
if err != nil { if err != nil {
t.Fatalf("failed to create cache: %v", err) t.Fatalf("failed to create cache: %v", err)