Add type-safe hash types for cache storage
Define ContentHash, VariantKey, and PathHash types to replace raw strings, providing compile-time type safety for storage operations. Update storage layer to use typed parameters, refactor cache to use variant storage keyed by VariantKey, and implement source content reuse on cache misses.
This commit is contained in:
@@ -80,39 +80,36 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
||||
|
||||
// Get retrieves a processed image, fetching and processing if necessary.
|
||||
func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) {
|
||||
// Check cache first
|
||||
// Check variant cache first (disk only, no DB)
|
||||
result, err := s.cache.Lookup(ctx, req)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNegativeCache) {
|
||||
return nil, fmt.Errorf("upstream returned error (cached)")
|
||||
}
|
||||
|
||||
s.log.Warn("cache lookup failed", "error", err)
|
||||
}
|
||||
|
||||
// Cache hit - serve from cache
|
||||
// Cache hit - serve directly from disk
|
||||
if result != nil && result.Hit {
|
||||
s.cache.IncrementStats(ctx, true, 0)
|
||||
|
||||
reader, size, err := s.cache.GetOutputWithSize(result.OutputHash)
|
||||
reader, size, contentType, err := s.cache.GetVariant(result.CacheKey)
|
||||
if err != nil {
|
||||
s.log.Error("failed to get cached output", "hash", result.OutputHash, "error", err)
|
||||
// Fall through to re-fetch
|
||||
s.log.Error("failed to get cached variant", "key", result.CacheKey, "error", err)
|
||||
// Fall through to re-process
|
||||
} else {
|
||||
s.cache.IncrementStats(ctx, true, 0)
|
||||
|
||||
return &ImageResponse{
|
||||
Content: reader,
|
||||
ContentLength: size,
|
||||
ContentType: result.ContentType,
|
||||
ContentType: contentType,
|
||||
CacheStatus: CacheHit,
|
||||
ETag: formatETag(result.OutputHash),
|
||||
ETag: formatETag(result.CacheKey),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Cache miss - need to fetch, process, and cache
|
||||
// Cache miss - check if we have source content cached
|
||||
cacheKey := CacheKey(req)
|
||||
s.cache.IncrementStats(ctx, false, 0)
|
||||
|
||||
response, err := s.fetchAndProcess(ctx, req)
|
||||
response, err := s.processFromSourceOrFetch(ctx, req, cacheKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -122,8 +119,62 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// processFromSourceOrFetch processes an image, using cached source content if available.
|
||||
func (s *Service) processFromSourceOrFetch(
|
||||
ctx context.Context,
|
||||
req *ImageRequest,
|
||||
cacheKey VariantKey,
|
||||
) (*ImageResponse, error) {
|
||||
// Check if we have cached source content
|
||||
contentHash, _, err := s.cache.LookupSource(ctx, req)
|
||||
if err != nil {
|
||||
s.log.Warn("source lookup failed", "error", err)
|
||||
}
|
||||
|
||||
var sourceData []byte
|
||||
var fetchBytes int64
|
||||
|
||||
if contentHash != "" {
|
||||
// We have cached source - load it
|
||||
s.log.Debug("using cached source", "hash", contentHash)
|
||||
|
||||
reader, err := s.cache.GetSourceContent(contentHash)
|
||||
if err != nil {
|
||||
s.log.Warn("failed to load cached source, fetching", "error", err)
|
||||
// Fall through to fetch
|
||||
} else {
|
||||
sourceData, err = io.ReadAll(reader)
|
||||
_ = reader.Close()
|
||||
|
||||
if err != nil {
|
||||
s.log.Warn("failed to read cached source, fetching", "error", err)
|
||||
// Fall through to fetch
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch from upstream if we don't have source data
|
||||
if sourceData == nil {
|
||||
resp, err := s.fetchAndProcess(ctx, req, cacheKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Process using cached source
|
||||
fetchBytes = int64(len(sourceData))
|
||||
|
||||
return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
|
||||
}
|
||||
|
||||
// fetchAndProcess fetches from upstream, processes, and caches the result.
|
||||
func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*ImageResponse, error) {
|
||||
func (s *Service) fetchAndProcess(
|
||||
ctx context.Context,
|
||||
req *ImageRequest,
|
||||
cacheKey VariantKey,
|
||||
) (*ImageResponse, error) {
|
||||
// Fetch from upstream
|
||||
sourceURL := req.SourceURL()
|
||||
|
||||
@@ -182,6 +233,17 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
|
||||
// Continue even if caching fails
|
||||
}
|
||||
|
||||
return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
|
||||
}
|
||||
|
||||
// processAndStore processes an image and stores the result.
|
||||
func (s *Service) processAndStore(
|
||||
ctx context.Context,
|
||||
req *ImageRequest,
|
||||
cacheKey VariantKey,
|
||||
sourceData []byte,
|
||||
fetchBytes int64,
|
||||
) (*ImageResponse, error) {
|
||||
// Process the image
|
||||
processStart := time.Now()
|
||||
|
||||
@@ -192,8 +254,16 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
|
||||
|
||||
processDuration := time.Since(processStart)
|
||||
|
||||
// Read processed content
|
||||
processedData, err := io.ReadAll(processResult.Content)
|
||||
_ = processResult.Content.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read processed content: %w", err)
|
||||
}
|
||||
|
||||
// Log conversion details
|
||||
outputSize := processResult.ContentLength
|
||||
outputSize := int64(len(processedData))
|
||||
sizePercent := float64(outputSize) / float64(fetchBytes) * 100.0 //nolint:mnd // percentage calculation
|
||||
|
||||
s.log.Info("image converted",
|
||||
@@ -211,30 +281,18 @@ func (s *Service) fetchAndProcess(ctx context.Context, req *ImageRequest) (*Imag
|
||||
"fit", req.FitMode,
|
||||
)
|
||||
|
||||
// Store output content to cache
|
||||
metaID, err := s.cache.GetSourceMetadataID(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get source metadata ID: %w", err)
|
||||
}
|
||||
|
||||
outputHash, err := s.cache.StoreOutput(ctx, req, metaID, processResult.Content, processResult.ContentType)
|
||||
_ = processResult.Content.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to store output content: %w", err)
|
||||
}
|
||||
|
||||
// Serve from the cached file on disk (same path as cache hits)
|
||||
reader, size, err := s.cache.GetOutputWithSize(outputHash)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read cached output: %w", err)
|
||||
// Store variant to cache
|
||||
if err := s.cache.StoreVariant(cacheKey, bytes.NewReader(processedData), processResult.ContentType); err != nil {
|
||||
s.log.Warn("failed to store variant", "error", err)
|
||||
// Continue even if caching fails
|
||||
}
|
||||
|
||||
return &ImageResponse{
|
||||
Content: reader,
|
||||
ContentLength: size,
|
||||
Content: io.NopCloser(bytes.NewReader(processedData)),
|
||||
ContentLength: outputSize,
|
||||
ContentType: processResult.ContentType,
|
||||
FetchedBytes: int64(len(sourceData)),
|
||||
ETag: formatETag(outputHash),
|
||||
FetchedBytes: fetchBytes,
|
||||
ETag: formatETag(cacheKey),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -309,8 +367,9 @@ func extractStatusCode(err error) int {
|
||||
// etagHashLength is the number of hash characters to use for ETags.
|
||||
const etagHashLength = 16
|
||||
|
||||
// formatETag formats a hash as a quoted ETag value.
|
||||
func formatETag(hash string) string {
|
||||
// formatETag formats a VariantKey as a quoted ETag value.
|
||||
func formatETag(key VariantKey) string {
|
||||
hash := string(key)
|
||||
// Use first 16 characters of hash for a shorter but still unique ETag
|
||||
if len(hash) > etagHashLength {
|
||||
hash = hash[:etagHashLength]
|
||||
|
||||
Reference in New Issue
Block a user