Make the cache stats count what is cached, fetched and transcoded (closes #56)
check / check (push) Successful in 3m7s
check / check (push) Successful in 3m7s
Stats read request_cache and output_content, which nothing writes, so TotalItems and TotalSizeBytes were always 0. They now count source_content plus variant_content, the size through UsageBytes; a failed query is still logged at warn. Get counts a miss after the work, passing the bytes fetched from upstream (0 for a cached source; still counted when the fetched source then fails), so upstream_fetch_count and upstream_fetch_bytes move. transform_count is incremented after each successful image processor call. request_cache and output_content stay in the schema; dropping them is a separate decision. Model: opus-5-5
This commit is contained in:
@@ -155,12 +155,14 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e
|
||||
}
|
||||
}
|
||||
|
||||
// Cache miss - check if we have source content cached
|
||||
// Cache miss - process the cached source or fetch it, then count the
|
||||
// miss with the bytes it fetched from upstream, also when it failed
|
||||
cacheKey := CacheKey(req)
|
||||
|
||||
s.cache.IncrementStats(ctx, false, 0)
|
||||
response, fetchedBytes, err := s.processFromSourceOrFetch(ctx, req, cacheKey)
|
||||
|
||||
s.cache.IncrementStats(ctx, false, fetchedBytes)
|
||||
|
||||
response, err := s.processFromSourceOrFetch(ctx, req, cacheKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -268,12 +270,13 @@ func (s *Service) loadCachedSource(contentHash ContentHash) []byte {
|
||||
}
|
||||
|
||||
// processFromSourceOrFetch processes an image, using cached source content
|
||||
// if available.
|
||||
// if available. It also returns the number of bytes fetched from upstream,
|
||||
// as fetchAndProcess does, or 0 when the cached source was used.
|
||||
func (s *Service) processFromSourceOrFetch(
|
||||
ctx context.Context,
|
||||
req *ImageRequest,
|
||||
cacheKey VariantKey,
|
||||
) (*ImageResponse, error) {
|
||||
) (*ImageResponse, int64, error) {
|
||||
// Check if we have cached source content
|
||||
contentHash, _, err := s.cache.LookupSource(ctx, req)
|
||||
if err != nil {
|
||||
@@ -292,26 +295,25 @@ func (s *Service) processFromSourceOrFetch(
|
||||
|
||||
// Fetch from upstream if we don't have source data or it's empty
|
||||
if len(sourceData) == 0 {
|
||||
resp, err := s.fetchAndProcess(ctx, req, cacheKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
return s.fetchAndProcess(ctx, req, cacheKey)
|
||||
}
|
||||
|
||||
// Process using cached source
|
||||
// Process using cached source; nothing was fetched from upstream
|
||||
fetchBytes = int64(len(sourceData))
|
||||
|
||||
return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
|
||||
resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
|
||||
|
||||
return resp, 0, err
|
||||
}
|
||||
|
||||
// fetchAndProcess fetches from upstream, processes, and caches the result.
|
||||
// It also returns the number of bytes fetched from upstream once the
|
||||
// response has been read, including when a later step fails.
|
||||
func (s *Service) fetchAndProcess(
|
||||
ctx context.Context,
|
||||
req *ImageRequest,
|
||||
cacheKey VariantKey,
|
||||
) (*ImageResponse, error) {
|
||||
) (*ImageResponse, int64, error) {
|
||||
// Fetch from upstream
|
||||
sourceURL := req.SourceURL()
|
||||
|
||||
@@ -330,7 +332,7 @@ func (s *Service) fetchAndProcess(
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("upstream fetch failed: %w", err)
|
||||
return nil, 0, fmt.Errorf("upstream fetch failed: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = fetchResult.Content.Close() }()
|
||||
@@ -338,7 +340,7 @@ func (s *Service) fetchAndProcess(
|
||||
// Read and validate the source content
|
||||
sourceData, err := io.ReadAll(fetchResult.Content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read upstream response: %w", err)
|
||||
return nil, 0, fmt.Errorf("failed to read upstream response: %w", err)
|
||||
}
|
||||
|
||||
// Calculate download bitrate
|
||||
@@ -368,7 +370,7 @@ func (s *Service) fetchAndProcess(
|
||||
// Validate magic bytes match content type
|
||||
err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("content validation failed: %w", err)
|
||||
return nil, fetchBytes, fmt.Errorf("content validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Store source content
|
||||
@@ -378,7 +380,9 @@ func (s *Service) fetchAndProcess(
|
||||
// Continue even if caching fails
|
||||
}
|
||||
|
||||
return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
|
||||
resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
|
||||
|
||||
return resp, fetchBytes, err
|
||||
}
|
||||
|
||||
// processAndStore processes an image and stores the result.
|
||||
@@ -406,6 +410,8 @@ func (s *Service) processAndStore(
|
||||
|
||||
processDuration := time.Since(processStart)
|
||||
|
||||
s.cache.IncrementTransformCount(ctx)
|
||||
|
||||
// Read processed content
|
||||
processedData, err := io.ReadAll(processResult.Content)
|
||||
_ = processResult.Content.Close()
|
||||
|
||||
Reference in New Issue
Block a user