From ae8b45e93f2410b6d0bba0fe22371a8b9fdd188c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 28 Sep 2026 23:54:28 +0000 Subject: [PATCH] Make the cache stats count what is cached, fetched and transcoded (closes #56) Stats read request_cache and output_content, which nothing writes, so TotalItems and TotalSizeBytes were always 0. They now count source_content plus variant_content, the size through UsageBytes; a failed query is still logged at warn. Get counts a miss after the work, passing the bytes fetched from upstream (0 for a cached source; still counted when the fetched source then fails), so upstream_fetch_count and upstream_fetch_bytes move. transform_count is incremented after each successful image processor call. request_cache and output_content stay in the schema; dropping them is a separate decision. Model: opus-5-5 --- TODO.md | 9 ++++++++ internal/imgcache/cache.go | 26 ++++++++++++++++------ internal/imgcache/imgcache.go | 6 +++-- internal/imgcache/service.go | 42 ++++++++++++++++++++--------------- 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/TODO.md b/TODO.md index a8ec7c3..51cb5a4 100644 --- a/TODO.md +++ b/TODO.md @@ -42,6 +42,15 @@ exhaustion that is sooner, never negative; an allowlisted host's URL that has an `exp` follows it too; `immutable` stays, as freshness now ends at the expiry; documented in `README.md`. +- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` + counts the cached source images and processed variants (`source_content` + plus `variant_content`) and takes their size from `Cache.UsageBytes`, + instead of reading `request_cache` and `output_content`, which nothing + writes; those two tables are left in the schema. A miss is counted after + it is served or fails, with the bytes it fetched from upstream, so + `upstream_fetch_count` and `upstream_fetch_bytes` move, including for a + fetched source that then fails the magic byte check; `transform_count` + counts each image the image processor transcodes. - 2026-09-28 strip metadata from processed images (closes #82): every output is exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC profile; the image is first turned upright with `AutoRotate` (before sizes are diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 172b40e..ed3905c 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -419,17 +419,16 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { return nil, fmt.Errorf("failed to get cache stats: %w", err) } - // Get actual item count and total size from content tables - err = c.db.QueryRowContext(ctx, - `SELECT COUNT(*) FROM request_cache`, - ).Scan(&stats.TotalItems) + // Count and size the cached source images and processed variants + err = c.db.QueryRowContext(ctx, ` + SELECT (SELECT COUNT(*) FROM source_content) + + (SELECT COUNT(*) FROM variant_content) + `).Scan(&stats.TotalItems) if err != nil { c.log.Warn("failed to count cache items for stats", "error", err) } - err = c.db.QueryRowContext(ctx, - `SELECT COALESCE(SUM(size_bytes), 0) FROM output_content`, - ).Scan(&stats.TotalSizeBytes) + stats.TotalSizeBytes, err = c.UsageBytes(ctx) if err != nil { c.log.Warn("failed to sum cache size for stats", "error", err) } @@ -481,6 +480,19 @@ func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) } } +// IncrementTransformCount counts one image transcoded by the image processor. +func (c *Cache) IncrementTransformCount(ctx context.Context) { + _, err := c.db.ExecContext(ctx, ` + UPDATE cache_stats + SET transform_count = transform_count + 1, + last_updated_at = CURRENT_TIMESTAMP + WHERE id = 1 + `) + if err != nil { + c.log.Warn("failed to count transform", "error", err) + } +} + // writeMetadataSidecar writes the JSON metadata sidecar of a stored source. // A failure is logged and is otherwise non-fatal; the metadata is in the // database. diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index a6a591e..a74e783 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -163,9 +163,11 @@ type ImageCache interface { // CacheStats contains cache statistics type CacheStats struct { - // TotalItems is the number of cached items + // TotalItems is the number of cached source images plus processed + // variants TotalItems int64 - // TotalSizeBytes is the total size of cached content + // TotalSizeBytes is the total size of cached source images and + // processed variants TotalSizeBytes int64 // HitCount is the number of cache hits HitCount int64 diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 3b295c7..dc45504 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -155,12 +155,14 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e } } - // Cache miss - check if we have source content cached + // Cache miss - process the cached source or fetch it, then count the + // miss with the bytes it fetched from upstream, also when it failed cacheKey := CacheKey(req) - s.cache.IncrementStats(ctx, false, 0) + response, fetchedBytes, err := s.processFromSourceOrFetch(ctx, req, cacheKey) + + s.cache.IncrementStats(ctx, false, fetchedBytes) - response, err := s.processFromSourceOrFetch(ctx, req, cacheKey) if err != nil { return nil, err } @@ -268,12 +270,13 @@ func (s *Service) loadCachedSource(contentHash ContentHash) []byte { } // processFromSourceOrFetch processes an image, using cached source content -// if available. +// if available. It also returns the number of bytes fetched from upstream, +// as fetchAndProcess does, or 0 when the cached source was used. func (s *Service) processFromSourceOrFetch( ctx context.Context, req *ImageRequest, cacheKey VariantKey, -) (*ImageResponse, error) { +) (*ImageResponse, int64, error) { // Check if we have cached source content contentHash, _, err := s.cache.LookupSource(ctx, req) if err != nil { @@ -292,26 +295,25 @@ func (s *Service) processFromSourceOrFetch( // Fetch from upstream if we don't have source data or it's empty if len(sourceData) == 0 { - resp, err := s.fetchAndProcess(ctx, req, cacheKey) - if err != nil { - return nil, err - } - - return resp, nil + return s.fetchAndProcess(ctx, req, cacheKey) } - // Process using cached source + // Process using cached source; nothing was fetched from upstream fetchBytes = int64(len(sourceData)) - return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + + return resp, 0, err } // fetchAndProcess fetches from upstream, processes, and caches the result. +// It also returns the number of bytes fetched from upstream once the +// response has been read, including when a later step fails. func (s *Service) fetchAndProcess( ctx context.Context, req *ImageRequest, cacheKey VariantKey, -) (*ImageResponse, error) { +) (*ImageResponse, int64, error) { // Fetch from upstream sourceURL := req.SourceURL() @@ -330,7 +332,7 @@ func (s *Service) fetchAndProcess( } } - return nil, fmt.Errorf("upstream fetch failed: %w", err) + return nil, 0, fmt.Errorf("upstream fetch failed: %w", err) } defer func() { _ = fetchResult.Content.Close() }() @@ -338,7 +340,7 @@ func (s *Service) fetchAndProcess( // Read and validate the source content sourceData, err := io.ReadAll(fetchResult.Content) if err != nil { - return nil, fmt.Errorf("failed to read upstream response: %w", err) + return nil, 0, fmt.Errorf("failed to read upstream response: %w", err) } // Calculate download bitrate @@ -368,7 +370,7 @@ func (s *Service) fetchAndProcess( // Validate magic bytes match content type err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) if err != nil { - return nil, fmt.Errorf("content validation failed: %w", err) + return nil, fetchBytes, fmt.Errorf("content validation failed: %w", err) } // Store source content @@ -378,7 +380,9 @@ func (s *Service) fetchAndProcess( // Continue even if caching fails } - return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + + return resp, fetchBytes, err } // processAndStore processes an image and stores the result. @@ -406,6 +410,8 @@ func (s *Service) processAndStore( processDuration := time.Since(processStart) + s.cache.IncrementTransformCount(ctx) + // Read processed content processedData, err := io.ReadAll(processResult.Content) _ = processResult.Content.Close()