sneak/pixa
1
0
Fork 0
Code Issues 31 Pull Requests 4 Actions 3 Packages Projects Releases Wiki Activity

Count interrupted misses and the upstream bytes they read (closes #56)

Browse Source 
The miss and transform counters are written with context.WithoutCancel,
so a client disconnect or the request timeout during or after the work
no longer loses them. A failed read of the upstream body now returns
the bytes read before the error, so an over-size or cut-off body still
moves the upstream fetch counters. processFromSourceOrFetch passes the
cached source's length directly instead of through a local named
fetchBytes.

Model: opus-5-5
This commit is contained in:
clawbot
2026-09-29 02:46:30 +00:00
parent 64108c05bb
commit 614bcd9eb6
2 changed files with 19 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO.md
+5 -4
View File
@@ -47,10 +47,11 @@ exhaustion
plus `variant_content`) and takes their size from `Cache.UsageBytes`,
instead of reading `request_cache` and `output_content`, which nothing
writes; those two tables are left in the schema. A miss is counted after
it is served or fails, with the bytes it fetched from upstream, so
`upstream_fetch_count` and `upstream_fetch_bytes` move, including for a
fetched source that then fails the magic byte check; `transform_count`
counts each image the image processor transcodes.
it is served or fails, even when the request context has ended by then,
with the bytes it read from upstream, so `upstream_fetch_count` and
`upstream_fetch_bytes` move, including for an upstream body that fails
partway or a fetched source that then fails the magic byte check;
`transform_count` counts each image the image processor transcodes.
- 2026-09-28 strip metadata from processed images (closes #82): every output is
exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC
profile; the image is first turned upright with `AutoRotate` (before sizes are
internal/imgcache/service.go
+14 -15
View File
@@ -156,12 +156,13 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e
}
// Cache miss - process the cached source or fetch it, then count the
// miss with the bytes it fetched from upstream, also when it failed
// miss with the bytes it fetched from upstream, also when it failed or
// the request context has ended meanwhile
cacheKey := CacheKey(req)
response, fetchedBytes, err := s.processFromSourceOrFetch(ctx, req, cacheKey)
s.cache.IncrementStats(ctx, false, fetchedBytes)
s.cache.IncrementStats(context.WithoutCancel(ctx), false, fetchedBytes)
if err != nil {
return nil, err
@@ -283,10 +284,7 @@ func (s *Service) processFromSourceOrFetch(
s.log.Warn("source lookup failed", "error", err)
}
var (
sourceData []byte
fetchBytes int64
)
var sourceData []byte
if contentHash != "" {
s.log.Debug("using cached source", "hash", contentHash)
@@ -299,16 +297,16 @@ func (s *Service) processFromSourceOrFetch(
}
// Process using cached source; nothing was fetched from upstream
fetchBytes = int64(len(sourceData))
resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes)
resp, err := s.processAndStore(
ctx, req, cacheKey, sourceData, int64(len(sourceData)),
)
return resp, 0, err
}
// fetchAndProcess fetches from upstream, processes, and caches the result.
// It also returns the number of bytes fetched from upstream once the
// response has been read, including when a later step fails.
// It also returns the number of bytes read from upstream, including when
// reading the response or a later step fails.
func (s *Service) fetchAndProcess(
ctx context.Context,
req *ImageRequest,
@@ -339,13 +337,13 @@ func (s *Service) fetchAndProcess(
// Read and validate the source content
sourceData, err := io.ReadAll(fetchResult.Content)
fetchBytes := int64(len(sourceData))
if err != nil {
return nil, 0, fmt.Errorf("failed to read upstream response: %w", err)
return nil, fetchBytes, fmt.Errorf("failed to read upstream response: %w", err)
}
// Calculate download bitrate
fetchBytes := int64(len(sourceData))
var downloadRate string
if fetchResult.FetchDurationMs > 0 {
@@ -410,7 +408,8 @@ func (s *Service) processAndStore(
processDuration := time.Since(processStart)
s.cache.IncrementTransformCount(ctx)
// Counted also when the request context has ended meanwhile
s.cache.IncrementTransformCount(context.WithoutCancel(ctx))
// Read processed content
processedData, err := io.ReadAll(processResult.Content)