diff --git a/README.md b/README.md index 1bbd2ee..356b6e1 100644 --- a/README.md +++ b/README.md @@ -319,18 +319,20 @@ fetch sends the first request's ID upstream, and the lines logged for the fetch and the transcode carry that ID. A transcode whose request's `downstream_timeout` has ended (for a shared one, -the first request's) stops when it gets its processing slot or before it begins -encoding, and gives the slot back. libvips cannot stop an encode once it has -begun, so an encode keeps its slot until it finishes, which can be after -`downstream_timeout`. The limit of 8192 on each side of the source and of the -output bounds how long that takes. With one libvips thread, as pixad runs it, -encoding 8192x8192 random pixels takes up to about 1 second as JPEG, 15 as PNG, -16 as GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the slowest of 8 or 16 -bits per sample, with or without an alpha channel. WebP, AVIF and JPEG XL are -that slow only with an alpha channel: 8-bit RGB takes about 44, 53 and 28 -seconds. Decoding such a source adds up to about 11 seconds. A WebP, AVIF or -JPEG XL encode can therefore hold its slot past the default `downstream_timeout` -of 60 seconds. +the first request's) stops waiting for a processing slot, or, once it has one, +gives it back without beginning to encode. A request waiting for a shared +transcode that stops this way is answered 503 with the error +`server busy, try again later`, though its own `downstream_timeout` may not have +ended. libvips cannot stop an encode once it has begun, so an encode keeps its +slot until it finishes, which can be after `downstream_timeout`. The limit of +8192 on each side of the source and of the output bounds how long that takes. +With one libvips thread, as pixad runs it, encoding 8192x8192 random pixels +takes up to about 1 second as JPEG, 15 as PNG, 16 as GIF, 63 as WebP, 110 as +AVIF and 96 as JPEG XL, the slowest of 8 or 16 bits per sample, with or without +an alpha channel. WebP, AVIF and JPEG XL are that slow only with an alpha +channel: 8-bit RGB takes about 44, 53 and 28 seconds. Decoding such a source +adds up to about 11 seconds. A WebP, AVIF or JPEG XL encode can therefore hold +its slot past the default `downstream_timeout` of 60 seconds. The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is diff --git a/TODO.md b/TODO.md index 2f5c128..f4b4b1a 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,24 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a transcode stops once its request's `downstream_timeout` has + ended, unless it has begun encoding (closes #243): the image processor looked + at its context only while it waited for a processing slot, so an image was + decoded and encoded, holding the slot, however long after its request's + deadline that took. `Process` now returns the context's error, and gives the + slot back, when the context has ended by the time it has the slot or before it + encodes. A request that waited for a shared transcode stopped at the first + request's deadline is answered 503 `server busy, try again later`, not 500, + though its own `downstream_timeout` may not have ended. libvips cannot stop an + encode once it has begun, so that encode keeps its slot until it finishes; the + limit of 8192 on each side of the source and of the output bounds how long + that takes. With one libvips thread, 8192x8192 random pixels take up to about + 1 second to encode as JPEG, 15 as PNG, 16 as GIF, 63 as WebP, 110 as AVIF and + 96 as JPEG XL, the last three with an alpha channel, plus up to about 11 + seconds to decode the source; `README.md` gives these times and no longer says + the work stops when `downstream_timeout` ends. The settings of the WebP, AVIF + and JPEG XL encoders, which can hold a slot past the default + `downstream_timeout` of 60 seconds, are unchanged. - 2026-10-15 a cached variant is served while its source URL is in the negative cache (closes #245): `Get` checked the negative cache before the variant cache, so once a fetch of the source failed, as one does when another variant @@ -59,21 +77,6 @@ P2: security: per-origin rate limiting and the query now have each `%` written as `%25` and each `:` as `%3A`; a path and query holding neither are signed and keyed as before. Signature Specification in `README.md` says so. -- 2026-10-15 a transcode stops once its request's `downstream_timeout` has - ended, unless it has begun encoding (closes #243): the image processor looked - at its context only while it waited for a processing slot, so an image was - decoded and encoded, holding the slot, however long after its request's - deadline that took. `Process` now returns the context's error, and gives the - slot back, when the context has ended by the time it has the slot or before it - encodes. libvips cannot stop an encode once it has begun, so that encode keeps - its slot until it finishes; the limit of 8192 on each side of the source and - of the output bounds how long that takes. With one libvips thread, 8192x8192 - random pixels take up to about 1 second to encode as JPEG, 15 as PNG, 16 as - GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the last three with an alpha - channel, plus up to about 11 seconds to decode the source; `README.md` gives - these times and no longer says the work stops when `downstream_timeout` ends. - The settings of the WebP, AVIF and JPEG XL encoders, which can hold a slot - past the default `downstream_timeout` of 60 seconds, are unchanged. - 2026-10-15 the example config allowlists no host (closes #249): it allowlisted `github.com` and `user-images.githubusercontent.com`, where anyone can put an image, so a deployment set up as Getting Started says served those images to diff --git a/internal/handlers/image.go b/internal/handlers/image.go index aefde8e..9b28da1 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -240,7 +240,8 @@ func (s *Handlers) respondImageError( } if errors.Is(err, httpfetcher.ErrTooManyConnections) || - errors.Is(err, imageprocessor.ErrTooManyImages) { + errors.Is(err, imageprocessor.ErrTooManyImages) || + errors.Is(err, imgcache.ErrProcessingTimedOut) { s.respondError(w, "server busy, try again later", http.StatusServiceUnavailable) diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 8da6d47..4da08eb 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -158,7 +158,8 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { case errors.Is(err, httpfetcher.ErrUpstreamTimeout): s.respondError(w, "upstream timeout", http.StatusGatewayTimeout) case errors.Is(err, httpfetcher.ErrTooManyConnections), - errors.Is(err, imageprocessor.ErrTooManyImages): + errors.Is(err, imageprocessor.ErrTooManyImages), + errors.Is(err, imgcache.ErrProcessingTimedOut): s.respondError(w, "server busy, try again later", http.StatusServiceUnavailable) case errors.Is(err, httpfetcher.ErrInvalidContentType), diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 2ba0e7f..1806a42 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -555,7 +555,13 @@ func (s *Service) processAndStore( FitMode: imageprocessor.FitMode(req.FitMode), } + // ctx has the deadline of the request doing the processing, which can + // end while a request waiting for its image has time left processResult, err := s.processor.Process(ctx, source, processReq) + if errors.Is(err, context.DeadlineExceeded) { + return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err) + } + if err != nil { return nil, fmt.Errorf("image processing failed: %w", err) }