From 2afe61e3019f22ceb0fdc5558a9bd10a0c42a4c0 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 03:51:58 +0200 Subject: [PATCH 001/131] Keep max-age within an expiring image URL's lifetime (closes #63) Both image routes sent Cache-Control: public, max-age=31536000, immutable unconditionally, so a browser or proxy could keep serving an image for a year after its signed or encrypted URL had expired. max-age is now the whole seconds left until the URL expires, never negative and at most one year; a URL with no expiry keeps one year. The 304 answer uses the same value. An encrypted URL's expiry now reaches ImageRequest.Expires through ToImageRequest. immutable stays: freshness now ends no later than the URL's expiry. README.md documents the header. Model: opus-5-5 --- README.md | 7 + TODO.md | 7 + internal/encurl/encurl.go | 9 +- internal/handlers/image.go | 20 +- .../image_cache_control_internal_test.go | 203 ++++++++++++++++++ .../handlers/image_signature_internal_test.go | 6 +- internal/handlers/imageenc.go | 4 +- internal/imgcache/imgcache.go | 3 +- 8 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 internal/handlers/image_cache_control_internal_test.go diff --git a/README.md b/README.md index 4765703..23fe41b 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,13 @@ than once, is refused with 400. - ``: one of `orig`, `png`, `jpeg`, `webp` - ``: `orig` or `x` (e.g. `800x600`) +An image is served with `Cache-Control: public, max-age=, immutable`. +When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), +`max-age` is the whole seconds left until then, at most one year, so no browser +or proxy cache keeps the image after pixa would refuse the URL. A URL with no +expiry gets one year. `immutable` only stops a client revalidating while its +copy is fresh. + The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is refused with 429 and a `Retry-After` header. Behind a reverse proxy the client diff --git a/TODO.md b/TODO.md index 56d0745..c10609b 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,13 @@ exhaustion # Completed Steps +- 2026-09-29 `max-age` never outlives an expiring URL (closes #63): both image + routes build `Cache-Control` from the request's `Expires`, which an encrypted + URL's expiry now fills too; `max-age` is one year, or the whole seconds left + until the `exp` of a `/v1/image/` URL or the expiry of an encrypted URL when + that is sooner, never negative; an allowlisted host's URL that has an `exp` + follows it too; `immutable` stays, as freshness now ends at the expiry; + documented in `README.md`. - 2026-09-28 strip metadata from processed images (closes #82): every output is exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC profile; the image is first turned upright with `AutoRotate` (before sizes are diff --git a/internal/encurl/encurl.go b/internal/encurl/encurl.go index 5c8819c..8815bf0 100644 --- a/internal/encurl/encurl.go +++ b/internal/encurl/encurl.go @@ -103,7 +103,8 @@ func (g *Generator) Parse(token string) (*Payload, error) { } // ToImageRequest converts the payload to an ImageRequest. -// Applies default values for omitted optional fields. +// Applies default values for omitted optional fields. An ExpiresAt of 0, a URL +// that never expires, gives the zero Expires. func (p *Payload) ToImageRequest() *imgcache.ImageRequest { format := p.Format if format == "" { @@ -120,6 +121,11 @@ func (p *Payload) ToImageRequest() *imgcache.ImageRequest { fitMode = DefaultFitMode } + var expires time.Time + if p.ExpiresAt != 0 { + expires = time.Unix(p.ExpiresAt, 0) + } + return &imgcache.ImageRequest{ SourceHost: p.SourceHost, SourcePath: p.SourcePath, @@ -131,6 +137,7 @@ func (p *Payload) ToImageRequest() *imgcache.ImageRequest { Format: format, Quality: quality, FitMode: fitMode, + Expires: expires, } } diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 00b271e..561ca1d 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -220,6 +220,24 @@ func (s *Handlers) respondImageError( s.respondError(w, "internal error", http.StatusInternalServerError) } +// cacheControl returns the Cache-Control header for an image served through a +// URL that expires at expires, or never when expires is the zero time. A cache +// may keep the image for a year, but not past the URL's expiry, after which +// pixa refuses the URL. The seconds left are rounded down and never negative. +// immutable only stops revalidation while the image is fresh, so it also ends +// at the expiry. +func cacheControl(expires time.Time) string { + const oneYear = 365 * 24 * time.Hour + + maxAge := oneYear + + if !expires.IsZero() { + maxAge = min(maxAge, max(time.Until(expires), 0)) + } + + return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second)) +} + // writeImageResponse writes headers and streams the image content, // handling conditional and HEAD requests. func (s *Handlers) writeImageResponse( @@ -235,7 +253,7 @@ func (s *Handlers) writeImageResponse( } // Cache control headers - w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + w.Header().Set("Cache-Control", cacheControl(req.Expires)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) if resp.ETag != "" { diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go new file mode 100644 index 0000000..bb6a6b9 --- /dev/null +++ b/internal/handlers/image_cache_control_internal_test.go @@ -0,0 +1,203 @@ +package handlers + +import ( + "image/color" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "testing/fstest" + "time" + + "github.com/go-chi/chi/v5" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// photoPath is the path of the JPEG that newSignedHostServer serves. +const photoPath = "/images/photo.jpg" + +// newSignedHostServer returns a router for both image routes, and the Handlers +// behind it, whose fetcher serves a JPEG at photoPath on signedHost. signedHost +// is not on the allowlist, so a /v1/image/ URL for it is served only with a +// valid signature. +func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) { + t.Helper() + + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + StateDir: t.TempDir(), + CacheTTL: time.Hour, + NegativeTTL: 5 * time.Minute, + }) + if err != nil { + t.Fatalf("imgcache.NewCache() error = %v", err) + } + + jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) + + svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + Cache: cache, + Fetcher: newMockFetcher(fstest.MapFS{ + signedHost + photoPath: &fstest.MapFile{Data: jpegData}, + }), + SigningKey: testSigningKey, + }) + if err != nil { + t.Fatalf("imgcache.NewService() error = %v", err) + } + + encGen, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator() error = %v", err) + } + + h := &Handlers{ + log: slog.New(slog.DiscardHandler), + imgSvc: svc, + encGen: encGen, + } + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + r.Get("/v1/e/{token}/*", h.HandleImageEnc()) + + return h, r +} + +// getMaxAge sends a GET for target to srv, requires a 200, and returns the +// max-age of the response's Cache-Control header, which must read +// "public, max-age=, immutable". +func getMaxAge(t *testing.T, srv http.Handler, target string) int { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil) + rec := httptest.NewRecorder() + + srv.ServeHTTP(rec, req) + + header := rec.Header().Get("Cache-Control") + t.Logf("GET %s: %d, Cache-Control: %s", target, rec.Code, header) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + + value, hasPrefix := strings.CutPrefix(header, "public, max-age=") + value, hasSuffix := strings.CutSuffix(value, ", immutable") + + maxAge, err := strconv.Atoi(value) + if !hasPrefix || !hasSuffix || err != nil { + t.Fatalf("Cache-Control = %q, want public, max-age=, immutable", + header) + } + + return maxAge +} + +// TestHandleImage_SignedURL_MaxAgeEndsAtExp verifies that an image served +// through a signed URL expiring in 60 seconds may be cached for at most those +// 60 seconds. The lower bound of 50 shows the max-age is the time left, not 0. +func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t) + + signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ + SourceHost: signedHost, + SourcePath: photoPath, + Size: imgcache.Size{Width: 50, Height: 50}, + Format: imgcache.FormatJPEG, + }, time.Minute) + if err != nil { + t.Fatalf("GenerateSignedURL() error = %v", err) + } + + maxAge := getMaxAge(t, srv, signedURL) + if maxAge < 50 || maxAge > 60 { + t.Errorf("max-age = %d, want 50 to 60", maxAge) + } +} + +// TestHandleImage_AllowlistedHost_MaxAge verifies the max-age of an image from +// an allowlisted host, which is served without checking sig or exp. A URL with +// no exp may be cached for a year. A URL whose exp has passed is the one request +// that reaches the header after its expiry, and must get 0, never less. +func TestHandleImage_AllowlistedHost_MaxAge(t *testing.T) { + t.Parallel() + + pastExp := strconv.FormatInt(time.Now().Add(-time.Hour).Unix(), 10) + + tests := []struct { + name string + query string + wantMaxAge int + }{ + {"no exp", "", 31536000}, + {"exp already past", "?exp=" + pastExp, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fix := setupTestHandler(t) + + r := chi.NewRouter() + r.Get("/v1/image/*", fix.handler.HandleImage()) + + maxAge := getMaxAge(t, r, + "/v1/image/"+fix.goodHost+"/images/photo.jpg/50x50.jpeg"+tt.query) + if maxAge != tt.wantMaxAge { + t.Errorf("max-age = %d, want %d", maxAge, tt.wantMaxAge) + } + }) + } +} + +// TestHandleImageEnc_MaxAge verifies that an image served through an encrypted +// URL with a 60 second TTL may be cached for at most those 60 seconds, that one +// with a two-year TTL may be cached for a year, and that one made without a +// TTL, which never expires, may be cached for a year. +func TestHandleImageEnc_MaxAge(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + expiresAt int64 + wantAtLeast int + wantAtMost int + }{ + {"60 second TTL", time.Now().Add(time.Minute).Unix(), 50, 60}, + {"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour).Unix(), 31536000, 31536000}, + {"no TTL", 0, 31536000, 31536000}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t) + + token, err := h.encGen.Generate(&encurl.Payload{ + SourceHost: signedHost, + SourcePath: photoPath, + Width: 50, + Height: 50, + Format: imgcache.FormatJPEG, + ExpiresAt: tt.expiresAt, + }) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + maxAge := getMaxAge(t, srv, "/v1/e/"+token+"/img.jpg") + if maxAge < tt.wantAtLeast || maxAge > tt.wantAtMost { + t.Errorf("max-age = %d, want %d to %d", + maxAge, tt.wantAtLeast, tt.wantAtMost) + } + }) + } +} diff --git a/internal/handlers/image_signature_internal_test.go b/internal/handlers/image_signature_internal_test.go index c435668..c057514 100644 --- a/internal/handlers/image_signature_internal_test.go +++ b/internal/handlers/image_signature_internal_test.go @@ -14,9 +14,9 @@ import ( ) // signedHost is not on the allowlist setupTestHandler builds, so a request -// for it needs a valid signature. No image is served for it: a request that -// passes the signature check gets 502 from the failed fetch, and one that -// fails the check gets 401. +// for it needs a valid signature. setupTestHandler serves no image for it: a +// request that passes the signature check gets 502 from the failed fetch, and +// one that fails the check gets 401. const signedHost = "signed.example.com" // getImage sends a GET for target to the image route of fix and returns the diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 8089f64..5effd8b 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -89,8 +89,8 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10)) } - // Cache headers - encrypted URLs can be cached since they're immutable - w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + // Cache headers: max-age ends at the URL's expiry + w.Header().Set("Cache-Control", cacheControl(req.Expires)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) // Stream the response diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index 7fb069b..a6a591e 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -95,7 +95,8 @@ type ImageRequest struct { FitMode FitMode // Signature is the HMAC signature for non-allowlisted hosts Signature string - // Expires is the signature expiration timestamp + // Expires is when the URL expires: the exp of a signed URL, or the expiry + // of an encrypted URL; the zero time if it has none Expires time.Time // AllowHTTP indicates whether HTTP (non-TLS) is allowed for this request AllowHTTP bool From ed3f8770e60ce83ee6187ff48563bd4a52f84f22 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 04:44:49 +0200 Subject: [PATCH 002/131] Give pixad a fixed uid and gid 65532 (closes #151) adduser took the first free uid, 1000, and the entrypoint gives a bind-mounted /var/lib/pixa to pixad, so on the host a person's login account ended up owning pixa's database and cache. The image now creates the pixad group with gid 65532 and the pixad user with uid 65532, which host login and system accounts do not use. The first-run step of "Running under upaas" in README.md names the uid and gid. Model: opus-5-5 --- Dockerfile | 8 ++++++-- README.md | 6 ++++-- TODO.md | 5 +++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 77510a0..45027be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,8 +68,12 @@ RUN apk add --no-cache \ COPY --from=builder /pixad /usr/local/bin/pixad COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -# Create non-root user, config directory, and data directory -RUN adduser -D -H -s /sbin/nologin pixad && \ +# Create non-root user, config directory, and data directory. pixad +# gets uid and gid 65532, which host login and system accounts do not +# use: a bind-mounted /var/lib/pixa is given to pixad, and on the host +# it must not belong to a person's account. +RUN addgroup -g 65532 pixad && \ + adduser -D -H -s /sbin/nologin -u 65532 -G pixad pixad && \ mkdir -p /var/lib/pixa /etc/pixa && \ chown pixad:pixad /var/lib/pixa diff --git a/README.md b/README.md index 23fe41b..ec673d1 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,10 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: `healthy`. The probe uses the port from `PORT` (default `8080`), so a port changed only in a mounted config file is not seen by it: change the port with `PORT`. -- **First run:** create the host directory. It may be owned by root: the - container gives it to its `pixad` user when it starts. +- **First run:** create the host directory, owned by root or by uid + `65532` and gid `65532`. The server runs as the container's `pixad` + user, which has that uid and gid, and the container gives the + directory to `pixad` when it starts. ## Rationale diff --git a/TODO.md b/TODO.md index c10609b..a8ec7c3 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,11 @@ exhaustion # Completed Steps +- 2026-09-29 fixed uid and gid for `pixad` (closes #151): the image creates the + `pixad` group with gid 65532 and the `pixad` user with uid 65532, instead of + the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad` + is not owned on the host by a person's login account; the first-run step of + "Running under upaas" in `README.md` names the uid and gid. - 2026-09-29 `max-age` never outlives an expiring URL (closes #63): both image routes build `Cache-Control` from the request's `Expires`, which an encrypted URL's expiry now fills too; `max-age` is one year, or the whole seconds left From e98b998cb6e644a4af2d78fc778034065d8f21b4 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 05:25:10 +0200 Subject: [PATCH 003/131] Make the cache stats count what is cached, fetched and transcoded (closes #56) Cache.Stats read request_cache and output_content, which nothing writes, so TotalItems and TotalSizeBytes were always 0. They now count source_content plus variant_content and use UsageBytes; a disabled disk cache reports 0 for both. The upstream fetch count and bytes and the transform count never moved: Get now passes the bytes it fetched (including those read before a failed body read) and counts each successful transcode. Hits, misses and these counters are written with context.WithoutCancel, so a client disconnect or the request timeout no longer loses them. The unused tables stay; metaCache is #70. Model: opus-5-5 --- TODO.md | 12 + internal/imgcache/cache.go | 39 ++- internal/imgcache/imgcache.go | 6 +- internal/imgcache/service.go | 58 +++-- internal/imgcache/stats_internal_test.go | 316 ++++++++++++++++++++++- 5 files changed, 390 insertions(+), 41 deletions(-) diff --git a/TODO.md b/TODO.md index a8ec7c3..622d81f 100644 --- a/TODO.md +++ b/TODO.md @@ -42,6 +42,18 @@ exhaustion that is sooner, never negative; an allowlisted host's URL that has an `exp` follows it too; `immutable` stays, as freshness now ends at the expiry; documented in `README.md`. +- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` + counts the cached source images and processed variants (`source_content` + plus `variant_content`) and takes their size from `Cache.UsageBytes`, + instead of reading `request_cache` and `output_content`, which nothing + writes; those two tables are left in the schema; a disabled disk cache + reports no items and no size. A hit is counted even when the request + context has ended. A miss is counted after it is served or fails, also + when the request context has ended by then, with the bytes it read from + upstream, so `upstream_fetch_count` and `upstream_fetch_bytes` move, + including for an upstream body that fails partway or a fetched source + that then fails the magic byte check; `transform_count` counts each image + the image processor transcodes. - 2026-09-28 strip metadata from processed images (closes #82): every output is exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC profile; the image is first turned upright with `AutoRotate` (before sizes are diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 172b40e..e15b90a 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -419,19 +419,21 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { return nil, fmt.Errorf("failed to get cache stats: %w", err) } - // Get actual item count and total size from content tables - err = c.db.QueryRowContext(ctx, - `SELECT COUNT(*) FROM request_cache`, - ).Scan(&stats.TotalItems) - if err != nil { - c.log.Warn("failed to count cache items for stats", "error", err) - } + // Count and size the cached source images and processed variants. A + // disabled cache holds none, whatever rows an earlier run left. + if !c.disabled { + err = c.db.QueryRowContext(ctx, ` + SELECT (SELECT COUNT(*) FROM source_content) + + (SELECT COUNT(*) FROM variant_content) + `).Scan(&stats.TotalItems) + if err != nil { + c.log.Warn("failed to count cache items for stats", "error", err) + } - err = c.db.QueryRowContext(ctx, - `SELECT COALESCE(SUM(size_bytes), 0) FROM output_content`, - ).Scan(&stats.TotalSizeBytes) - if err != nil { - c.log.Warn("failed to sum cache size for stats", "error", err) + stats.TotalSizeBytes, err = c.UsageBytes(ctx) + if err != nil { + c.log.Warn("failed to sum cache size for stats", "error", err) + } } // Compute hit rate as a ratio @@ -481,6 +483,19 @@ func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) } } +// IncrementTransformCount counts one image transcoded by the image processor. +func (c *Cache) IncrementTransformCount(ctx context.Context) { + _, err := c.db.ExecContext(ctx, ` + UPDATE cache_stats + SET transform_count = transform_count + 1, + last_updated_at = CURRENT_TIMESTAMP + WHERE id = 1 + `) + if err != nil { + c.log.Warn("failed to count transform", "error", err) + } +} + // writeMetadataSidecar writes the JSON metadata sidecar of a stored source. // A failure is logged and is otherwise non-fatal; the metadata is in the // database. diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index a6a591e..a74e783 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -163,9 +163,11 @@ type ImageCache interface { // CacheStats contains cache statistics type CacheStats struct { - // TotalItems is the number of cached items + // TotalItems is the number of cached source images plus processed + // variants TotalItems int64 - // TotalSizeBytes is the total size of cached content + // TotalSizeBytes is the total size of cached source images and + // processed variants TotalSizeBytes int64 // HitCount is the number of cache hits HitCount int64 diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 3b295c7..d85dee8 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -143,7 +143,8 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e s.log.Error("failed to get cached variant", "key", result.CacheKey, "error", err) // Fall through to re-process } else { - s.cache.IncrementStats(ctx, true, 0) + // Counted also when the request context has ended meanwhile + s.cache.IncrementStats(context.WithoutCancel(ctx), true, 0) return &ImageResponse{ Content: reader, @@ -155,12 +156,15 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e } } - // Cache miss - check if we have source content cached + // Cache miss - process the cached source or fetch it, then count the + // miss with the bytes it fetched from upstream, also when it failed or + // the request context has ended meanwhile cacheKey := CacheKey(req) - s.cache.IncrementStats(ctx, false, 0) + response, fetchedBytes, err := s.processFromSourceOrFetch(ctx, req, cacheKey) + + s.cache.IncrementStats(context.WithoutCancel(ctx), false, fetchedBytes) - response, err := s.processFromSourceOrFetch(ctx, req, cacheKey) if err != nil { return nil, err } @@ -268,22 +272,20 @@ func (s *Service) loadCachedSource(contentHash ContentHash) []byte { } // processFromSourceOrFetch processes an image, using cached source content -// if available. +// if available. It also returns the number of bytes fetched from upstream, +// as fetchAndProcess does, or 0 when the cached source was used. func (s *Service) processFromSourceOrFetch( ctx context.Context, req *ImageRequest, cacheKey VariantKey, -) (*ImageResponse, error) { +) (*ImageResponse, int64, error) { // Check if we have cached source content contentHash, _, err := s.cache.LookupSource(ctx, req) if err != nil { s.log.Warn("source lookup failed", "error", err) } - var ( - sourceData []byte - fetchBytes int64 - ) + var sourceData []byte if contentHash != "" { s.log.Debug("using cached source", "hash", contentHash) @@ -292,26 +294,25 @@ func (s *Service) processFromSourceOrFetch( // Fetch from upstream if we don't have source data or it's empty if len(sourceData) == 0 { - resp, err := s.fetchAndProcess(ctx, req, cacheKey) - if err != nil { - return nil, err - } - - return resp, nil + return s.fetchAndProcess(ctx, req, cacheKey) } - // Process using cached source - fetchBytes = int64(len(sourceData)) + // Process using cached source; nothing was fetched from upstream + resp, err := s.processAndStore( + ctx, req, cacheKey, sourceData, int64(len(sourceData)), + ) - return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + return resp, 0, err } // fetchAndProcess fetches from upstream, processes, and caches the result. +// It also returns the number of bytes read from upstream, including when +// reading the response or a later step fails. func (s *Service) fetchAndProcess( ctx context.Context, req *ImageRequest, cacheKey VariantKey, -) (*ImageResponse, error) { +) (*ImageResponse, int64, error) { // Fetch from upstream sourceURL := req.SourceURL() @@ -330,20 +331,20 @@ func (s *Service) fetchAndProcess( } } - return nil, fmt.Errorf("upstream fetch failed: %w", err) + return nil, 0, fmt.Errorf("upstream fetch failed: %w", err) } defer func() { _ = fetchResult.Content.Close() }() // Read and validate the source content sourceData, err := io.ReadAll(fetchResult.Content) + fetchBytes := int64(len(sourceData)) + if err != nil { - return nil, fmt.Errorf("failed to read upstream response: %w", err) + return nil, fetchBytes, fmt.Errorf("failed to read upstream response: %w", err) } // Calculate download bitrate - fetchBytes := int64(len(sourceData)) - var downloadRate string if fetchResult.FetchDurationMs > 0 { @@ -368,7 +369,7 @@ func (s *Service) fetchAndProcess( // Validate magic bytes match content type err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) if err != nil { - return nil, fmt.Errorf("content validation failed: %w", err) + return nil, fetchBytes, fmt.Errorf("content validation failed: %w", err) } // Store source content @@ -378,7 +379,9 @@ func (s *Service) fetchAndProcess( // Continue even if caching fails } - return s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + + return resp, fetchBytes, err } // processAndStore processes an image and stores the result. @@ -406,6 +409,9 @@ func (s *Service) processAndStore( processDuration := time.Since(processStart) + // Counted also when the request context has ended meanwhile + s.cache.IncrementTransformCount(context.WithoutCancel(ctx)) + // Read processed content processedData, err := io.ReadAll(processResult.Content) _ = processResult.Content.Close() diff --git a/internal/imgcache/stats_internal_test.go b/internal/imgcache/stats_internal_test.go index 5076179..94ee241 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imgcache/stats_internal_test.go @@ -4,6 +4,9 @@ import ( "bytes" "context" "database/sql" + "image/color" + "io" + "io/fs" "log/slog" "math" "strings" @@ -11,6 +14,7 @@ import ( "time" "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/httpfetcher" ) func setupStatsTestDB(t *testing.T) *sql.DB { @@ -125,7 +129,7 @@ func TestStats_LogsFailedCountQueries(t *testing.T) { } _, err = db.ExecContext(t.Context(), - `DROP TABLE request_cache; DROP TABLE output_content`) + `DROP TABLE source_content; DROP TABLE variant_content`) if err != nil { t.Fatal(err) } @@ -183,3 +187,313 @@ func TestIncrementStats_LogsFailedUpdates(t *testing.T) { } } } + +// TestStats_TotalsCountSourcesAndVariants verifies that TotalItems and +// TotalSizeBytes cover the stored source images and processed variants. +func TestStats_TotalsCountSourcesAndVariants(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + storeEvictionTestSource(t, cache, testHostCDN, testPathCat, + bytes.Repeat([]byte{0xAA}, 1000)) + storeEvictionTestVariant(t, cache, testVariantKeyOne, + bytes.Repeat([]byte{0xAB}, 500)) + storeEvictionTestVariant(t, cache, testVariantKeyTwo, + bytes.Repeat([]byte{0xAC}, 250)) + + stats, err := cache.Stats(t.Context()) + if err != nil { + t.Fatalf("Stats() error = %v", err) + } + + if stats.TotalItems != 3 { + t.Errorf("TotalItems = %d, want 3 (1 source, 2 variants)", stats.TotalItems) + } + + if stats.TotalSizeBytes != 1750 { + t.Errorf("TotalSizeBytes = %d, want 1750 (1000+500+250)", + stats.TotalSizeBytes) + } +} + +// TestStats_DisabledCacheReportsNoItems verifies that a disabled disk cache +// reports no items and no size, even when its database still holds the +// rows of an earlier run with the disk cache enabled. +func TestStats_DisabledCacheReportsNoItems(t *testing.T) { + t.Parallel() + + enabled, _ := newEvictionTestCache(t, 1<<30) + + storeEvictionTestSource(t, enabled, testHostCDN, testPathCat, + bytes.Repeat([]byte{0xAA}, 1000)) + storeEvictionTestVariant(t, enabled, testVariantKeyOne, + bytes.Repeat([]byte{0xAB}, 500)) + + disabled, err := NewCache(enabled.db, CacheConfig{ + StateDir: t.TempDir(), + CacheTTL: time.Hour, + NegativeTTL: 5 * time.Minute, + DisableDiskCache: true, + }) + if err != nil { + t.Fatal(err) + } + + stats, err := disabled.Stats(t.Context()) + if err != nil { + t.Fatalf("Stats() error = %v", err) + } + + if stats.TotalItems != 0 || stats.TotalSizeBytes != 0 { + t.Errorf("TotalItems = %d, TotalSizeBytes = %d, want 0 and 0", + stats.TotalItems, stats.TotalSizeBytes) + } +} + +// cacheStatsCounters holds the counters of the cache_stats row, in column +// order. +type cacheStatsCounters struct { + hitCount int64 + missCount int64 + upstreamFetchCount int64 + upstreamFetchBytes int64 + transformCount int64 +} + +// readCacheStatsCounters reads the counters of the cache_stats row. +func readCacheStatsCounters(t *testing.T, cache *Cache) cacheStatsCounters { + t.Helper() + + var got cacheStatsCounters + + err := cache.db.QueryRowContext(t.Context(), ` + SELECT hit_count, miss_count, upstream_fetch_count, + upstream_fetch_bytes, transform_count + FROM cache_stats WHERE id = 1 + `).Scan(&got.hitCount, &got.missCount, &got.upstreamFetchCount, + &got.upstreamFetchBytes, &got.transformCount) + if err != nil { + t.Fatalf("failed to read cache_stats: %v", err) + } + + return got +} + +// TestService_Get_CountsStats walks Get through a miss that fetches the +// source, a hit, a miss that reuses the cached source, and two misses whose +// source cannot be used, checking every cache_stats counter after each. +func TestService_Get_CountsStats(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + + // NewTestFS builds the same files the test service's fetcher serves. + testFS, _ := NewTestFS(t) + + photo, err := fs.ReadFile(testFS, fixtures.GoodHostJPEG) + if err != nil { + t.Fatal(err) + } + + fake, err := fs.ReadFile(testFS, fixtures.InvalidFile) + if err != nil { + t.Fatal(err) + } + + photoBytes, fakeBytes := int64(len(photo)), int64(len(fake)) + + // want is hits, misses, upstream fetches, upstream bytes, transforms. + steps := []struct { + name string + path string + size int + wantErr bool + want cacheStatsCounters + }{ + {"miss that fetches the source", testPathPhoto, 50, false, + cacheStatsCounters{0, 1, 1, photoBytes, 1}}, + {"hit", testPathPhoto, 50, false, + cacheStatsCounters{1, 1, 1, photoBytes, 1}}, + {"miss that reuses the cached source", testPathPhoto, 25, false, + cacheStatsCounters{1, 2, 1, photoBytes, 2}}, + {"miss whose source fails the magic byte check", "/images/fake.jpg", 50, true, + cacheStatsCounters{1, 3, 2, photoBytes + fakeBytes, 2}}, + {"miss whose source is not found", "/images/nonexistent.jpg", 50, true, + cacheStatsCounters{1, 4, 2, photoBytes + fakeBytes, 2}}, + } + + for _, step := range steps { + resp, err := svc.Get(t.Context(), &ImageRequest{ + SourceHost: fixtures.GoodHost, + SourcePath: step.path, + Size: Size{Width: step.size, Height: step.size}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + }) + if (err != nil) != step.wantErr { + t.Fatalf("%s: Get() error = %v, want error %t", step.name, err, step.wantErr) + } + + if err == nil { + _ = resp.Content.Close() + } + + got := readCacheStatsCounters(t, svc.cache) + if got != step.want { + t.Fatalf("after the %s: counters = %+v, want %+v", step.name, got, step.want) + } + } +} + +// fakeUpstream answers every fetch with itself as a JPEG body. The body +// serves data, then calls cancel, when set, and returns err; io.EOF ends +// the body normally. +type fakeUpstream struct { + data *bytes.Reader + cancel context.CancelFunc + err error +} + +func (u *fakeUpstream) Fetch( + context.Context, string, +) (*httpfetcher.FetchResult, error) { + return &httpfetcher.FetchResult{ + Content: io.NopCloser(u), + ContentLength: -1, + ContentType: testContentTypeJPEG, + }, nil +} + +func (u *fakeUpstream) Read(p []byte) (int, error) { + if u.data.Len() > 0 { + return u.data.Read(p) + } + + if u.cancel != nil { + u.cancel() + } + + return 0, u.err +} + +// TestService_Get_CountsInterruptedMisses checks every cache_stats counter +// after a miss whose request context ends during or after the upstream +// fetch, and after a miss whose upstream body is over the size limit. +func TestService_Get_CountsInterruptedMisses(t *testing.T) { + t.Parallel() + + photo := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) + half := len(photo) / 2 + + // want is hits, misses, upstream fetches, upstream bytes, transforms. + tests := []struct { + name string + served int // bytes of the photo the upstream body serves + cancel bool // whether the body then ends the request context + readErr error // what the body then returns + wantErr bool + want cacheStatsCounters + }{ + {"request context ends during the fetch", half, true, context.Canceled, true, + cacheStatsCounters{0, 1, 1, int64(half), 0}}, + {"request context ends after the fetch", len(photo), true, io.EOF, false, + cacheStatsCounters{0, 1, 1, int64(len(photo)), 1}}, + {"upstream body over the size limit", half, false, + httpfetcher.ErrResponseTooLarge, true, + cacheStatsCounters{0, 1, 1, int64(half), 0}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + upstream := &fakeUpstream{ + data: bytes.NewReader(photo[:tc.served]), + err: tc.readErr, + } + if tc.cancel { + upstream.cancel = cancel + } + + svc.fetcher = upstream + + resp, err := svc.Get(ctx, &ImageRequest{ + SourceHost: fixtures.GoodHost, + SourcePath: testPathPhoto, + Size: Size{Width: 50, Height: 50}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + }) + t.Logf("Get() error = %v", err) + + if (err != nil) != tc.wantErr { + t.Fatalf("Get() error = %v, want error %t", err, tc.wantErr) + } + + if err == nil { + _ = resp.Content.Close() + } + + got := readCacheStatsCounters(t, svc.cache) + if got != tc.want { + t.Errorf("counters = %+v, want %+v", got, tc.want) + } + }) + } +} + +// TestService_Get_CountsHitAfterRequestEnds checks every cache_stats counter +// after a hit served with a request context that has already ended: only +// the hit count moves. +func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + + req := &ImageRequest{ + SourceHost: fixtures.GoodHost, + SourcePath: testPathPhoto, + Size: Size{Width: 50, Height: 50}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } + + // A first request caches the variant. + resp, err := svc.Get(t.Context(), req) + if err != nil { + t.Fatalf("first Get() error = %v", err) + } + + _ = resp.Content.Close() + + want := readCacheStatsCounters(t, svc.cache) + want.hitCount++ + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + resp, err = svc.Get(ctx, req) + if err != nil { + t.Fatalf("Get() with an ended request context: error = %v", err) + } + + _ = resp.Content.Close() + + if resp.CacheStatus != CacheHit { + t.Fatalf("CacheStatus = %v, want %v", resp.CacheStatus, CacheHit) + } + + got := readCacheStatsCounters(t, svc.cache) + if got != want { + t.Errorf("counters = %+v, want %+v", got, want) + } +} From 1b920fe000b06a139a7b0eb9081708441b4655f6 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 06:05:44 +0200 Subject: [PATCH 004/131] Correct trusted_proxies advice and state signature padding (closes #150) The README told operators to set trusted_proxies to the proxy's own address. A proxy on the Docker host that connects over 127.0.0.1 reaches pixa from the Docker network's gateway, so that advice made pixa count every user as one client for the login limit. The login-limit paragraph, the trusted_proxies entry and config.example.yml now say to use the address pixa sees for requests through the proxy, that a proxy connecting through another host address is seen with that address, and how to read it from the request log. The signature section now says sig is base64url with the = padding kept, since pixa compares it exactly, and shows the example's sig for a stated key, computed with pixa's signer. Model: opus-5-5 --- README.md | 53 ++++++++++++++++++++++++++++++++-------------- TODO.md | 8 +++++++ config.example.yml | 8 ++++++- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ec673d1..4102390 100644 --- a/README.md +++ b/README.md @@ -112,13 +112,19 @@ copy is fresh. The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is refused with 429 and a `Retry-After` header. Behind a reverse proxy the client -address comes from `X-Forwarded-For` only when the proxy's address is in -`trusted_proxies`; otherwise all users behind the proxy are counted as one -client. With the default `trusted_proxies` (the RFC 1918 ranges), a client -with a private address can choose the address it is counted by through its own -`X-Forwarded-For`, whether it connects directly or through the proxy, because -its own address is trusted too. Setting `trusted_proxies` to the proxy's own -address closes this. +address comes from `X-Forwarded-For` only when the address pixa sees for +requests that come through the proxy is in `trusted_proxies`; otherwise all +users behind the proxy are counted as one client. That address is not always +the proxy's own: a proxy on the Docker host that connects to pixa over +`127.0.0.1` is seen as the gateway of the container's Docker network, such as +`172.17.0.1` on the default bridge, and one that connects through another of the +host's addresses is seen with that address. To be sure, read it as `remoteIP` in +pixa's request log while it is not in `trusted_proxies` (see `trusted_proxies` +under Configuration). With the default `trusted_proxies` (the RFC 1918 ranges), +a client with a private address can choose the address it is counted by through +its own `X-Forwarded-For`, whether it connects directly or through the proxy, +because its own address is trusted too. Setting `trusted_proxies` to only the +address pixa sees for requests that come through the proxy closes this. ### Image Metadata @@ -172,19 +178,27 @@ Where: outside), or `cover` when the URL has no `fit`; a request whose `fit` is anything else, an empty `fit=` included, is refused with 400 -**Example:** resize `https://cdn.example.com/photos/cat.jpg` to 800x600 -WebP with expiration 1704067200, default quality and fit: +The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet +of RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa +compares it exactly, so a signature encoded without padding, as Node's +`base64url` and Go's `base64.RawURLEncoding` do, is refused with 401. + +**Example:** with the signing key `example-signing-key-for-documentation`, +resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP with +expiration 1704067200, default quality and fit: 1. Build input: `cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover` -2. Compute HMAC-SHA256 with your secret key -3. Base64URL-encode the result +2. Compute HMAC-SHA256 of it with the signing key +3. Base64URL-encode the result, keeping the `=` padding: + `-ay7KHpfqmtIGbibDGbUuBDkymi-Ymdn0NkC6j5EJag=` 4. URL: - `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=&exp=1704067200` + `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=-ay7KHpfqmtIGbibDGbUuBDkymi-Ymdn0NkC6j5EJag=&exp=1704067200` For the same image at quality 40 with fit `contain`, the input ends in -`:40:contain` and the URL is -`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=&exp=1704067200&q=40&fit=contain`. +`:40:contain`, the signature is `5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=`, +and the URL is +`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`. **Allowlist patterns:** @@ -246,8 +260,15 @@ Key settings in more detail: `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a private network; an explicitly empty list (`[]`) trusts no one, and an explicit list replaces the default. An invalid CIDR aborts - startup. Set this to your proxy's address range if it is not already - covered by the defaults + startup. Set this to the address pixa sees for requests that come through + your proxy, such as `172.17.0.1/32`, when the defaults do not cover it, or + to trust nothing else (see the login limit under Routes). For a proxy on + the Docker host that connects to pixa over `127.0.0.1`, that address is the + gateway of the container's Docker network (`172.17.0.1` on the default + bridge), not the proxy's own address; a proxy that connects through another of + the host's addresses is seen with that address. To be sure which address it + is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request + through the proxy, and read `remoteIP` in pixa's request log line for it - `upstream_fetch_timeout` — timeout for origin requests - `upstream_max_response_size` — max origin response size - `downstream_timeout` — client response timeout diff --git a/TODO.md b/TODO.md index 622d81f..5c0ff89 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ exhaustion # Completed Steps +- 2026-09-29 `trusted_proxies` advice and signature padding in `README.md` + (closes #150): the login-limit paragraph, the `trusted_proxies` entry and + `config.example.yml` say to set `trusted_proxies` to the address pixa sees for + requests that come through the proxy, which the request log shows as + `remoteIP` while it is not trusted; for a proxy on the Docker host that + connects over `127.0.0.1` that is the Docker network's gateway, not the + proxy's own address; the signature section says `sig` is base64url with the + `=` padding kept, and gives the example's `sig` for a stated signing key. - 2026-09-29 fixed uid and gid for `pixad` (closes #151): the image creates the `pixad` group with gid 65532 and the `pixad` user with uid 65532, instead of the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad` diff --git a/config.example.yml b/config.example.yml index 8df6a35..f92b752 100644 --- a/config.example.yml +++ b/config.example.yml @@ -50,7 +50,13 @@ allowlist_hosts: # 172.16.0.0/12, 192.168.0.0/16), since pixa is deployed behind a proxy on # a private network. An explicitly empty list ([]) trusts no one; an # explicit list replaces the default. An invalid CIDR aborts startup. -# Uncomment to override the defaults with your proxy's address range. +# Uncomment to override the defaults with the address pixa sees for +# requests that come through your proxy. That is not always the proxy's own +# address: a proxy on the Docker host that connects over 127.0.0.1 is seen +# as the gateway of the container's Docker network (172.17.0.1 on the +# default bridge), and one that connects through another host address is +# seen with that address. To be sure, look it up in the request log as the +# trusted_proxies entry in README.md describes. # trusted_proxies: # - 10.0.0.0/8 # - 2001:db8::/32 From 46a55ec15da19e8939e4bb20d0bf732d25cfff01 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 07:42:05 +0200 Subject: [PATCH 005/131] Move migrations to internal/db/migrations (closes #96) REPO_POLICIES.md puts migrations in internal/db/migrations/ as 000_migration.sql and 001_schema.sql. The two files move there with their contents unchanged. go:embed cannot reach outside its own package, so internal/db/migrations has a small package that embeds them, and internal/database reads them through its FS(). The database package stays where CONVENTIONS.md puts it; moving it would change existing test files in other packages. The version still comes from the filename prefix, so a database that has recorded versions 0 and 1 runs neither again. A new test applies the migrations twice to one database file and checks that the second run applies nothing. Model: opus-5-5 --- TODO.md | 7 +++ internal/database/database.go | 32 ++++++----- internal/database/migrations_internal_test.go | 54 +++++++++++++++++++ .../migrations/000_migration.sql} | 0 .../migrations/001_schema.sql} | 0 internal/db/migrations/migrations.go | 15 ++++++ 6 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 internal/database/migrations_internal_test.go rename internal/{database/schema/000.sql => db/migrations/000_migration.sql} (100%) rename internal/{database/schema/001_initial_schema.sql => db/migrations/001_schema.sql} (100%) create mode 100644 internal/db/migrations/migrations.go diff --git a/TODO.md b/TODO.md index 5c0ff89..8f07fa8 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,13 @@ exhaustion # Completed Steps +- 2026-09-29 migrations at the path `REPO_POLICIES.md` sets (closes #96): the + migration files moved, contents unchanged, from `internal/database/schema/` + to `internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the + `internal/db/migrations` package embeds them and `internal/database` reads + them through its `FS()`; the `internal/database` package itself stays; the + version still comes from the filename prefix, so a database that has recorded + versions 0 and 1 runs neither again. - 2026-09-29 `trusted_proxies` advice and signature padding in `README.md` (closes #150): the login-limit paragraph, the `trusted_proxies` entry and `config.example.yml` say to set `trusted_proxies` to the address pixa sees for diff --git a/internal/database/database.go b/internal/database/database.go index f29560c..ab59f75 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -4,9 +4,9 @@ package database import ( "context" "database/sql" - "embed" "errors" "fmt" + "io/fs" "log/slog" "path/filepath" "sort" @@ -15,14 +15,12 @@ import ( "go.uber.org/fx" "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/db/migrations" "sneak.berlin/go/pixa/internal/logger" _ "modernc.org/sqlite" // SQLite driver registration ) -//go:embed schema/*.sql -var schemaFS embed.FS - // bootstrapVersion is the migration that creates the schema_migrations // table itself. It is applied before the normal migration loop. const bootstrapVersion = 0 @@ -113,29 +111,29 @@ func New(lc fx.Lifecycle, params Params) (*Database, error) { return s, nil } -// collectMigrations reads the embedded schema directory and returns +// collectMigrations reads the embedded migrations directory and returns // migration filenames sorted lexicographically. func collectMigrations() ([]string, error) { - entries, err := schemaFS.ReadDir("schema") + entries, err := fs.ReadDir(migrations.FS(), ".") if err != nil { - return nil, fmt.Errorf("failed to read schema directory: %w", err) + return nil, fmt.Errorf("failed to read migrations directory: %w", err) } - var migrations []string + var filenames []string for _, entry := range entries { if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") { - migrations = append(migrations, entry.Name()) + filenames = append(filenames, entry.Name()) } } - sort.Strings(migrations) + sort.Strings(filenames) - return migrations, nil + return filenames, nil } // bootstrapMigrationsTable ensures the schema_migrations table exists -// by applying 000.sql if the table is missing. +// by applying 000_migration.sql if the table is missing. func bootstrapMigrationsTable(ctx context.Context, db *sql.DB, log *slog.Logger) error { var tableExists int @@ -150,9 +148,9 @@ func bootstrapMigrationsTable(ctx context.Context, db *sql.DB, log *slog.Logger) return nil } - content, err := schemaFS.ReadFile("schema/000.sql") + content, err := fs.ReadFile(migrations.FS(), "000_migration.sql") if err != nil { - return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err) + return fmt.Errorf("failed to read bootstrap migration 000_migration.sql: %w", err) } if log != nil { @@ -177,12 +175,12 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error { return err } - migrations, err := collectMigrations() + filenames, err := collectMigrations() if err != nil { return err } - for _, migration := range migrations { + for _, migration := range filenames { version, parseErr := ParseMigrationVersion(migration) if parseErr != nil { return parseErr @@ -208,7 +206,7 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error { } // Read and apply migration. - content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration)) + content, readErr := fs.ReadFile(migrations.FS(), migration) if readErr != nil { return fmt.Errorf("failed to read migration %s: %w", migration, readErr) } diff --git a/internal/database/migrations_internal_test.go b/internal/database/migrations_internal_test.go new file mode 100644 index 0000000..6fb7282 --- /dev/null +++ b/internal/database/migrations_internal_test.go @@ -0,0 +1,54 @@ +package database + +import ( + "bytes" + "database/sql" + "log/slog" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" // SQLite driver registration +) + +// TestApplyMigrations_SecondRunAppliesNothing applies the migrations twice +// to one database file, as happens when pixad starts again on the database +// it created, and checks that the second run applies none of them. +// ApplyMigrations logs a message starting with "applying" before it runs +// any migration, the bootstrap one included. +func TestApplyMigrations_SecondRunAppliesNothing(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "state.sqlite3")) + if err != nil { + t.Fatalf("failed to open test db: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + var firstLog bytes.Buffer + + err = ApplyMigrations(ctx, db, slog.New(slog.NewTextHandler(&firstLog, nil))) + if err != nil { + t.Fatalf("first ApplyMigrations failed: %v", err) + } + + if !strings.Contains(firstLog.String(), "applying") { + t.Fatalf("first ApplyMigrations logged no applied migration:\n%s", + firstLog.String()) + } + + var secondLog bytes.Buffer + + err = ApplyMigrations(ctx, db, slog.New(slog.NewTextHandler(&secondLog, nil))) + if err != nil { + t.Fatalf("second ApplyMigrations failed: %v", err) + } + + if strings.Contains(secondLog.String(), "applying") { + t.Errorf("second ApplyMigrations ran a migration again:\n%s", + secondLog.String()) + } +} diff --git a/internal/database/schema/000.sql b/internal/db/migrations/000_migration.sql similarity index 100% rename from internal/database/schema/000.sql rename to internal/db/migrations/000_migration.sql diff --git a/internal/database/schema/001_initial_schema.sql b/internal/db/migrations/001_schema.sql similarity index 100% rename from internal/database/schema/001_initial_schema.sql rename to internal/db/migrations/001_schema.sql diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go new file mode 100644 index 0000000..62402ad --- /dev/null +++ b/internal/db/migrations/migrations.go @@ -0,0 +1,15 @@ +// Package migrations provides the embedded SQL migration files. +package migrations + +import ( + "embed" + "io/fs" +) + +//go:embed *.sql +var files embed.FS + +// FS returns the embedded filesystem containing the migration files. +func FS() fs.FS { + return files +} From 56217cbf4ab25350c63f81401153dbf936601626 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 08:49:18 +0200 Subject: [PATCH 006/131] Add the four settings the README documented but pixa lacked (closes #61) README.md documented access_control_allow_origin, upstream_fetch_timeout, upstream_max_response_size and downstream_timeout, but pixa did not know them, so a config following the README aborted startup. Each is now a setting with its PIXA_ variable, defaulting to the value that was fixed in the code: *, 30s, 50 MiB and 60s. Durations are Go duration strings and must be positive; the size is whole bytes, at most 1 GiB. The origin is * or one http or https origin written exactly as a browser sends it; anything else aborts startup. downstream_timeout sets both the server's write timeout and the per-request timeout. The owner approved the edits to existing tests. Model: opus-5-5 --- README.md | 24 +- TODO.md | 12 +- config.example.yml | 25 ++ internal/config/config.go | 176 ++++++++++++- .../config/config_validation_internal_test.go | 242 ++++++++++++++++++ internal/config/env_internal_test.go | 33 +++ internal/handlers/handlers.go | 2 + internal/middleware/middleware.go | 2 +- .../middleware/middleware_internal_test.go | 47 ++++ internal/server/http.go | 3 +- internal/server/http_internal_test.go | 10 +- .../server/login_rate_limit_internal_test.go | 11 +- internal/server/routes.go | 2 +- 13 files changed, 571 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4102390..7a3a013 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,10 @@ variables set by the file's `env:` section are checked the same way. | `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 | | `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` | | `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` | +| `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` | +| `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB | +| `PIXA_DOWNSTREAM_TIMEOUT` | `downstream_timeout` | Time allowed for answering one client request; default `60s` | +| `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN` | `access_control_allow_origin` | CORS origin allowed to read responses: `*` or one origin; default `*` | | `PIXA_METRICS_USERNAME` | `metrics.username` | Username for `/metrics`, which is served only when both are set | | `PIXA_METRICS_PASSWORD` | `metrics.password` | Password for `/metrics`; set together with the username | | `PIXA_SENTRY_DSN` | `sentry_dsn` | Sentry DSN for error reporting; empty disables it | @@ -243,7 +247,14 @@ variables set by the file's `env:` section are checked the same way. Key settings in more detail: -- `access_control_allow_origin` — CORS origin +- `access_control_allow_origin` — the origin a browser lets read pixa's + responses, sent as the CORS `Access-Control-Allow-Origin` header: `*`, the + default, is any site; otherwise one `http` or `https` origin such as + `https://example.com`, whose host is a lowercase host name (letters, + digits, hyphens and dots, with a letter in its last part) or an IP address + (IPv6 in brackets, in its shortest form), with an optional port 1-65535 + that has no leading zero and is not the scheme's default. Any other value, + including another scheme such as a browser extension's, aborts startup - `allowlist_hosts` — list of allowed upstream hosts - `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added to the always-enforced built-in ranges (loopback, private, @@ -269,9 +280,14 @@ Key settings in more detail: the host's addresses is seen with that address. To be sure which address it is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request through the proxy, and read `remoteIP` in pixa's request log line for it -- `upstream_fetch_timeout` — timeout for origin requests -- `upstream_max_response_size` — max origin response size -- `downstream_timeout` — client response timeout +- `upstream_fetch_timeout` — time allowed for one fetch from an upstream + host, as a duration such as `30s` (the default) or `2m` +- `upstream_max_response_size` — largest upstream response accepted, in + bytes; default `52428800` (50 MiB). It also limits the image data pixa + decodes +- `downstream_timeout` — time allowed for answering one client request, as a + duration; default `60s`. The upstream fetch counts toward it, so keep it + longer than `upstream_fetch_timeout` - `signing_key` — HMAC secret for URL signatures - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk cache entirely; omitted defaults to 75% of the free space on diff --git a/TODO.md b/TODO.md index 8f07fa8..e6ad439 100644 --- a/TODO.md +++ b/TODO.md @@ -57,6 +57,17 @@ exhaustion that is sooner, never negative; an allowlisted host's URL that has an `exp` follows it too; `immutable` stays, as freshness now ends at the expiry; documented in `README.md`. +- 2026-09-28 add the four settings `README.md` documented but pixa did not + have, which aborted startup as unknown keys (closes #61): + `access_control_allow_origin` (default `*`, the CORS origin), + `upstream_fetch_timeout` (default `30s`), `upstream_max_response_size` + (default 50 MiB) and `downstream_timeout` (default `60s`, both the + server's write timeout and the per-request timeout); each has a + `PIXA_` variable; durations are positive Go duration strings, the size a + whole number of bytes up to 1 GiB, the origin `*` or one `http` or + `https` origin as `README.md` describes it; an invalid value + aborts startup naming the key and the value; documented in + `config.example.yml` and `README.md`. - 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` counts the cached source images and processed variants (`source_content` plus `variant_content`) and takes their size from `Cache.UsageBytes`, @@ -306,7 +317,6 @@ exhaustion - X-Request-ID propagation - P2: auto format selection (format=auto based on Accept header) - P2: configuration - - add all configuration options from README - YAML config file support - P2: operational - optional Sentry error reporting diff --git a/config.example.yml b/config.example.yml index f92b752..27c2422 100644 --- a/config.example.yml +++ b/config.example.yml @@ -8,6 +8,10 @@ # this file's env: section is set while the file loads, so it overrides # both the environment the process was started with and this file's own # key. +# +# Durations are Go duration strings such as 30s or 2m and must be +# positive; a bare number has no unit and aborts startup. Sizes are a +# whole number of bytes. # Server settings port: 8080 @@ -67,6 +71,27 @@ allow_http: false # Maximum concurrent connections per upstream host (default: 20) upstream_connections_per_host: 20 +# Time allowed for one fetch from an upstream host (default: 30s) +upstream_fetch_timeout: 30s + +# Largest upstream response accepted, in bytes, at most 1073741824 +# (1 GiB) (default: 52428800, 50 MiB) +upstream_max_response_size: 52428800 + +# Time allowed for answering one client request, the upstream fetch +# included, so keep it longer than upstream_fetch_timeout (default: 60s) +downstream_timeout: 60s + +# The origin a browser lets read pixa's responses, sent as the CORS +# Access-Control-Allow-Origin header: "*" (the default) is any site; +# otherwise one http or https origin such as https://example.com, whose +# host is a lowercase host name (letters, digits, hyphens and dots, with a +# letter in its last part) or an IP address (IPv6 in brackets, in its +# shortest form), with an optional port 1-65535 that has no leading zero +# and is not the scheme's default. Any other value, including another +# scheme such as a browser extension's, aborts startup. +access_control_allow_origin: "*" + # Maximum disk cache size in bytes. Explicit values are used exactly as # given; 0 disables the disk cache entirely (every request fetches and # processes uncached). When omitted, the default is 75% of the free diff --git a/internal/config/config.go b/internal/config/config.go index 3d948be..5c9504f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -13,6 +13,7 @@ import ( "sort" "strconv" "strings" + "time" "git.eeqj.de/sneak/smartconfig" "go.uber.org/fx" @@ -24,6 +25,10 @@ const ( DefaultPort = 8080 DefaultStateDir = "/var/lib/pixa" DefaultUpstreamConnectionsPerHost = 20 + DefaultAccessControlAllowOrigin = "*" + DefaultUpstreamFetchTimeout = 30 * time.Second + DefaultUpstreamMaxResponseSize = 50 << 20 // 50 MiB + DefaultDownstreamTimeout = 60 * time.Second ) // Configuration key names. @@ -44,6 +49,10 @@ const ( keyCacheMaxBytes = "cache_max_bytes" keyBlockedNetworks = "blocked_networks" keyTrustedProxies = "trusted_proxies" + keyAccessControlAllowOrigin = "access_control_allow_origin" + keyUpstreamFetchTimeout = "upstream_fetch_timeout" + keyUpstreamMaxResponseSize = "upstream_max_response_size" + keyDownstreamTimeout = "downstream_timeout" ) // placeholderSigningKey is the dummy signing_key shipped in @@ -69,6 +78,7 @@ var ( errEmptyEntry = errors.New("contains an empty entry") errNotAValidURL = errors.New("not a valid URL") errPortOutOfRange = errors.New("outside the valid port range") + errSizeOutOfRange = errors.New("outside the accepted range") errTooFewConnections = errors.New("must be at least 1") errValueTooShort = errors.New("value too short") errPlaceholderKey = errors.New( @@ -86,6 +96,10 @@ var ( errNotBareHostname = errors.New( "must be a bare hostname without scheme, path, or whitespace") errNoHostnameLabels = errors.New("contains no hostname labels") + errNotADuration = errors.New("not a duration such as 30s or 2m") + errMustBePositive = errors.New("must be positive") + errNotAnOrigin = errors.New( + `not "*" or an origin such as https://example.com`) ) // Params defines dependencies for Config. @@ -112,6 +126,23 @@ type Config struct { AllowHTTP bool // Allow non-TLS upstream (testing only) UpstreamConnectionsPerHost int // Max concurrent connections per upstream host + // UpstreamFetchTimeout is the time allowed for one fetch from an + // upstream host. UpstreamMaxResponseSize is the largest upstream + // response accepted, in bytes, and also the image processor's input + // limit. + UpstreamFetchTimeout time.Duration + UpstreamMaxResponseSize int64 + + // AccessControlAllowOrigin is the origin the CORS middleware allows + // to read responses: "*" for any, or one origin such as + // https://example.com. + AccessControlAllowOrigin string + + // DownstreamTimeout bounds how long answering one client request may + // take. It is both the HTTP server's write timeout and the deadline + // of the per-request timeout middleware. + DownstreamTimeout time.Duration + // BlockedNetworks are operator-supplied CIDR ranges to refuse in // addition to the built-in SSRF blocklist. Enforced by the upstream // fetcher's dialer; the built-in ranges always apply. @@ -240,6 +271,14 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { AllowHTTP: loader.boolVal(keyAllowHTTP, false), UpstreamConnectionsPerHost: loader.intVal( keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost), + UpstreamFetchTimeout: loader.durationVal( + keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout), + UpstreamMaxResponseSize: loader.int64Val( + keyUpstreamMaxResponseSize, DefaultUpstreamMaxResponseSize), + AccessControlAllowOrigin: loader.stringVal( + keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin), + DownstreamTimeout: loader.durationVal( + keyDownstreamTimeout, DefaultDownstreamTimeout), CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0), BlockedNetworks: blockedNetworks, TrustedProxies: trustedProxies, @@ -354,7 +393,8 @@ func isKnownConfigKey(key string) bool { case keyDebug, keyMaintenanceMode, keyPort, keyStateDir, keySentryDSN, keyDBURL, keyMetrics, keySigningKey, keyAllowlistHosts, keyAllowHTTP, keyUpstreamConnectionsPerHost, keyCacheMaxBytes, keyBlockedNetworks, - keyTrustedProxies, "env": + keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout, + keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env": return true } @@ -382,6 +422,10 @@ func envVarNames() map[string]string { keyCacheMaxBytes: "PIXA_CACHE_MAX_BYTES", keyBlockedNetworks: "PIXA_BLOCKED_NETWORKS", keyTrustedProxies: "PIXA_TRUSTED_PROXIES", + keyAccessControlAllowOrigin: "PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", + keyUpstreamFetchTimeout: "PIXA_UPSTREAM_FETCH_TIMEOUT", + keyUpstreamMaxResponseSize: "PIXA_UPSTREAM_MAX_RESPONSE_SIZE", + keyDownstreamTimeout: "PIXA_DOWNSTREAM_TIMEOUT", } } @@ -535,6 +579,11 @@ func (c *Config) validate() error { settingName(keyCacheMaxBytes), c.CacheMaxBytes, errMustNotBeNegative) } + err = c.validateUpstreamMaxResponseSize() + if err != nil { + return err + } + for _, host := range c.AllowlistHosts { err := validateAllowlistHost(host) if err != nil { @@ -556,6 +605,82 @@ func (c *Config) validate() error { errMustBeSetTogether) } + return c.validateAccessControlAllowOrigin() +} + +// validateUpstreamMaxResponseSize checks that upstream_max_response_size +// is from 1 byte to 1 GiB. An upstream response is read whole into +// memory, and the image processor reads one byte past this limit, which +// must not overflow. +func (c *Config) validateUpstreamMaxResponseSize() error { + const maxUpstreamMaxResponseSize = 1 << 30 // 1 GiB + if c.UpstreamMaxResponseSize < 1 || + c.UpstreamMaxResponseSize > maxUpstreamMaxResponseSize { + return fmt.Errorf("%s: value %d is %w 1-%d", + settingName(keyUpstreamMaxResponseSize), c.UpstreamMaxResponseSize, + errSizeOutOfRange, maxUpstreamMaxResponseSize) + } + + return nil +} + +// validateAccessControlAllowOrigin accepts "*" or an origin exactly as a browser +// sends it: http or https, an IP address as netip writes it or a lowercase name +// with a letter in its last part, and an optional port 1-65535, not the default. +func (c *Config) validateAccessControlAllowOrigin() error { + origin := c.AccessControlAllowOrigin + if origin == "*" { + return nil + } + + errOrigin := fmt.Errorf("%s: value %q is %w", + settingName(keyAccessControlAllowOrigin), origin, errNotAnOrigin) + + parsed, err := url.Parse(origin) + if err != nil { + return errOrigin + } + + defaultPort := map[string]string{"http": "80", "https": "443"}[parsed.Scheme] + if defaultPort == "" { + return errOrigin + } + + const letters = "abcdefghijklmnopqrstuvwxyz" + + host := parsed.Hostname() + lastPart := host[strings.LastIndex(host, ".")+1:] + + addr, err := netip.ParseAddr(host) + + switch { + case err == nil && addr.Is6(): + host = "[" + addr.String() + "]" + case err == nil: + host = addr.String() + case strings.Trim(host, letters+"0123456789-.") != "": // a character other than these + return errOrigin + case !strings.ContainsAny(lastPart, letters): + return errOrigin + } + + // The value must be exactly the origin rebuilt from its parts. + rebuilt := parsed.Scheme + "://" + host + + port := parsed.Port() + if port != "" { + _, err := strconv.ParseUint(port, 10, 16) + if err != nil || port[0] == '0' || port == defaultPort { + return errOrigin + } + + rebuilt += ":" + port + } + + if rebuilt != origin { + return errOrigin + } + return nil } @@ -673,6 +798,19 @@ func (l *strictLoader) int64Val(key string, defaultVal int64) int64 { return val } +func (l *strictLoader) durationVal(key string, defaultVal time.Duration) time.Duration { + if l.err != nil { + return 0 + } + + val, err := getDuration(l.sc, key, defaultVal) + if err != nil { + l.err = err + } + + return val +} + func (l *strictLoader) boolVal(key string, defaultVal bool) bool { if l.err != nil { return false @@ -794,6 +932,42 @@ func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, erro } } +// getDuration returns the duration value for key, or defaultVal if the +// key is omitted. A present value must be a positive Go duration string +// such as "30s" or "2m", read with time.ParseDuration; a bare number has +// no unit and is an error, as is an explicit null. +func getDuration( + sc *smartconfig.Config, key string, defaultVal time.Duration, +) (time.Duration, error) { + raw, ok := lookupValue(sc, key) + if !ok { + return defaultVal, nil + } + + if raw == nil { + return 0, errNullConfigValue(key) + } + + str, ok := raw.(string) + if !ok { + return 0, fmt.Errorf("config key %q: value %v (%T) is %w", + key, raw, raw, errNotADuration) + } + + parsed, err := time.ParseDuration(strings.TrimSpace(str)) + if err != nil { + return 0, fmt.Errorf("%s: value %q is %w", + settingName(key), str, errNotADuration) + } + + if parsed <= 0 { + return 0, fmt.Errorf("%s: value %q %w", + settingName(key), str, errMustBePositive) + } + + return parsed, nil +} + // getBool returns the boolean value for key, or defaultVal if the key // is omitted. A present value that is not a boolean (or a ParseBool-able // string), or is explicitly null, is an error; numbers are not accepted diff --git a/internal/config/config_validation_internal_test.go b/internal/config/config_validation_internal_test.go index c12d27d..c269a38 100644 --- a/internal/config/config_validation_internal_test.go +++ b/internal/config/config_validation_internal_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" "git.eeqj.de/sneak/smartconfig" ) @@ -599,3 +600,244 @@ func TestEnsureStateDirFailsOnUncreatablePath(t *testing.T) { t.Errorf("error %q does not name the offending key state_dir", err.Error()) } } + +// TestOmittedOriginTimeoutsAndSizeUseDefaults checks that the CORS +// origin, the upstream fetch timeout, the upstream response size limit +// and the downstream timeout default to the values pixa used before they +// could be configured. +func TestOmittedOriginTimeoutsAndSizeUseDefaults(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine) + if err != nil { + t.Fatalf("minimal config should be valid, got error: %v", err) + } + + if c.AccessControlAllowOrigin != "*" { + t.Errorf("AccessControlAllowOrigin = %q, want *", c.AccessControlAllowOrigin) + } + + if c.UpstreamFetchTimeout != 30*time.Second { + t.Errorf("UpstreamFetchTimeout = %v, want 30s", c.UpstreamFetchTimeout) + } + + if c.UpstreamMaxResponseSize != 50<<20 { + t.Errorf("UpstreamMaxResponseSize = %d, want %d (50 MiB)", + c.UpstreamMaxResponseSize, 50<<20) + } + + if c.DownstreamTimeout != 60*time.Second { + t.Errorf("DownstreamTimeout = %v, want 60s", c.DownstreamTimeout) + } +} + +// TestExplicitOriginTimeoutsAndSizeAreUsed checks that valid values for +// the CORS origin, the two timeouts and the response size limit are used +// as given. The size is the largest accepted, 1 GiB. +func TestExplicitOriginTimeoutsAndSizeAreUsed(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+` +access_control_allow_origin: https://app.example.com +upstream_fetch_timeout: 10s +upstream_max_response_size: 1073741824 +downstream_timeout: 2m +`) + if err != nil { + t.Fatalf("valid config should load, got error: %v", err) + } + + if c.AccessControlAllowOrigin != "https://app.example.com" { + t.Errorf("AccessControlAllowOrigin = %q, want https://app.example.com", + c.AccessControlAllowOrigin) + } + + if c.UpstreamFetchTimeout != 10*time.Second { + t.Errorf("UpstreamFetchTimeout = %v, want 10s", c.UpstreamFetchTimeout) + } + + if c.UpstreamMaxResponseSize != 1073741824 { + t.Errorf("UpstreamMaxResponseSize = %d, want 1073741824", + c.UpstreamMaxResponseSize) + } + + if c.DownstreamTimeout != 2*time.Minute { + t.Errorf("DownstreamTimeout = %v, want 2m", c.DownstreamTimeout) + } +} + +// TestOriginWithPortOrAnyOriginIsAccepted checks the other accepted forms +// of access_control_allow_origin: "*", an origin with a port, and origins +// whose host is an IPv4 or IPv6 address. +func TestOriginWithPortOrAnyOriginIsAccepted(t *testing.T) { + t.Parallel() + + for _, origin := range []string{ + "*", "http://localhost:3000", "http://192.0.2.1", "http://[2001:db8::1]:8080", + } { + c, err := configFromYAML(t, signingKeyLine+ + "access_control_allow_origin: \""+origin+"\"\n") + if err != nil { + t.Fatalf("origin %q should be accepted, got error: %v", origin, err) + } + + if c.AccessControlAllowOrigin != origin { + t.Errorf("AccessControlAllowOrigin = %q, want %q", + c.AccessControlAllowOrigin, origin) + } + } +} + +// invalidTimeoutCases are configs where upstream_fetch_timeout or +// downstream_timeout is not a positive Go duration string; each must +// abort startup naming the key and the value. +func invalidTimeoutCases() []abortCase { + return []abortCase{ + { + name: "upstream_fetch_timeout not a duration", + yaml: signingKeyLine + "upstream_fetch_timeout: soon\n", + wantErrSubstrings: []string{keyUpstreamFetchTimeout, "soon"}, + }, + { + name: "upstream_fetch_timeout number without a unit", + yaml: signingKeyLine + "upstream_fetch_timeout: 45\n", + wantErrSubstrings: []string{keyUpstreamFetchTimeout, "45"}, + }, + { + name: "upstream_fetch_timeout zero", + yaml: signingKeyLine + "upstream_fetch_timeout: 0s\n", + wantErrSubstrings: []string{keyUpstreamFetchTimeout, "0s"}, + }, + { + name: "upstream_fetch_timeout negative", + yaml: signingKeyLine + "upstream_fetch_timeout: -5s\n", + wantErrSubstrings: []string{keyUpstreamFetchTimeout, "-5s"}, + }, + { + name: "upstream_fetch_timeout null", + yaml: signingKeyLine + "upstream_fetch_timeout: null\n", + wantErrSubstrings: []string{keyUpstreamFetchTimeout, nullValueText}, + }, + { + name: "downstream_timeout not a duration", + yaml: signingKeyLine + "downstream_timeout: 1 minute\n", + wantErrSubstrings: []string{keyDownstreamTimeout, "1 minute"}, + }, + { + name: "downstream_timeout zero", + yaml: signingKeyLine + "downstream_timeout: 0s\n", + wantErrSubstrings: []string{keyDownstreamTimeout, "0s"}, + }, + { + name: "downstream_timeout negative", + yaml: signingKeyLine + "downstream_timeout: -1m\n", + wantErrSubstrings: []string{keyDownstreamTimeout, "-1m"}, + }, + { + name: "downstream_timeout null", + yaml: signingKeyLine + "downstream_timeout:\n", + wantErrSubstrings: []string{keyDownstreamTimeout, nullValueText}, + }, + } +} + +// invalidSizeAndOriginCases are configs where upstream_max_response_size +// is not a whole number of bytes from 1 to 1 GiB, or +// access_control_allow_origin is neither "*" nor an origin; each must +// abort startup naming the key and the value. +func invalidSizeAndOriginCases() []abortCase { + badOrigins := []string{ + "", // empty + "example.com", // no scheme + "https://example.com/images", // a path + "https://example.com/", // a trailing slash + // The CORS middleware reads a * inside an origin as a pattern + // that lets other sites read responses. + "https://*", + "https://*.example.com", + "https://*example.com", + "https://a.com,b.com", // two hosts + "https://example.com:", // an empty port + "https://:8443", // no host + "https://example.com:0", // a port below 1 + "https://example.com:99999", // a port above 65535 + "https://exämple.com", // a host name that is not ASCII + "https://example.com:443", // the default port for https + "http://example.com:80", // the default port for http + "https://example.com:08080", // a port with a leading zero + "https://01.2.3.4", // an IPv4 address with a leading zero + "https://10.0.0", // an IPv4 address with three parts + "https://192.168.1.256", // an IPv4 address part above 255 + "https://example.123", // a host name whose last part is a number + "https://[0:0:0:0:0:0:0:1]", // an IPv6 address not in its shortest form + "file://example.com", // a scheme other than http or https + "HTTPS://example.com", // a scheme in upper case + "https://Example.com", // a host name in upper case + } + + cases := make([]abortCase, 0, len(badOrigins)) + for _, origin := range badOrigins { + cases = append(cases, abortCase{ + name: "access_control_allow_origin " + origin, + yaml: signingKeyLine + + "access_control_allow_origin: \"" + origin + "\"\n", + wantErrSubstrings: []string{keyAccessControlAllowOrigin, origin}, + }) + } + + return append(cases, []abortCase{ + { + name: "access_control_allow_origin null", + yaml: signingKeyLine + "access_control_allow_origin: null\n", + wantErrSubstrings: []string{keyAccessControlAllowOrigin, nullValueText}, + }, + { + name: "upstream_max_response_size with a unit", + yaml: signingKeyLine + "upstream_max_response_size: 50MB\n", + wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "50MB"}, + }, + { + name: "upstream_max_response_size fractional", + yaml: signingKeyLine + "upstream_max_response_size: 1.5\n", + wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "1.5"}, + }, + { + name: "upstream_max_response_size zero", + yaml: signingKeyLine + "upstream_max_response_size: 0\n", + wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "0"}, + }, + { + name: "upstream_max_response_size negative", + yaml: signingKeyLine + "upstream_max_response_size: -1\n", + wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "-1"}, + }, + { + name: "upstream_max_response_size null", + yaml: signingKeyLine + "upstream_max_response_size: null\n", + wantErrSubstrings: []string{keyUpstreamMaxResponseSize, nullValueText}, + }, + { + name: "upstream_max_response_size above 1 GiB", + yaml: signingKeyLine + "upstream_max_response_size: 1073741825\n", + wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "1073741825"}, + }, + { + name: "upstream_max_response_size largest 64-bit integer", + yaml: signingKeyLine + + "upstream_max_response_size: 9223372036854775807\n", + wantErrSubstrings: []string{ + keyUpstreamMaxResponseSize, "9223372036854775807", + }, + }, + }...) +} + +// TestInvalidOriginTimeoutOrSizeAbortsStartup verifies the +// no-silent-fallback rule for the CORS origin, the two timeouts and the +// response size limit: a value that does not parse or is out of range +// aborts startup naming the key and the value. +func TestInvalidOriginTimeoutOrSizeAbortsStartup(t *testing.T) { + t.Parallel() + + runAbortCases(t, append(invalidTimeoutCases(), invalidSizeAndOriginCases()...)) +} diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go index 57f1f15..cdc8790 100644 --- a/internal/config/env_internal_test.go +++ b/internal/config/env_internal_test.go @@ -8,6 +8,7 @@ import ( "slices" "strings" "testing" + "time" "sneak.berlin/go/pixa/internal/globals" "sneak.berlin/go/pixa/internal/logger" @@ -69,6 +70,10 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { t.Setenv("PIXA_CACHE_MAX_BYTES", "1024") t.Setenv("PIXA_BLOCKED_NETWORKS", "203.0.113.0/24") t.Setenv("PIXA_TRUSTED_PROXIES", "192.0.2.0/24") + t.Setenv("PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", "https://app.example.com") + t.Setenv("PIXA_UPSTREAM_FETCH_TIMEOUT", "10s") + t.Setenv("PIXA_UPSTREAM_MAX_RESPONSE_SIZE", "1048576") + t.Setenv("PIXA_DOWNSTREAM_TIMEOUT", "2m") c, err := newFromSmartConfig(nil) if err != nil { @@ -92,6 +97,10 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { cacheMaxBytesExplicit: true, BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, TrustedProxies: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}, + AccessControlAllowOrigin: "https://app.example.com", + UpstreamFetchTimeout: 10 * time.Second, + UpstreamMaxResponseSize: 1048576, + DownstreamTimeout: 2 * time.Minute, } if !reflect.DeepEqual(*c, want) { @@ -280,6 +289,30 @@ func TestInvalidDebugFromEnvironmentAbortsStartup(t *testing.T) { wantStartupError(t, err, "PIXA_DEBUG", "maybe") } +// TestInvalidOriginTimeoutOrSizeFromEnvironmentAbortsStartup checks that +// an invalid CORS origin, timeout or response size limit in its variable +// aborts startup naming the variable and the value. +func TestInvalidOriginTimeoutOrSizeFromEnvironmentAbortsStartup(t *testing.T) { + cases := []struct { + variable string + value string + }{ + {"PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", "example.com"}, + {"PIXA_UPSTREAM_FETCH_TIMEOUT", "soon"}, + {"PIXA_UPSTREAM_MAX_RESPONSE_SIZE", "50MB"}, + {"PIXA_DOWNSTREAM_TIMEOUT", "0s"}, + } + + for _, tc := range cases { + t.Run(tc.variable, func(t *testing.T) { + t.Setenv(tc.variable, tc.value) + + _, err := configFromYAML(t, signingKeyLine) + wantStartupError(t, err, tc.variable, tc.value) + }) + } +} + // TestConfigFileAloneBehavesAsBefore checks that with no variables set // (TestMain unsets them) the config file's values are used and omitted // keys take their defaults. diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 09f3ffc..3da580d 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -106,6 +106,8 @@ func (s *Handlers) initImageService() error { // Create the fetcher config fetcherCfg := httpfetcher.DefaultConfig() fetcherCfg.AllowHTTP = s.config.AllowHTTP + fetcherCfg.Timeout = s.config.UpstreamFetchTimeout + fetcherCfg.MaxResponseSize = s.config.UpstreamMaxResponseSize if s.config.UpstreamConnectionsPerHost > 0 { fetcherCfg.MaxConnectionsPerHost = s.config.UpstreamConnectionsPerHost diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index 30feaa5..492ba87 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -172,7 +172,7 @@ func (s *Middleware) Logging() func(http.Handler) http.Handler { // CORS returns a CORS middleware. func (s *Middleware) CORS() func(http.Handler) http.Handler { return cors.Handler(cors.Options{ - AllowedOrigins: []string{"*"}, + AllowedOrigins: []string{s.config.AccessControlAllowOrigin}, AllowedMethods: []string{"GET", "HEAD", "OPTIONS"}, AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"}, ExposedHeaders: []string{"Link"}, diff --git a/internal/middleware/middleware_internal_test.go b/internal/middleware/middleware_internal_test.go index 9992a60..5dac1be 100644 --- a/internal/middleware/middleware_internal_test.go +++ b/internal/middleware/middleware_internal_test.go @@ -9,6 +9,53 @@ import ( "sneak.berlin/go/pixa/internal/config" ) +// TestCORSAnswersWithConfiguredOrigin checks that the CORS middleware +// answers with access_control_allow_origin, where "*" lets any origin read +// responses and a single origin lets only that origin read them. +func TestCORSAnswersWithConfiguredOrigin(t *testing.T) { + t.Parallel() + + const appOrigin = "https://app.example.com" + + cases := []struct { + configured string + requestOrigin string + want string + }{ + {"*", "https://any.example.com", "*"}, + {appOrigin, appOrigin, appOrigin}, + {appOrigin, "https://other.example.com", ""}, + } + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + for _, tc := range cases { + mw := &Middleware{ + log: slog.Default(), + config: &config.Config{AccessControlAllowOrigin: tc.configured}, + } + + handler := mw.CORS()(testHandler) + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/v1/image/example.com/a.jpg/1x1.png", nil) + req.Header.Set("Origin", tc.requestOrigin) + + rec := httptest.NewRecorder() + + handler.ServeHTTP(rec, req) + + got := rec.Header().Get("Access-Control-Allow-Origin") + if got != tc.want { + t.Errorf("configured %q, request from %q: "+ + "Access-Control-Allow-Origin = %q, want %q", + tc.configured, tc.requestOrigin, got, tc.want) + } + } +} + func TestSecurityHeaders(t *testing.T) { t.Parallel() diff --git a/internal/server/http.go b/internal/server/http.go index bdd1238..13162e0 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -14,7 +14,6 @@ const ( // short, so a slowloris client dribbling headers is dropped well // before it ties up a connection for the whole ReadTimeout window. HTTPReadHeaderTimeout = 10 * time.Second - HTTPWriteTimeout = 60 * time.Second // HTTPIdleTimeout bounds how long an idle keep-alive connection is // held open, so idle connections cannot accumulate without limit on a // service targeting high concurrency. @@ -30,7 +29,7 @@ func (s *Server) newHTTPServer() *http.Server { Addr: fmt.Sprintf(":%d", s.config.Port), ReadTimeout: HTTPReadTimeout, ReadHeaderTimeout: HTTPReadHeaderTimeout, - WriteTimeout: HTTPWriteTimeout, + WriteTimeout: s.config.DownstreamTimeout, IdleTimeout: HTTPIdleTimeout, MaxHeaderBytes: HTTPMaxHeaderBytes, Handler: s, diff --git a/internal/server/http_internal_test.go b/internal/server/http_internal_test.go index d94a380..25b60fb 100644 --- a/internal/server/http_internal_test.go +++ b/internal/server/http_internal_test.go @@ -11,11 +11,15 @@ import ( // carries every hardening timeout wired onto it, including the slowloris // defense (ReadHeaderTimeout) and the keep-alive bound (IdleTimeout). This // guards against a field being defined but never set on the server, so -// each assertion compares the server field to its constant. +// each assertion compares the server field to its constant, or, for +// WriteTimeout, to downstream_timeout from the config. func TestNewHTTPServerTimeouts(t *testing.T) { t.Parallel() - s := &Server{config: &config.Config{Port: 8080}} + s := &Server{config: &config.Config{ + Port: 8080, + DownstreamTimeout: 45 * time.Second, + }} srv := s.newHTTPServer() @@ -26,7 +30,7 @@ func TestNewHTTPServerTimeouts(t *testing.T) { }{ {"ReadTimeout", srv.ReadTimeout, HTTPReadTimeout}, {"ReadHeaderTimeout", srv.ReadHeaderTimeout, HTTPReadHeaderTimeout}, - {"WriteTimeout", srv.WriteTimeout, HTTPWriteTimeout}, + {"WriteTimeout", srv.WriteTimeout, 45 * time.Second}, {"IdleTimeout", srv.IdleTimeout, HTTPIdleTimeout}, } diff --git a/internal/server/login_rate_limit_internal_test.go b/internal/server/login_rate_limit_internal_test.go index 13195c0..c37d86e 100644 --- a/internal/server/login_rate_limit_internal_test.go +++ b/internal/server/login_rate_limit_internal_test.go @@ -51,11 +51,12 @@ func newTestServer(t *testing.T) *Server { stateDir := t.TempDir() cfg := &config.Config{ - Debug: true, - SigningKey: testSigningKey, - StateDir: stateDir, - DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), - TrustedProxies: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, + Debug: true, + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + TrustedProxies: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, + DownstreamTimeout: config.DefaultDownstreamTimeout, } lc := fxtest.NewLifecycle(t) diff --git a/internal/server/routes.go b/internal/server/routes.go index 0bb2748..4623b75 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -33,7 +33,7 @@ func (s *Server) SetupRoutes() { } s.router.Use(s.mw.CORS()) - s.router.Use(middleware.Timeout(HTTPWriteTimeout)) + s.router.Use(middleware.Timeout(s.config.DownstreamTimeout)) if s.sentryEnabled { sentryHandler := sentryhttp.New(sentryhttp.Options{ From 0c99be939a1072602226e8932f6502558bf42ace Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 09:11:20 +0200 Subject: [PATCH 007/131] Install Dockerfile build dependencies through script/bootstrap (closes #95) The Dockerfile lint and build stages and Dockerfile.lint each carried their own apk add list, a copy of what script/bootstrap installs. They now copy script/, go.mod and go.sum and run script/bootstrap, so that layer is reused until one of those changes. script/bootstrap gains a C compiler check: the golang image has none, and cgo needs one. The build adds -trimpath and -s -w; CGO_ENABLED=1 stays, as govips links libvips. ARG VERSION moves to just above the build, so a new version reruns neither script/bootstrap nor the tests. Model: opus-5-5 --- Dockerfile | 33 ++++++++++++++++----------------- Dockerfile.lint | 14 +++++++------- TODO.md | 8 ++++++++ script/bootstrap | 11 +++++++++-- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 45027be..76d9bac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,14 @@ # golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint -RUN apk add --no-cache make build-base vips-dev libheif-dev pkgconfig - WORKDIR /src -# Copy go mod files first for better layer caching +# script/bootstrap installs the build dependencies and downloads the Go +# modules. Only script/, go.mod and go.sum are copied first, so this +# layer is reused until one of them changes. +COPY script/ ./script/ COPY go.mod go.sum ./ -RUN go mod download +RUN script/bootstrap # Copy source code COPY . . @@ -28,20 +29,12 @@ FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66 # Depend on lint stage passing COPY --from=lint /src/go.sum /dev/null -ARG VERSION=dev - -# Install build dependencies for CGO image libraries -RUN apk add --no-cache \ - build-base \ - vips-dev \ - libheif-dev \ - pkgconfig - WORKDIR /src -# Copy go mod files first for better layer caching +# Build dependencies and Go modules, as in the lint stage +COPY script/ ./script/ COPY go.mod go.sum ./ -RUN GOTOOLCHAIN=auto go mod download +RUN script/bootstrap # Copy source code COPY . . @@ -49,8 +42,14 @@ COPY . . # Run tests RUN make test -# Build with CGO enabled -RUN CGO_ENABLED=1 GOTOOLCHAIN=auto go build -ldflags "-X main.Version=${VERSION}" -o /pixad ./cmd/pixad +# VERSION is declared here, not earlier: a new value reruns only the +# build, not script/bootstrap or the tests. CGO stays enabled for +# govips; -trimpath keeps build paths out of the binary, and -s -w +# leave out the symbol table and debug information. +ARG VERSION=dev +RUN CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \ + -ldflags "-s -w -X main.Version=${VERSION}" \ + -o /pixad ./cmd/pixad # Runtime stage # alpine:3.21, 2026-02-25 diff --git a/Dockerfile.lint b/Dockerfile.lint index a07d8ca..41e350b 100644 --- a/Dockerfile.lint +++ b/Dockerfile.lint @@ -6,16 +6,16 @@ # golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 -# pixa is CGO/libvips: the type-aware linters compile every package, so -# this image needs the same C libraries the build does. -RUN apk add --no-cache build-base vips-dev libheif-dev pkgconfig - WORKDIR /src -# Modules first for layer caching; go.mod/go.sum settle this layer's -# result, so it may safely be reused between runs. +# pixa is CGO/libvips: the type-aware linters compile every package, so +# this image needs the same C libraries the build does. script/bootstrap +# installs them and downloads the Go modules. Only script/, go.mod and +# go.sum are copied first; they settle this layer's result, so it may +# safely be reused between runs. +COPY script/ ./script/ COPY go.mod go.sum ./ -RUN go mod download +RUN script/bootstrap COPY . . diff --git a/TODO.md b/TODO.md index e6ad439..c252259 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ exhaustion # Completed Steps +- 2026-09-29 Dockerfiles install through `script/bootstrap` (closes #95): the + `Dockerfile` lint and build stages and `Dockerfile.lint` copy `script/`, + `go.mod` and `go.sum`, then run `script/bootstrap` in place of their own + `apk add` lines, so the build dependencies are listed in one place; + `script/bootstrap` now also installs a C compiler when `gcc` is missing; the + build uses `-trimpath` and `-s -w` and keeps `CGO_ENABLED=1` for govips; + `ARG VERSION` sits just above the build, so a new version reruns neither + `script/bootstrap` nor the tests. - 2026-09-29 migrations at the path `REPO_POLICIES.md` sets (closes #96): the migration files moved, contents unchanged, from `internal/database/schema/` to `internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the diff --git a/script/bootstrap b/script/bootstrap index 83d2fc5..54c04ea 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -5,8 +5,10 @@ # or apk (detected in that order); assumes NOTHING is present (not git, # make, or go). The linter is never installed on the host: golangci-lint # runs only inside a container, Dockerfile.lint or the Dockerfile lint -# stage (see script/lint). CGO image libraries (pkg-config, vips, -# libheif) are installed for the govips bindings. +# stage (see script/lint). A C compiler and the CGO image libraries +# (pkg-config, vips, libheif) are installed for the govips bindings. +# Both Dockerfiles run this script too, so their build dependencies are +# the ones listed here. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -53,6 +55,11 @@ missing() { # CGO dependencies for govips (image processing) ensure_cgo_deps() { + # cgo compiles with gcc on Linux; build-base and build-essential + # also bring the C library headers. + if missing gcc; then + pkg_install gcc build-essential gcc build-base + fi if missing pkg-config; then pkg_install pkg-config pkg-config pkg-config pkgconfig fi From baf457d21e928596af08b804ed5f38eb95587dc8 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 10:32:16 +0200 Subject: [PATCH 008/131] Bound concurrent image processing and upstream fetches (closes #64) Nothing bounded total in-flight work, so a burst of cache misses across hosts could exhaust memory. Two settings now do: max_concurrent_processing (default the CPUs Go uses) and upstream_connections (default 64, beside the per-host limit). A request that finds either full waits up to 10 seconds, then gets 503; a slot is released on every path. No request holds source bytes while it waits: a cached source is read only after the processing slot is taken, and a fetched one only while it holds its upstream connection. libvips runs one worker thread per image with its operation cache off. Both waits count toward downstream_timeout, as the README says. Model: opus-5-5 --- README.md | 18 +- TODO.md | 14 +- config.example.yml | 19 +- .../concurrency_limits_internal_test.go | 162 ++++++++++ internal/config/config.go | 71 +++- internal/config/env_internal_test.go | 4 + internal/handlers/handlers.go | 12 +- internal/handlers/image.go | 9 + internal/handlers/imageenc.go | 5 + .../handlers/server_busy_internal_test.go | 49 +++ internal/httpfetcher/httpfetcher.go | 105 ++++-- .../max_connections_internal_test.go | 165 ++++++++++ internal/imageprocessor/imageprocessor.go | 79 ++++- ...max_concurrent_processing_internal_test.go | 302 ++++++++++++++++++ internal/imgcache/cache.go | 11 +- ...max_concurrent_processing_internal_test.go | 125 ++++++++ internal/imgcache/service.go | 81 +++-- 17 files changed, 1137 insertions(+), 94 deletions(-) create mode 100644 internal/config/concurrency_limits_internal_test.go create mode 100644 internal/handlers/server_busy_internal_test.go create mode 100644 internal/httpfetcher/max_connections_internal_test.go create mode 100644 internal/imageprocessor/max_concurrent_processing_internal_test.go create mode 100644 internal/imgcache/max_concurrent_processing_internal_test.go diff --git a/README.md b/README.md index 7a3a013..001d863 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,8 @@ variables set by the file's `env:` section are checked the same way. | `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 | | `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` | | `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` | +| `PIXA_UPSTREAM_CONNECTIONS` | `upstream_connections` | Concurrent connections to all upstream hosts together; default `64` | +| `PIXA_MAX_CONCURRENT_PROCESSING` | `max_concurrent_processing` | Images processed at once; default the number of CPUs | | `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` | | `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB | | `PIXA_DOWNSTREAM_TIMEOUT` | `downstream_timeout` | Time allowed for answering one client request; default `60s` | @@ -286,12 +288,24 @@ Key settings in more detail: bytes; default `52428800` (50 MiB). It also limits the image data pixa decodes - `downstream_timeout` — time allowed for answering one client request, as a - duration; default `60s`. The upstream fetch counts toward it, so keep it - longer than `upstream_fetch_timeout` + duration; default `60s`. The upstream fetch counts toward it, and so do the + waits for an upstream connection and for a processing slot (up to 10 seconds + each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds - `signing_key` — HMAC secret for URL signatures - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk cache entirely; omitted defaults to 75% of the free space on the filesystem containing `/cache/` (minimum 500 MiB) +- `upstream_connections` — the most connections to upstream hosts at once, all + hosts together, on top of `upstream_connections_per_host`; default `64`. A + fetch holds its connection until its image has been processed. A fetch that + finds all of them in use waits up to 10 seconds for one to free up; if none + does, and `downstream_timeout` has not ended first, the request is answered + 503 with the error `server busy, try again later` +- `max_concurrent_processing` — the most images decoded and encoded at once; + default the number of CPUs pixa can use (`GOMAXPROCS`), which follows a + container's CPU limit. A request that finds all of them in use waits up to 10 + seconds for one to free up; if none does, and `downstream_timeout` has not + ended first, it is answered 503 the same way See `config.example.yml` for all options with defaults. diff --git a/TODO.md b/TODO.md index c252259..8276df8 100644 --- a/TODO.md +++ b/TODO.md @@ -25,11 +25,20 @@ The disk cache is now size-bounded with LRU eviction # Next Step -P1: rate limit global concurrent upstream fetches to prevent resource -exhaustion +P2: security: referer blacklist # Completed Steps +- 2026-09-29 bound concurrent image processing and upstream fetches (closes + #64): `max_concurrent_processing` (default the number of CPUs pixa can use) + limits the images decoded and encoded at once, and `upstream_connections` + (default 64) the connections to all upstream hosts together, on top of + `upstream_connections_per_host`; a fetch holds its connection until its image + has been processed, and a request whose source is cached reads it only once it + has a processing slot; a request that finds either limit reached waits up to + 10 seconds for a free one, then gets 503 `server busy, try again later`; + libvips runs one worker thread per image with its operation cache off; + documented in `README.md` and `config.example.yml`. - 2026-09-29 Dockerfiles install through `script/bootstrap` (closes #95): the `Dockerfile` lint and build stages and `Dockerfile.lint` copy `script/`, `go.mod` and `go.sum`, then run `script/bootstrap` in place of their own @@ -316,7 +325,6 @@ exhaustion # Future Steps - P2: security - - referer blacklist - per-IP rate limiting on the image routes - per-origin rate limiting - P2: HTTP response handling diff --git a/config.example.yml b/config.example.yml index 27c2422..d64d77e 100644 --- a/config.example.yml +++ b/config.example.yml @@ -71,6 +71,19 @@ allow_http: false # Maximum concurrent connections per upstream host (default: 20) upstream_connections_per_host: 20 +# Maximum concurrent connections to all upstream hosts together, on top of +# the per-host limit (default: 64). A fetch holds its connection until its +# image has been processed. A fetch that finds none free waits up to 10 +# seconds for one, and if none frees up the request is answered 503, unless +# downstream_timeout has ended first. +upstream_connections: 64 + +# Maximum number of images decoded and encoded at once (default: the +# number of CPUs pixa can use, which follows a container's CPU limit). A +# request that finds none free waits up to 10 seconds for one, and if none +# frees up it is answered 503, unless downstream_timeout has ended first. +# max_concurrent_processing: 4 + # Time allowed for one fetch from an upstream host (default: 30s) upstream_fetch_timeout: 30s @@ -78,8 +91,10 @@ upstream_fetch_timeout: 30s # (1 GiB) (default: 52428800, 50 MiB) upstream_max_response_size: 52428800 -# Time allowed for answering one client request, the upstream fetch -# included, so keep it longer than upstream_fetch_timeout (default: 60s) +# Time allowed for answering one client request (default: 60s). The +# upstream fetch counts toward it, and so do the waits for an upstream +# connection and for a processing slot (up to 10 seconds each), so keep it +# longer than upstream_fetch_timeout plus 20 seconds. downstream_timeout: 60s # The origin a browser lets read pixa's responses, sent as the CORS diff --git a/internal/config/concurrency_limits_internal_test.go b/internal/config/concurrency_limits_internal_test.go new file mode 100644 index 0000000..c5f4b20 --- /dev/null +++ b/internal/config/concurrency_limits_internal_test.go @@ -0,0 +1,162 @@ +package config + +import ( + "runtime" + "testing" +) + +// The variables that set the two concurrency limits. +const ( + testMaxConcurrentProcessingVar = "PIXA_MAX_CONCURRENT_PROCESSING" + testUpstreamConnectionsVar = "PIXA_UPSTREAM_CONNECTIONS" +) + +// TestOmittedConcurrencyLimitsUseDefaults checks that an omitted +// max_concurrent_processing is the number of CPUs Go uses and an omitted +// upstream_connections is 64. +func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine) + if err != nil { + t.Fatalf("minimal config should be valid, got error: %v", err) + } + + if c.MaxConcurrentProcessing != runtime.GOMAXPROCS(0) { + t.Errorf("MaxConcurrentProcessing = %d, want %d, one per CPU", + c.MaxConcurrentProcessing, runtime.GOMAXPROCS(0)) + } + + if c.UpstreamConnections != 64 { + t.Errorf("UpstreamConnections = %d, want 64", c.UpstreamConnections) + } +} + +// TestExplicitConcurrencyLimitsAreUsed checks that valid values for the +// two limits are used as given. +func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+ + "max_concurrent_processing: 3\nupstream_connections: 10\n") + if err != nil { + t.Fatalf("valid config should load, got error: %v", err) + } + + if c.MaxConcurrentProcessing != 3 { + t.Errorf("MaxConcurrentProcessing = %d, want 3", c.MaxConcurrentProcessing) + } + + if c.UpstreamConnections != 10 { + t.Errorf("UpstreamConnections = %d, want 10", c.UpstreamConnections) + } +} + +// TestInvalidConcurrencyLimitAbortsStartup checks that a limit that is +// not a whole number of at least 1, or is null, aborts startup naming the +// key and the value, and the variable too where the value could have come +// from it. +func TestInvalidConcurrencyLimitAbortsStartup(t *testing.T) { + t.Parallel() + + processing := keyMaxConcurrentProcessing + connections := keyUpstreamConnections + + runAbortCases(t, []abortCase{ + { + name: "max_concurrent_processing zero", + yaml: signingKeyLine + processing + ": 0\n", + wantErrSubstrings: []string{ + processing, testMaxConcurrentProcessingVar, "value 0", + }, + }, + { + name: "max_concurrent_processing negative", + yaml: signingKeyLine + processing + ": -2\n", + wantErrSubstrings: []string{ + processing, testMaxConcurrentProcessingVar, "value -2", + }, + }, + { + name: "max_concurrent_processing not a number", + yaml: signingKeyLine + processing + ": lots\n", + wantErrSubstrings: []string{ + processing, testMaxConcurrentProcessingVar, "lots", + }, + }, + { + name: "max_concurrent_processing fractional", + yaml: signingKeyLine + processing + ": 1.5\n", + wantErrSubstrings: []string{processing, "1.5"}, + }, + { + name: "max_concurrent_processing null", + yaml: signingKeyLine + processing + ": null\n", + wantErrSubstrings: []string{processing, nullValueText}, + }, + { + name: "upstream_connections zero", + yaml: signingKeyLine + connections + ": 0\n", + wantErrSubstrings: []string{ + connections, testUpstreamConnectionsVar, "value 0", + }, + }, + { + name: "upstream_connections negative", + yaml: signingKeyLine + connections + ": -5\n", + wantErrSubstrings: []string{ + connections, testUpstreamConnectionsVar, "value -5", + }, + }, + { + name: "upstream_connections not a number", + yaml: signingKeyLine + connections + ": many\n", + wantErrSubstrings: []string{ + connections, testUpstreamConnectionsVar, "many", + }, + }, + { + name: "upstream_connections null", + yaml: signingKeyLine + connections + ": null\n", + wantErrSubstrings: []string{connections, nullValueText}, + }, + }) +} + +// TestConcurrencyLimitsFromEnvironment checks that the two variables set +// the limits over the config file, and that an invalid value in either +// aborts startup naming the variable and the value. +func TestConcurrencyLimitsFromEnvironment(t *testing.T) { + t.Setenv(testMaxConcurrentProcessingVar, "3") + t.Setenv(testUpstreamConnectionsVar, "10") + + c, err := configFromYAML(t, signingKeyLine+ + "max_concurrent_processing: 5\nupstream_connections: 50\n") + if err != nil { + t.Fatalf("limits from the environment should load: %v", err) + } + + if c.MaxConcurrentProcessing != 3 || c.UpstreamConnections != 10 { + t.Errorf("limits = %d and %d, want 3 and 10 from the environment", + c.MaxConcurrentProcessing, c.UpstreamConnections) + } + + cases := []struct { + variable string + value string + }{ + {testMaxConcurrentProcessingVar, "lots"}, + {testMaxConcurrentProcessingVar, "0"}, + {testUpstreamConnectionsVar, "-1"}, + {testUpstreamConnectionsVar, "ten"}, + } + + for _, tc := range cases { + t.Run(tc.variable+"="+tc.value, func(t *testing.T) { + t.Setenv(tc.variable, tc.value) + + _, err := configFromYAML(t, signingKeyLine) + wantStartupError(t, err, tc.variable, tc.value) + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 5c9504f..b8e041b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,7 @@ import ( "net/url" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -25,6 +26,7 @@ const ( DefaultPort = 8080 DefaultStateDir = "/var/lib/pixa" DefaultUpstreamConnectionsPerHost = 20 + DefaultUpstreamConnections = 64 DefaultAccessControlAllowOrigin = "*" DefaultUpstreamFetchTimeout = 30 * time.Second DefaultUpstreamMaxResponseSize = 50 << 20 // 50 MiB @@ -46,6 +48,8 @@ const ( keyAllowlistHosts = "allowlist_hosts" keyAllowHTTP = "allow_http" keyUpstreamConnectionsPerHost = "upstream_connections_per_host" + keyUpstreamConnections = "upstream_connections" + keyMaxConcurrentProcessing = "max_concurrent_processing" keyCacheMaxBytes = "cache_max_bytes" keyBlockedNetworks = "blocked_networks" keyTrustedProxies = "trusted_proxies" @@ -79,7 +83,7 @@ var ( errNotAValidURL = errors.New("not a valid URL") errPortOutOfRange = errors.New("outside the valid port range") errSizeOutOfRange = errors.New("outside the accepted range") - errTooFewConnections = errors.New("must be at least 1") + errMustBeAtLeastOne = errors.New("must be at least 1") errValueTooShort = errors.New("value too short") errPlaceholderKey = errors.New( "is the placeholder from config.example.yml; " + @@ -126,6 +130,12 @@ type Config struct { AllowHTTP bool // Allow non-TLS upstream (testing only) UpstreamConnectionsPerHost int // Max concurrent connections per upstream host + // UpstreamConnections is the most concurrent connections to all + // upstream hosts together, on top of the per-host limit. + // MaxConcurrentProcessing is the most images processed at once. + UpstreamConnections int + MaxConcurrentProcessing int + // UpstreamFetchTimeout is the time allowed for one fetch from an // upstream host. UpstreamMaxResponseSize is the largest upstream // response accepted, in bytes, and also the image processor's input @@ -226,14 +236,12 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) { // unparseable or invalid is an error: defaults apply only to omitted // keys, never to invalid explicit values. func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { - if sc != nil { - err := validateKnownKeys(sc) - if err != nil { - return nil, err - } + err := validateKnownKeys(sc) + if err != nil { + return nil, err } - err := validateAllowlistHostsValue(sc) + err = validateAllowlistHostsValue(sc) if err != nil { return nil, err } @@ -271,6 +279,12 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { AllowHTTP: loader.boolVal(keyAllowHTTP, false), UpstreamConnectionsPerHost: loader.intVal( keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost), + UpstreamConnections: loader.intVal( + keyUpstreamConnections, DefaultUpstreamConnections), + // Decoding and encoding are CPU-bound, so the default is one image + // per CPU Go uses, which follows a container's CPU limit. + MaxConcurrentProcessing: loader.intVal( + keyMaxConcurrentProcessing, runtime.GOMAXPROCS(0)), UpstreamFetchTimeout: loader.durationVal( keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout), UpstreamMaxResponseSize: loader.int64Val( @@ -321,8 +335,13 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { // being silently ignored, and rejects keys that are explicitly set to // null: a null is a SET value, never an omission, so it must not // silently take the default. The env section is permitted because -// smartconfig consumes it for environment variable injection. +// smartconfig consumes it for environment variable injection. A nil sc +// means no config file, which has no keys to check. func validateKnownKeys(sc *smartconfig.Config) error { + if sc == nil { + return nil + } + var unknown, nullKeys []string for key, value := range sc.Data() { @@ -392,7 +411,8 @@ func isKnownConfigKey(key string) bool { switch key { case keyDebug, keyMaintenanceMode, keyPort, keyStateDir, keySentryDSN, keyDBURL, keyMetrics, keySigningKey, keyAllowlistHosts, keyAllowHTTP, - keyUpstreamConnectionsPerHost, keyCacheMaxBytes, keyBlockedNetworks, + keyUpstreamConnectionsPerHost, keyUpstreamConnections, + keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks, keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout, keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env": return true @@ -419,6 +439,8 @@ func envVarNames() map[string]string { keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS", keyAllowHTTP: "PIXA_ALLOW_HTTP", keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST", + keyUpstreamConnections: "PIXA_UPSTREAM_CONNECTIONS", + keyMaxConcurrentProcessing: "PIXA_MAX_CONCURRENT_PROCESSING", keyCacheMaxBytes: "PIXA_CACHE_MAX_BYTES", keyBlockedNetworks: "PIXA_BLOCKED_NETWORKS", keyTrustedProxies: "PIXA_TRUSTED_PROXIES", @@ -562,10 +584,9 @@ func (c *Config) validate() error { settingName(keyPort), c.Port, errPortOutOfRange, maxPort) } - if c.UpstreamConnectionsPerHost < 1 { - return fmt.Errorf("%s: value %d %w", - settingName(keyUpstreamConnectionsPerHost), - c.UpstreamConnectionsPerHost, errTooFewConnections) + err = c.validateConcurrencyLimits() + if err != nil { + return err } if c.StateDir == "" { @@ -684,6 +705,30 @@ func (c *Config) validateAccessControlAllowOrigin() error { return nil } +// validateConcurrencyLimits checks that the two upstream connection limits +// and the image processing limit are at least 1. +func (c *Config) validateConcurrencyLimits() error { + if c.UpstreamConnectionsPerHost < 1 { + return fmt.Errorf("%s: value %d %w", + settingName(keyUpstreamConnectionsPerHost), + c.UpstreamConnectionsPerHost, errMustBeAtLeastOne) + } + + if c.UpstreamConnections < 1 { + return fmt.Errorf("%s: value %d %w", + settingName(keyUpstreamConnections), + c.UpstreamConnections, errMustBeAtLeastOne) + } + + if c.MaxConcurrentProcessing < 1 { + return fmt.Errorf("%s: value %d %w", + settingName(keyMaxConcurrentProcessing), + c.MaxConcurrentProcessing, errMustBeAtLeastOne) + } + + return nil +} + // validateAllowlistHost checks that an allowlist_hosts entry is a bare // hostname, optionally with a leading dot for suffix matching. URLs, // paths, and whitespace indicate a misconfigured entry. An entry with diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go index cdc8790..e61ffbf 100644 --- a/internal/config/env_internal_test.go +++ b/internal/config/env_internal_test.go @@ -67,6 +67,8 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com") t.Setenv("PIXA_ALLOW_HTTP", "true") t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5") + t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10") + t.Setenv("PIXA_MAX_CONCURRENT_PROCESSING", "3") t.Setenv("PIXA_CACHE_MAX_BYTES", "1024") t.Setenv("PIXA_BLOCKED_NETWORKS", "203.0.113.0/24") t.Setenv("PIXA_TRUSTED_PROXIES", "192.0.2.0/24") @@ -93,6 +95,8 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { AllowlistHosts: []string{testHostS3, ".example.com"}, AllowHTTP: true, UpstreamConnectionsPerHost: 5, + UpstreamConnections: 10, + MaxConcurrentProcessing: 3, CacheMaxBytes: 1024, cacheMaxBytesExplicit: true, BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 3da580d..b350595 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -113,15 +113,17 @@ func (s *Handlers) initImageService() error { fetcherCfg.MaxConnectionsPerHost = s.config.UpstreamConnectionsPerHost } + fetcherCfg.MaxConnections = s.config.UpstreamConnections fetcherCfg.BlockedNetworks = s.config.BlockedNetworks // Create the service svc, err := imgcache.NewService(&imgcache.ServiceConfig{ - Cache: cache, - FetcherConfig: fetcherCfg, - SigningKey: s.config.SigningKey, - Allowlist: s.config.AllowlistHosts, - Logger: s.log, + Cache: cache, + FetcherConfig: fetcherCfg, + SigningKey: s.config.SigningKey, + Allowlist: s.config.AllowlistHosts, + MaxConcurrentProcessing: s.config.MaxConcurrentProcessing, + Logger: s.log, }) if err != nil { return err diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 561ca1d..3c94bc8 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -12,6 +12,7 @@ import ( "github.com/go-chi/chi/v5" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -217,6 +218,14 @@ func (s *Handlers) respondImageError( return } + if errors.Is(err, httpfetcher.ErrTooManyConnections) || + errors.Is(err, imageprocessor.ErrTooManyImages) { + s.respondError(w, "server busy, try again later", + http.StatusServiceUnavailable) + + return + } + s.respondError(w, "internal error", http.StatusInternalServerError) } diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 5effd8b..56af31b 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -12,6 +12,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -124,6 +125,10 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { s.respondError(w, "upstream error", http.StatusBadGateway) case errors.Is(err, httpfetcher.ErrUpstreamTimeout): s.respondError(w, "upstream timeout", http.StatusGatewayTimeout) + case errors.Is(err, httpfetcher.ErrTooManyConnections), + errors.Is(err, imageprocessor.ErrTooManyImages): + s.respondError(w, "server busy, try again later", + http.StatusServiceUnavailable) default: s.log.Error("image request failed", "error", err) s.respondError(w, "internal error", http.StatusInternalServerError) diff --git a/internal/handlers/server_busy_internal_test.go b/internal/handlers/server_busy_internal_test.go new file mode 100644 index 0000000..2f3471c --- /dev/null +++ b/internal/handlers/server_busy_internal_test.go @@ -0,0 +1,49 @@ +package handlers + +import ( + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestServerBusyAnswers503 checks that both image routes answer 503 with +// a clear error when the image service gives up waiting for a free upstream +// connection or processing slot, wrapped as the service wraps them. +func TestServerBusyAnswers503(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.jpg"} + + for _, err := range []error{ + fmt.Errorf("upstream fetch failed: %w", httpfetcher.ErrTooManyConnections), + fmt.Errorf("image processing failed: %w", imageprocessor.ErrTooManyImages), + } { + plain := httptest.NewRecorder() + h.respondImageError(plain, req, err) + + encrypted := httptest.NewRecorder() + h.handleImageError(encrypted, err) + + for route, rec := range map[string]*httptest.ResponseRecorder{ + "/v1/image/": plain, "/v1/e/": encrypted, + } { + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("%s for %v: status = %d, want %d", + route, err, rec.Code, http.StatusServiceUnavailable) + } + + if !strings.Contains(rec.Body.String(), "server busy, try again later") { + t.Errorf("%s for %v: body = %q, want the server busy error", + route, err, rec.Body.String()) + } + } + } +} diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 64e27eb..d65f2da 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -1,5 +1,6 @@ // Package httpfetcher fetches content from upstream HTTP origins with SSRF -// protection, per-host connection limits, and content-type validation. +// protection, connection limits per host and for all hosts together, and +// content-type validation. package httpfetcher import ( @@ -28,8 +29,13 @@ const ( DefaultIdleConnTimeout = 90 * time.Second DefaultMaxRedirects = 10 DefaultMaxConnectionsPerHost = 20 + DefaultMaxConnections = 64 ) +// ConnectionWaitTimeout is how long Fetch waits for a free connection when +// MaxConnections fetches are already in progress. +const ConnectionWaitTimeout = 10 * time.Second + // MIME content types. const ( contentTypeJPEG = "image/jpeg" @@ -70,6 +76,7 @@ var ( ErrInvalidContentType = errors.New("invalid or unsupported content type") ErrUpstreamError = errors.New("upstream server error") ErrUpstreamTimeout = errors.New("upstream request timeout") + ErrTooManyConnections = errors.New("too many concurrent upstream connections") ) // Internal fetcher errors. @@ -122,6 +129,9 @@ type Config struct { AllowHTTP bool // MaxConnectionsPerHost limits concurrent connections to each upstream host. MaxConnectionsPerHost int + // MaxConnections limits concurrent connections to all upstream hosts + // together. + MaxConnections int // BlockedNetworks are operator-supplied CIDR ranges refused by the // dialer, in addition to the always-enforced built-in ranges. BlockedNetworks []netip.Prefix @@ -143,15 +153,22 @@ func DefaultConfig() *Config { }, AllowHTTP: false, MaxConnectionsPerHost: DefaultMaxConnectionsPerHost, + MaxConnections: DefaultMaxConnections, } } -// HTTPFetcher implements Fetcher with SSRF protection and per-host connection limits. +// HTTPFetcher implements Fetcher with SSRF protection and connection limits +// per host and for all hosts together. type HTTPFetcher struct { client *http.Client config *Config hostSems map[string]chan struct{} // per-host semaphores hostSemMu sync.Mutex // protects hostSems map + // allHostsSemaphore has one slot per connection allowed to all hosts + // together (config.MaxConnections). + allHostsSemaphore chan struct{} + // connectionWaitTimeout is ConnectionWaitTimeout; tests shorten it. + connectionWaitTimeout time.Duration } // New creates a new HTTPFetcher with SSRF protection. @@ -192,13 +209,18 @@ func New(config *Config) *HTTPFetcher { } return &HTTPFetcher{ - client: client, - config: config, - hostSems: make(map[string]chan struct{}), + client: client, + config: config, + hostSems: make(map[string]chan struct{}), + allHostsSemaphore: make(chan struct{}, config.MaxConnections), + connectionWaitTimeout: ConnectionWaitTimeout, } } -// Fetch retrieves content from the given URL with SSRF protection. +// Fetch retrieves content from the given URL with SSRF protection. When +// MaxConnections fetches are already in progress, it waits up to +// ConnectionWaitTimeout for one to finish, then fails with +// ErrTooManyConnections. func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, error) { // Validate URL before making request err := validateURL(ctx, url, f.config.AllowHTTP) @@ -206,24 +228,17 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro return nil, err } - // Extract host for rate limiting - host := extractHost(url) - - // Acquire semaphore slot for this host - sem := f.getHostSemaphore(host) - select { - case sem <- struct{}{}: - // Acquired slot - case <-ctx.Done(): - return nil, ctx.Err() + release, err := f.acquireConnection(ctx, extractHost(url)) + if err != nil { + return nil, err } - // If we fail before returning a result, release the slot + // If we fail before returning a result, release the connection success := false defer func() { if !success { - <-sem + release() } }() @@ -267,17 +282,52 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro return nil, fmt.Errorf("upstream request failed: %w", err) } - result, err := f.buildResult(resp, remoteAddr, fetchDuration, sem) + result, err := f.buildResult(resp, remoteAddr, fetchDuration, release) if err != nil { return nil, err } - // Mark success so defer doesn't release the semaphore + // Mark success so defer doesn't release the connection; closing the + // result's Content does success = true return result, nil } +// acquireConnection takes a slot for host, then one of the slots shared by +// all hosts, and returns the func that gives both back. The host's slot +// comes first, so fetches queued for one busy host hold no shared slot. +// Only the wait for a shared slot is bounded: after connectionWaitTimeout +// it fails with ErrTooManyConnections. +func (f *HTTPFetcher) acquireConnection( + ctx context.Context, host string, +) (func(), error) { + hostSem := f.getHostSemaphore(host) + + select { + case hostSem <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + select { + case f.allHostsSemaphore <- struct{}{}: + case <-time.After(f.connectionWaitTimeout): + <-hostSem + + return nil, ErrTooManyConnections + case <-ctx.Done(): + <-hostSem + + return nil, ctx.Err() + } + + return func() { + <-hostSem + <-f.allHostsSemaphore + }, nil +} + // getHostSemaphore returns the semaphore for a host, creating it if necessary. func (f *HTTPFetcher) getHostSemaphore(host string) chan struct{} { f.hostSemMu.Lock() @@ -293,12 +343,12 @@ func (f *HTTPFetcher) getHostSemaphore(host string) chan struct{} { } // buildResult validates the upstream response and assembles a FetchResult -// whose Content releases the host semaphore slot when closed. +// whose Content calls release when closed. func (f *HTTPFetcher) buildResult( resp *http.Response, remoteAddr string, fetchDuration time.Duration, - sem chan struct{}, + release func(), ) (*FetchResult, error) { // Extract HTTP version (strip "HTTP/" prefix) httpVersion := strings.TrimPrefix(resp.Proto, "HTTP/") @@ -333,7 +383,7 @@ func (f *HTTPFetcher) buildResult( } return &FetchResult{ - Content: &semaphoreReleasingReadCloser{limitedBody, resp.Body, sem}, + Content: &semaphoreReleasingReadCloser{limitedBody, resp.Body, release}, ContentLength: resp.ContentLength, ContentType: contentType, Headers: resp.Header, @@ -574,17 +624,18 @@ func (r *limitedReader) Read(p []byte) (int, error) { return n, err } -// semaphoreReleasingReadCloser releases a semaphore slot when closed. +// semaphoreReleasingReadCloser releases the fetch's connection slots when +// closed. type semaphoreReleasingReadCloser struct { *limitedReader - closer io.Closer - sem chan struct{} + closer io.Closer + release func() } func (r *semaphoreReleasingReadCloser) Close() error { err := r.closer.Close() - <-r.sem // Release semaphore slot + r.release() return err } diff --git a/internal/httpfetcher/max_connections_internal_test.go b/internal/httpfetcher/max_connections_internal_test.go new file mode 100644 index 0000000..d23816c --- /dev/null +++ b/internal/httpfetcher/max_connections_internal_test.go @@ -0,0 +1,165 @@ +package httpfetcher + +import ( + "context" + "errors" + "net" + "strconv" + "testing" + "time" +) + +// imageURLOnPort is the fake upstream's image route on testPublicHost at +// port. Each port is a different host to the per-host limit, while the test +// dialer sends every port to the one test server. +func imageURLOnPort(port int) string { + return "http://" + net.JoinHostPort(testPublicHost, strconv.Itoa(port)) + + "/image" +} + +func TestDefaultConfigMaxConnections(t *testing.T) { + t.Parallel() + + if got := DefaultConfig().MaxConnections; got != DefaultMaxConnections { + t.Errorf("MaxConnections = %d, want %d", got, DefaultMaxConnections) + } +} + +// TestFetchLimitsConnectionsToAllHostsTogether checks that MaxConnections +// counts the fetches to every host together, apart from the per-host +// limit: with MaxConnections at 2 and two responses open from two hosts, a +// fetch from a third host, which has nothing open, waits the whole wait +// timeout and fails with ErrTooManyConnections. Closing one response lets +// it through. +func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + + cfg := DefaultConfig() + cfg.MaxConnections = 2 + + f, _ := newServerFetcher(t, srv, cfg) + f.connectionWaitTimeout = 100 * time.Millisecond + + first, err := f.Fetch(testContext(t), imageURLOnPort(81)) + if err != nil { + t.Fatalf("first Fetch() error = %v", err) + } + + second, err := f.Fetch(testContext(t), imageURLOnPort(82)) + if err != nil { + t.Fatalf("second Fetch() error = %v", err) + } + + defer func() { _ = second.Content.Close() }() + + start := time.Now() + + _, err = f.Fetch(testContext(t), imageURLOnPort(83)) + if !errors.Is(err, ErrTooManyConnections) { + t.Fatalf("third Fetch() error = %v, want ErrTooManyConnections", err) + } + + if waited := time.Since(start); waited < f.connectionWaitTimeout { + t.Errorf("third Fetch() failed after %v, before waiting %v", + waited, f.connectionWaitTimeout) + } + + if held := semLen(f, testPublicHost+":83"); held != 0 { + t.Errorf("the refused fetch kept its host's slot: %d held", held) + } + + err = first.Content.Close() + if err != nil { + t.Fatalf("close first body: %v", err) + } + + third, err := f.Fetch(testContext(t), imageURLOnPort(83)) + if err != nil { + t.Fatalf("Fetch() after a response was closed: error = %v", err) + } + + _ = third.Content.Close() +} + +// TestFetchFreesHostSlotWhenContextEndsWaitingForConnection checks that a +// fetch whose request context ends while it waits for a connection shared +// by all hosts gives its host's slot back. With MaxConnections at 1 and one +// response open, a fetch from another host takes that host's slot and waits; +// its context ends long before the 10 second wait timeout. +func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + + cfg := DefaultConfig() + cfg.MaxConnections = 1 + + f, _ := newServerFetcher(t, srv, cfg) + + first, err := f.Fetch(testContext(t), imageURLOnPort(81)) + if err != nil { + t.Fatalf("first Fetch() error = %v", err) + } + + defer func() { _ = first.Content.Close() }() + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + _, err = f.Fetch(ctx, imageURLOnPort(82)) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("second Fetch() error = %v, want context.DeadlineExceeded", err) + } + + if held := semLen(f, testPublicHost+":82"); held != 0 { + t.Errorf("the fetch kept its host's slot after its context ended: "+ + "%d held", held) + } +} + +// TestFetchReleasesConnectionOnError checks that a fetch that fails after +// taking its connection gives it back: with MaxConnections at 1, the slot +// must be free after the failure and the next fetch must succeed. +func TestFetchReleasesConnectionOnError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + url string + want error + }{ + {"upstream answers 500", upstreamURL("/status/500"), ErrUpstreamError}, + {"upstream sends HTML", upstreamURL("/html"), ErrInvalidContentType}, + // 198.51.100.7 (TEST-NET-2) passes the SSRF checks, and the test + // dialer refuses every host but testPublicHost. + {"connecting fails", "http://198.51.100.7/image", errUnexpectedDial}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + + cfg := DefaultConfig() + cfg.MaxConnections = 1 + + f, _ := newServerFetcher(t, srv, cfg) + f.connectionWaitTimeout = 100 * time.Millisecond + + _, err := f.Fetch(testContext(t), tc.url) + if !errors.Is(err, tc.want) { + t.Fatalf("Fetch() error = %v, want %v", err, tc.want) + } + + if held := len(f.allHostsSemaphore); held != 0 { + t.Fatalf("connection still held after the error: %d held", held) + } + + res := fetchImage(t, f, "/image") + _ = res.Content.Close() + }) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index dd77ac1..6a9a92d 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "io" + "runtime" "sync" + "time" "github.com/davidbyttow/govips/v2/vips" ) @@ -17,11 +19,21 @@ import ( //nolint:gochecknoglobals // package-level sync.Once for one-time vips init var vipsOnce sync.Once -// initVips initializes libvips with quiet logging. +// initVips initializes libvips with quiet logging, one worker thread per +// image and no operation cache. Process already works on one image per CPU +// by default, so more threads per image would only compete for the CPUs. +// Each request decodes different source bytes, so the operation cache +// would rarely be hit and would hold memory outside MaxConcurrentProcessing; +// repeated requests are served from pixa's disk cache instead. func initVips() { vipsOnce.Do(func() { vips.LoggingSettings(nil, vips.LogLevelError) - vips.Startup(nil) + vips.Startup(&vips.Config{ + ConcurrencyLevel: 1, + MaxCacheSize: 0, + MaxCacheMem: 0, + MaxCacheFiles: 0, + }) }) } @@ -106,9 +118,23 @@ var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size") // not supported. var ErrUnsupportedOutputFormat = errors.New("unsupported output format") +// ErrTooManyImages is returned when MaxConcurrentProcessing images are being +// processed and none finishes within ProcessingWaitTimeout. +var ErrTooManyImages = errors.New("too many images being processed at once") + +// ProcessingWaitTimeout is how long Process waits for a free slot when +// MaxConcurrentProcessing images are already being processed. +const ProcessingWaitTimeout = 10 * time.Second + // ImageProcessor implements image transformation using libvips via govips. type ImageProcessor struct { maxInputBytes int64 + // processingSemaphore has one slot per image that may be processed at + // once. Process holds a slot from before it reads its input until it + // returns, so the input, the decoded image and the output all count. + processingSemaphore chan struct{} + // processingWaitTimeout is ProcessingWaitTimeout; tests shorten it. + processingWaitTimeout time.Duration } // Params holds configuration for creating an ImageProcessor. @@ -117,6 +143,9 @@ type Params struct { // MaxInputBytes is the maximum allowed input size in bytes. // If <= 0, DefaultMaxInputBytes is used. MaxInputBytes int64 + // MaxConcurrentProcessing is the most images processed at once. + // If <= 0, the number of CPUs Go uses (runtime.GOMAXPROCS(0)) is used. + MaxConcurrentProcessing int } // New creates a new image processor with the given parameters. @@ -129,17 +158,34 @@ func New(params Params) *ImageProcessor { maxInputBytes = DefaultMaxInputBytes } + maxConcurrentProcessing := params.MaxConcurrentProcessing + if maxConcurrentProcessing <= 0 { + maxConcurrentProcessing = runtime.GOMAXPROCS(0) + } + return &ImageProcessor{ - maxInputBytes: maxInputBytes, + maxInputBytes: maxInputBytes, + processingSemaphore: make(chan struct{}, maxConcurrentProcessing), + processingWaitTimeout: ProcessingWaitTimeout, } } -// Process transforms an image according to the request. +// Process transforms an image according to the request. When +// MaxConcurrentProcessing images are already being processed, it waits up +// to ProcessingWaitTimeout for one to finish, then fails with +// ErrTooManyImages. func (p *ImageProcessor) Process( - _ context.Context, + ctx context.Context, input io.Reader, req *Request, ) (*Result, error) { + release, err := p.acquireSlot(ctx) + if err != nil { + return nil, err + } + + defer release() + // Read input with a size limit to prevent unbounded memory consumption. // We read at most maxInputBytes+1 so we can detect if the input exceeds // the limit without consuming additional memory. @@ -285,6 +331,29 @@ func FormatToMIME(format Format) string { } } +// acquireSlot takes a slot in processingSemaphore, waiting at most +// processingWaitTimeout for one to free up, and returns the func that gives +// it back. A free slot is taken even when ctx has ended; only the wait for +// one stops when ctx ends, as the rest of Process does not check ctx. +func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) { + release := func() { <-p.processingSemaphore } + + select { + case p.processingSemaphore <- struct{}{}: + return release, nil + default: + } + + select { + case p.processingSemaphore <- struct{}{}: + return release, nil + case <-time.After(p.processingWaitTimeout): + return nil, ErrTooManyImages + case <-ctx.Done(): + return nil, ctx.Err() + } +} + // detectFormat returns the format string from a vips image. func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string { format := img.Format() diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go new file mode 100644 index 0000000..dc826c4 --- /dev/null +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -0,0 +1,302 @@ +package imageprocessor + +import ( + "bytes" + "context" + "errors" + "io" + "runtime" + "strings" + "sync" + "testing" + "testing/iotest" + "time" +) + +// errTestReadFailed is the error the unreadable test input returns. +var errTestReadFailed = errors.New("test input cannot be read") + +// readingCounter counts the Process calls reading their input at the same +// time and remembers the most there ever were. +type readingCounter struct { + mu sync.Mutex + reading int + most int +} + +func (c *readingCounter) start() { + c.mu.Lock() + defer c.mu.Unlock() + + c.reading++ + c.most = max(c.most, c.reading) +} + +func (c *readingCounter) stop() { + c.mu.Lock() + defer c.mu.Unlock() + + c.reading-- +} + +func (c *readingCounter) mostReading() int { + c.mu.Lock() + defer c.mu.Unlock() + + return c.most +} + +// gatedReader is a Process input. Its first Read counts the call in, +// reports it on entered and blocks until gate is closed; it counts the call +// out when it returns io.EOF. Process reads its input only while it holds a +// processing slot, so the count never goes above MaxConcurrentProcessing. +type gatedReader struct { + data *bytes.Reader + gate <-chan struct{} + entered chan<- struct{} + counter *readingCounter + started bool +} + +func (r *gatedReader) Read(p []byte) (int, error) { + if !r.started { + r.started = true + r.counter.start() + + r.entered <- struct{}{} + + <-r.gate + } + + n, err := r.data.Read(p) + if errors.Is(err, io.EOF) { + r.counter.stop() + } + + return n, err +} + +// smallJPEGRequest asks for a 5x5 JPEG. +func smallJPEGRequest() *Request { + return &Request{ + Size: Size{Width: 5, Height: 5}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } +} + +// processInBackground runs Process on reader in a new goroutine and sends +// its error on results. +func processInBackground( + proc *ImageProcessor, reader *gatedReader, results chan<- error, +) { + go func() { + result, err := proc.Process(context.Background(), reader, smallJPEGRequest()) + if err == nil { + _ = result.Content.Close() + } + + results <- err + }() +} + +// waitForEntries fails the test unless count Process calls report on +// entered within a few seconds. +func waitForEntries(t *testing.T, entered <-chan struct{}, count int) { + t.Helper() + + for range count { + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("Process calls did not start reading their input") + } + } +} + +func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) { + t.Parallel() + + for _, limit := range []int{0, -1} { + proc := New(Params{MaxConcurrentProcessing: limit}) + if got := cap(proc.processingSemaphore); got != runtime.GOMAXPROCS(0) { + t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, one per CPU", + limit, got, runtime.GOMAXPROCS(0)) + } + } + + proc := New(Params{MaxConcurrentProcessing: 3}) + if got := cap(proc.processingSemaphore); got != 3 { + t.Errorf("MaxConcurrentProcessing 3: %d slots, want 3", got) + } +} + +// TestProcessNeverExceedsMaxConcurrentProcessing starts more Process calls +// than MaxConcurrentProcessing allows and holds the first ones inside +// Process until the test lets them go. No more than the limit may be +// working at once, and the calls held back must wait for a slot and then +// succeed. +func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) { + t.Parallel() + + const ( + limit = 2 + calls = 6 + ) + + proc := New(Params{MaxConcurrentProcessing: limit}) + input := createTestJPEG(t, 50, 50) + + counter := &readingCounter{} + gate := make(chan struct{}) + entered := make(chan struct{}, calls) + results := make(chan error, calls) + + openGate := sync.OnceFunc(func() { close(gate) }) + t.Cleanup(openGate) + + for range calls { + processInBackground(proc, &gatedReader{ + data: bytes.NewReader(input), gate: gate, entered: entered, + counter: counter, + }, results) + } + + waitForEntries(t, entered, limit) + + // A call beyond the limit would start reading its input now. + select { + case <-entered: + t.Fatalf("a Process call started while %d were already working", limit) + case <-time.After(100 * time.Millisecond): + } + + openGate() + + for range calls { + err := <-results + if err != nil { + t.Errorf("Process() error = %v, want nil once a slot is free", err) + } + } + + if most := counter.mostReading(); most > limit { + t.Errorf("%d Process calls worked at once, want at most %d", most, limit) + } +} + +// TestProcessWaitsThenFailsWhenNoSlotFrees holds the only slot and checks +// that another call waits the whole wait timeout, then fails with +// ErrTooManyImages instead of processing anyway. +func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { + t.Parallel() + + proc := New(Params{MaxConcurrentProcessing: 1}) + proc.processingWaitTimeout = 100 * time.Millisecond + + input := createTestJPEG(t, 10, 10) + + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + held := make(chan error, 1) + + openGate := sync.OnceFunc(func() { close(gate) }) + t.Cleanup(openGate) + + processInBackground(proc, &gatedReader{ + data: bytes.NewReader(input), gate: gate, entered: entered, + counter: &readingCounter{}, + }, held) + waitForEntries(t, entered, 1) + + start := time.Now() + + _, err := proc.Process(context.Background(), bytes.NewReader(input), + smallJPEGRequest()) + if !errors.Is(err, ErrTooManyImages) { + t.Fatalf("Process() error = %v, want ErrTooManyImages", err) + } + + if waited := time.Since(start); waited < proc.processingWaitTimeout { + t.Errorf("Process() failed after %v, before waiting %v", + waited, proc.processingWaitTimeout) + } + + openGate() + + err = <-held + if err != nil { + t.Errorf("Process() holding the slot: error = %v, want nil", err) + } +} + +// TestProcessReleasesSlotOnError checks that Process gives its slot back +// when it fails, whether it fails early or late: with one slot, the slot +// must be free after the failure and the next call must succeed. +func TestProcessReleasesSlotOnError(t *testing.T) { + t.Parallel() + + valid := createTestJPEG(t, 10, 10) + + unsupported := smallJPEGRequest() + unsupported.Format = "bmp" + + cases := []struct { + name string + input io.Reader + req *Request + // want is the error Process must return; nil means any error. + want error + }{ + { + name: "input cannot be read", + input: iotest.ErrReader(errTestReadFailed), + req: smallJPEGRequest(), + want: errTestReadFailed, + }, + { + name: "input over the byte limit", + input: bytes.NewReader(createTestJPEG(t, 800, 600)), + req: smallJPEGRequest(), + want: ErrInputDataTooLarge, + }, + { + name: "input not an image", + input: strings.NewReader("not an image"), + req: smallJPEGRequest(), + }, + { + name: "output format not supported", + input: bytes.NewReader(valid), + req: unsupported, + want: ErrUnsupportedOutputFormat, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1}) + proc.processingWaitTimeout = 100 * time.Millisecond + + _, err := proc.Process(context.Background(), tc.input, tc.req) + if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { + t.Fatalf("Process() error = %v, want %v", err, tc.want) + } + + if held := len(proc.processingSemaphore); held != 0 { + t.Fatalf("slot still held after the error: %d held", held) + } + + result, err := proc.Process(context.Background(), bytes.NewReader(valid), + smallJPEGRequest()) + if err != nil { + t.Fatalf("Process() after the error = %v, want nil", err) + } + + _ = result.Content.Close() + }) + } +} diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index e15b90a..ddac1b6 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -383,13 +383,16 @@ func (c *Cache) GetSourceMetadataID( return id, nil } -// GetSourceContent returns a reader for cached source content by its hash. -func (c *Cache) GetSourceContent(contentHash ContentHash) (io.ReadCloser, error) { +// GetSourceContent returns a reader for cached source content by its hash, +// and the content's size in bytes. +func (c *Cache) GetSourceContent( + contentHash ContentHash, +) (io.ReadCloser, int64, error) { if c.disabled { - return nil, ErrNotFound + return nil, 0, ErrNotFound } - return c.srcContent.Load(contentHash) + return c.srcContent.LoadWithSize(contentHash) } // CleanExpired removes expired entries from the cache. diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imgcache/max_concurrent_processing_internal_test.go new file mode 100644 index 0000000..e4c310e --- /dev/null +++ b/internal/imgcache/max_concurrent_processing_internal_test.go @@ -0,0 +1,125 @@ +package imgcache + +import ( + "image/color" + "image/jpeg" + "io" + "os" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imageprocessor" +) + +// widthOnlyRequest asks for the test photo at width, its height scaled to +// keep the photo's aspect ratio. +func widthOnlyRequest(fixtures *TestFixtures, width int) *ImageRequest { + return &ImageRequest{ + SourceHost: fixtures.GoodHost, + SourcePath: testPathPhoto, + Size: Size{Width: width}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } +} + +// holdProcessingSlot takes one of proc's processing slots and returns the +// func that gives it back. Process takes its slot before it reads its input, +// so once it has read a byte from the pipe it holds the slot, until the pipe +// is closed. +func holdProcessingSlot( + t *testing.T, proc *imageprocessor.ImageProcessor, +) func() { + t.Helper() + + input, feed := io.Pipe() + + go func() { + _, _ = proc.Process(t.Context(), input, &imageprocessor.Request{}) + }() + + _, err := feed.Write([]byte{0}) + if err != nil { + t.Fatalf("Process call to hold the slot did not start: %v", err) + } + + release := func() { _ = feed.Close() } + t.Cleanup(release) + + return release +} + +// TestService_Get_WaitsForSlotBeforeReadingCachedSource checks that a +// request whose source is cached holds none of it while it waits for a +// processing slot: it reads the cached file only once it has a slot. With +// the only slot held, a request for a new width of the cached 100x100 photo +// waits; the cached file is then rewritten as a 100x50 image before the slot +// is freed, so the request must answer with that image scaled to 40x20. +func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + svc.processor = imageprocessor.New( + imageprocessor.Params{MaxConcurrentProcessing: 1}, + ) + + // A first request caches the photo as a source. + resp, err := svc.Get(t.Context(), widthOnlyRequest(fixtures, 50)) + if err != nil { + t.Fatalf("first Get() error = %v", err) + } + + _ = resp.Content.Close() + + contentHash, _, err := svc.cache.LookupSource(t.Context(), + widthOnlyRequest(fixtures, 50)) + if err != nil || contentHash == "" { + t.Fatalf("LookupSource() = %q, %v; want the cached source", + contentHash, err) + } + + release := holdProcessingSlot(t, svc.processor) + + var ( + waited *ImageResponse + waitedErr error + ) + + done := make(chan struct{}) + + go func() { + defer close(done) + + waited, waitedErr = svc.Get(t.Context(), widthOnlyRequest(fixtures, 40)) + }() + + // Give the request time to reach the slot: had it read the cached source + // before waiting, it would have read it by now. + time.Sleep(100 * time.Millisecond) + + err = os.WriteFile(svc.cache.srcContent.hashToPath(contentHash), + generateTestJPEG(t, 100, 50, color.RGBA{0, 0, 255, 255}), 0o600) + if err != nil { + t.Fatalf("failed to rewrite the cached source: %v", err) + } + + release() + <-done + + if waitedErr != nil { + t.Fatalf("Get() error = %v", waitedErr) + } + + defer func() { _ = waited.Content.Close() }() + + output, err := jpeg.DecodeConfig(waited.Content) + if err != nil { + t.Fatalf("failed to decode the response: %v", err) + } + + if output.Width != 40 || output.Height != 20 { + t.Errorf("response is %dx%d, want 40x20: the request read the cached "+ + "source before it had a processing slot", output.Width, output.Height) + } +} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index d85dee8..e1152dc 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -43,6 +43,9 @@ type ServiceConfig struct { SigningKey string // Allowlist is the list of hosts that don't require signatures Allowlist []string + // MaxConcurrentProcessing is the most images processed at once; zero + // uses the image processor's default, one per CPU + MaxConcurrentProcessing int // Logger for logging Logger *slog.Logger } @@ -91,9 +94,10 @@ func NewService(cfg *ServiceConfig) (*Service, error) { } maxResponseSize := fetcherCfg.MaxResponseSize - processor := imageprocessor.New( - imageprocessor.Params{MaxInputBytes: maxResponseSize}, - ) + processor := imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: maxResponseSize, + MaxConcurrentProcessing: cfg.MaxConcurrentProcessing, + }) return &Service{ cache: cfg.Cache, @@ -237,38 +241,37 @@ func (s *Service) GenerateSignedURL( baseURL, path, sig, exp, req.Quality, req.FitMode), nil } -// loadCachedSource attempts to load source content from cache, returning nil -// if the cached data is unavailable or exceeds maxResponseSize. -func (s *Service) loadCachedSource(contentHash ContentHash) []byte { - reader, err := s.cache.GetSourceContent(contentHash) +// loadCachedSource opens source content from cache, without reading it, and +// returns it with its size; nil if the cached data is unavailable, empty or +// exceeds maxResponseSize. +func (s *Service) loadCachedSource( + contentHash ContentHash, +) (io.ReadCloser, int64) { + reader, size, err := s.cache.GetSourceContent(contentHash) if err != nil { s.log.Warn("failed to load cached source, fetching", "error", err) - return nil + return nil, 0 } - // Bound the read to maxResponseSize to prevent unbounded memory use - // from unexpectedly large cached files. - limited := io.LimitReader(reader, s.maxResponseSize+1) - data, err := io.ReadAll(limited) - _ = reader.Close() + if size > s.maxResponseSize { + _ = reader.Close() - if err != nil { - s.log.Warn("failed to read cached source, fetching", "error", err) - - return nil - } - - if int64(len(data)) > s.maxResponseSize { s.log.Warn("cached source exceeds max response size, discarding", "hash", contentHash, "max_bytes", s.maxResponseSize, ) - return nil + return nil, 0 } - return data + if size == 0 { + _ = reader.Close() + + return nil, 0 + } + + return reader, size } // processFromSourceOrFetch processes an image, using cached source content @@ -285,22 +288,27 @@ func (s *Service) processFromSourceOrFetch( s.log.Warn("source lookup failed", "error", err) } - var sourceData []byte + var ( + source io.ReadCloser + sourceSize int64 + ) if contentHash != "" { s.log.Debug("using cached source", "hash", contentHash) - sourceData = s.loadCachedSource(contentHash) + source, sourceSize = s.loadCachedSource(contentHash) } // Fetch from upstream if we don't have source data or it's empty - if len(sourceData) == 0 { + if source == nil { return s.fetchAndProcess(ctx, req, cacheKey) } - // Process using cached source; nothing was fetched from upstream - resp, err := s.processAndStore( - ctx, req, cacheKey, sourceData, int64(len(sourceData)), - ) + defer func() { _ = source.Close() }() + + // Process using cached source; nothing was fetched from upstream. The + // image processor reads the source only once it has a processing slot, + // so a request waiting for one holds none of it in memory. + resp, err := s.processAndStore(ctx, req, cacheKey, source, sourceSize) return resp, 0, err } @@ -334,6 +342,10 @@ func (s *Service) fetchAndProcess( return nil, 0, fmt.Errorf("upstream fetch failed: %w", err) } + // Closing the body frees the upstream connection. It is closed only + // after processing, so the fetcher's connection limit also bounds the + // fetched sources held in memory while their requests wait for a + // processing slot. defer func() { _ = fetchResult.Content.Close() }() // Read and validate the source content @@ -379,17 +391,20 @@ func (s *Service) fetchAndProcess( // Continue even if caching fails } - resp, err := s.processAndStore(ctx, req, cacheKey, sourceData, fetchBytes) + resp, err := s.processAndStore( + ctx, req, cacheKey, bytes.NewReader(sourceData), fetchBytes, + ) return resp, fetchBytes, err } -// processAndStore processes an image and stores the result. +// processAndStore processes the image read from source and stores the +// result. func (s *Service) processAndStore( ctx context.Context, req *ImageRequest, cacheKey VariantKey, - sourceData []byte, + source io.Reader, fetchBytes int64, ) (*ImageResponse, error) { // Process the image @@ -402,7 +417,7 @@ func (s *Service) processAndStore( FitMode: imageprocessor.FitMode(req.FitMode), } - processResult, err := s.processor.Process(ctx, bytes.NewReader(sourceData), processReq) + processResult, err := s.processor.Process(ctx, source, processReq) if err != nil { return nil, fmt.Errorf("image processing failed: %w", err) } From 99735f479b70db82ecdb56ab33e383fff662d751 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 11:01:30 +0200 Subject: [PATCH 009/131] Answer image requests with 503 in maintenance mode (closes #71) maintenance_mode was only reported by the health check; every request was still served. One middleware in routes.go, applied to /v1/image/ and /v1/e/ only, now answers them with 503, a Retry-After of MaintenanceRetryAfterSeconds and the JSON error body while it is on. It calls Server.MaintenanceMode(), which had no caller. The health check stays 200 and reports maintenance_mode: the image's Docker HEALTHCHECK requests it, a 503 there would make the container unhealthy, and upaas marks a deploy failed when its container is unhealthy. The login and URL generator pages and /metrics keep working. Documented in README.md and config.example.yml. Model: opus-5-5 --- README.md | 9 +- TODO.md | 8 + config.example.yml | 6 + .../server/login_rate_limit_internal_test.go | 10 +- internal/server/maintenance_internal_test.go | 169 ++++++++++++++++++ internal/server/routes.go | 57 +++++- 6 files changed, 249 insertions(+), 10 deletions(-) create mode 100644 internal/server/maintenance_internal_test.go diff --git a/README.md b/README.md index 001d863..808d192 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ variables set by the file's `env:` section are checked the same way. | `PIXA_METRICS_PASSWORD` | `metrics.password` | Password for `/metrics`; set together with the username | | `PIXA_SENTRY_DSN` | `sentry_dsn` | Sentry DSN for error reporting; empty disables it | | `PIXA_DEBUG` | `debug` | Debug logging and plain-HTTP local development; default `false` | -| `PIXA_MAINTENANCE_MODE` | `maintenance_mode` | Maintenance flag reported by the health check; default `false` | +| `PIXA_MAINTENANCE_MODE` | `maintenance_mode` | Answer image requests with 503; the health check stays 200; default `false` | Key settings in more detail: @@ -306,6 +306,13 @@ Key settings in more detail: container's CPU limit. A request that finds all of them in use waits up to 10 seconds for one to free up; if none does, and `downstream_timeout` has not ended first, it is answered 503 the same way +- `maintenance_mode` — while `true`, the image routes (`/v1/image/` and + `/v1/e/`) answer every request with 503, a `Retry-After` header and a JSON + error body. The health check (`/.well-known/healthcheck.json`) still answers + 200 and reports `"maintenance_mode": true`. It stays 200 because the image's + Docker `HEALTHCHECK` requests it: a 503 there would make the container + unhealthy, and upaas marks a deploy failed when its container is unhealthy. + The login and URL generator pages and `/metrics` keep working See `config.example.yml` for all options with defaults. diff --git a/TODO.md b/TODO.md index 8276df8..4674052 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,14 @@ P2: security: referer blacklist # Completed Steps +- 2026-09-29 maintenance mode refuses image requests (closes #71): while + `maintenance_mode` is on, `/v1/image/` and `/v1/e/` answer 503 with a + `Retry-After` header and the JSON error body, from one middleware in + `internal/server/routes.go`; the health check stays 200 and reports + `maintenance_mode`, as the image's Docker `HEALTHCHECK` requests it and upaas + marks a deploy failed when its container is unhealthy; the login and URL + generator pages and `/metrics` keep working; documented in `README.md` and + `config.example.yml`. - 2026-09-29 bound concurrent image processing and upstream fetches (closes #64): `max_concurrent_processing` (default the number of CPUs pixa can use) limits the images decoded and encoded at once, and `upstream_connections` diff --git a/config.example.yml b/config.example.yml index d64d77e..ebbc833 100644 --- a/config.example.yml +++ b/config.example.yml @@ -16,6 +16,12 @@ # Server settings port: 8080 debug: false + +# While true, the image routes (/v1/image/ and /v1/e/) answer every request +# with 503 and a Retry-After header. The health check keeps answering 200 and +# reports maintenance_mode as true. It stays 200 because the image's Docker +# HEALTHCHECK requests it: a 503 there would make the container unhealthy, and +# upaas marks a deploy failed when its container is unhealthy. maintenance_mode: false # Data directory for SQLite database and cache files diff --git a/internal/server/login_rate_limit_internal_test.go b/internal/server/login_rate_limit_internal_test.go index c37d86e..f5d8c81 100644 --- a/internal/server/login_rate_limit_internal_test.go +++ b/internal/server/login_rate_limit_internal_test.go @@ -18,6 +18,7 @@ import ( "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/globals" "sneak.berlin/go/pixa/internal/handlers" + "sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/middleware" ) @@ -71,8 +72,15 @@ func newTestServer(t *testing.T) *Server { t.Fatalf("database.New() error = %v", err) } + hc, err := healthcheck.New(lc, healthcheck.Params{ + Globals: &globals.Globals{}, Config: cfg, Logger: log, Database: db, + }) + if err != nil { + t.Fatalf("healthcheck.New() error = %v", err) + } + h, err := handlers.New(lc, handlers.Params{ - Logger: log, Database: db, Config: cfg, + Logger: log, Healthcheck: hc, Database: db, Config: cfg, }) if err != nil { t.Fatalf("handlers.New() error = %v", err) diff --git a/internal/server/maintenance_internal_test.go b/internal/server/maintenance_internal_test.go new file mode 100644 index 0000000..96b7272 --- /dev/null +++ b/internal/server/maintenance_internal_test.go @@ -0,0 +1,169 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "sneak.berlin/go/pixa/internal/healthcheck" +) + +// unsignedImagePath is an image URL that carries no signature. +const unsignedImagePath = "/v1/image/cdn.example.com/cat.jpg/100x100.jpeg" + +// TestMaintenanceModeRefusesImageRequests verifies that while maintenance +// mode is on, both image routes answer 503 Service Unavailable with a +// Retry-After header and the JSON error body the image handlers send. +func TestMaintenanceModeRefusesImageRequests(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.MaintenanceMode = true + + requests := []struct { + method string + path string + }{ + {http.MethodGet, unsignedImagePath}, + {http.MethodHead, unsignedImagePath}, + {http.MethodGet, "/v1/e/token/cat.jpg"}, + } + + for _, tc := range requests { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), tc.method, tc.path, nil)) + t.Logf("status %d, body %s", rec.Code, rec.Body.String()) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", + rec.Code, http.StatusServiceUnavailable) + } + + retryAfter := rec.Header().Get("Retry-After") + + seconds, err := strconv.Atoi(retryAfter) + if err != nil || seconds <= 0 { + t.Errorf("Retry-After = %q, want a positive number of seconds", + retryAfter) + } + + // A HEAD response carries no body. + if tc.method == http.MethodHead { + return + } + + var body struct { + Error string `json:"error"` + Status int `json:"status"` + Timestamp string `json:"timestamp"` + } + + err = json.NewDecoder(rec.Body).Decode(&body) + if err != nil { + t.Fatalf("body is not JSON: %v", err) + } + + if body.Error == "" || body.Status != http.StatusServiceUnavailable || + body.Timestamp == "" { + t.Errorf("body = %+v, want an error, status %d and a timestamp", + body, http.StatusServiceUnavailable) + } + }) + } +} + +// TestImageRequestsServedWithoutMaintenanceMode verifies that while +// maintenance mode is off, image requests reach the image handlers instead +// of the 503. The handlers refuse an unsigned image URL with 401 and a token +// they cannot decrypt with 400, so either status shows a request got through. +func TestImageRequestsServedWithoutMaintenanceMode(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.MaintenanceMode = false + + requests := []struct { + method string + path string + want int + }{ + {http.MethodGet, unsignedImagePath, http.StatusUnauthorized}, + {http.MethodHead, unsignedImagePath, http.StatusUnauthorized}, + {http.MethodGet, "/v1/e/token/cat.jpg", http.StatusBadRequest}, + } + + for _, tc := range requests { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), tc.method, tc.path, nil)) + t.Logf("status %d, body %s", rec.Code, rec.Body.String()) + + if rec.Code != tc.want { + t.Errorf("status = %d, want %d from the image handler", + rec.Code, tc.want) + } + }) + } +} + +// TestMaintenanceModeKeepsOtherRoutes verifies that while maintenance mode +// is on, the health check still answers 200 and reports it, and the login +// page and /metrics still answer 200. The image's Docker HEALTHCHECK +// requests the health check: a 503 there would make the container +// unhealthy, and upaas marks a deploy failed when its container is +// unhealthy. +func TestMaintenanceModeKeepsOtherRoutes(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.MaintenanceMode = true + + // /metrics is routed only when its username is set. + s.config.MetricsUsername = "metrics" + s.config.MetricsPassword = "metrics-password" + s.SetupRoutes() + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), + http.MethodGet, "/.well-known/healthcheck.json", nil)) + t.Logf("health check status %d, body %s", rec.Code, rec.Body.String()) + + if rec.Code != http.StatusOK { + t.Fatalf("health check status = %d, want %d", rec.Code, http.StatusOK) + } + + var health healthcheck.Response + + err := json.NewDecoder(rec.Body).Decode(&health) + if err != nil || !health.Maintenance { + t.Errorf("health check maintenance_mode = %v (error %v), want true", + health.Maintenance, err) + } + + rec = httptest.NewRecorder() + s.ServeHTTP(rec, clientRequest(t, http.MethodGet, nil, firstClient, "")) + + if rec.Code != http.StatusOK { + t.Errorf("login page status = %d, want %d", rec.Code, http.StatusOK) + } + + req := httptest.NewRequestWithContext(t.Context(), + http.MethodGet, "/metrics", nil) + req.SetBasicAuth(s.config.MetricsUsername, s.config.MetricsPassword) + + rec = httptest.NewRecorder() + s.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("/metrics status = %d, want %d", rec.Code, http.StatusOK) + } +} diff --git a/internal/server/routes.go b/internal/server/routes.go index 4623b75..ade30ee 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -1,7 +1,9 @@ package server import ( + "encoding/json" "net/http" + "strconv" "time" sentryhttp "github.com/getsentry/sentry-go/http" @@ -17,6 +19,10 @@ import ( // make per minute; the next is refused with 429 Too Many Requests. const LoginAttemptsPerMinute = 5 +// MaintenanceRetryAfterSeconds is the Retry-After, in seconds, sent with +// the 503 that the image routes answer while maintenance mode is on. +const MaintenanceRetryAfterSeconds = 300 + // SetupRoutes configures all HTTP routes. func (s *Server) SetupRoutes() { s.router = chi.NewRouter() @@ -68,15 +74,23 @@ func (s *Server) SetupRoutes() { s.router.Get("/logout", s.h.HandleLogout()) - // Main image proxy route - // /v1/image///x. - s.router.Get("/v1/image/*", s.h.HandleImage()) - s.router.Head("/v1/image/*", s.h.HandleImage()) + // Image routes, refused while maintenance mode is on. Only these: the + // image's Docker HEALTHCHECK requests the health check, a 503 there + // would make the container unhealthy, and upaas marks a deploy failed + // when its container is unhealthy. + s.router.Group(func(r chi.Router) { + r.Use(s.refuseDuringMaintenance) - // Encrypted image URL route - // The trailing filename (e.g., /img.jpg) is ignored but helps - // browsers with content type - s.router.Get("/v1/e/{token}/*", s.h.HandleImageEnc()) + // Main image proxy route + // /v1/image///x. + r.Get("/v1/image/*", s.h.HandleImage()) + r.Head("/v1/image/*", s.h.HandleImage()) + + // Encrypted image URL route + // The trailing filename (e.g., /img.jpg) is ignored but helps + // browsers with content type + r.Get("/v1/e/{token}/*", s.h.HandleImageEnc()) + }) // Metrics endpoint with auth if s.config.MetricsUsername != "" { @@ -86,3 +100,30 @@ func (s *Server) SetupRoutes() { }) } } + +// refuseDuringMaintenance answers a request with 503 Service Unavailable, +// a Retry-After header and a JSON error body while maintenance mode is on, +// and passes it on otherwise. The body has the fields of the JSON errors +// the image handlers send. +func (s *Server) refuseDuringMaintenance(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !s.MaintenanceMode() { + next.ServeHTTP(w, r) + + return + } + + w.Header().Set("Retry-After", strconv.Itoa(MaintenanceRetryAfterSeconds)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + + err := json.NewEncoder(w).Encode(map[string]any{ + "error": "down for maintenance, try again later", + "status": http.StatusServiceUnavailable, + "timestamp": time.Now().UTC().Format(time.RFC3339), + }) + if err != nil { + s.log.Error("json encode error", "error", err) + } + }) +} From bce8860c2e20950e6c253d49bf491a3f795aa06c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 12:00:10 +0200 Subject: [PATCH 010/131] Keep variant content types in memory for cache hits (closes #70) Cache.metaCache was declared and never used, so every cache hit read and parsed the variant's .meta file. It is now an LRU (github.com/hashicorp/golang-lru/v2) of up to 10,000 variants' content types, filled by StoreVariant and by a read of a .meta file, so a hit for a variant it holds skips the .meta read. Only a type from a .meta file or a store ever enters memory, never the application/octet-stream fallback, and a stored type is never replaced by an older one from disk. The variant file itself is still opened on every hit, so nothing is served from memory alone. Model: opus-5-5 --- README.md | 5 +- TODO.md | 11 + go.mod | 1 + go.sum | 2 + internal/imgcache/cache.go | 73 +++- internal/imgcache/eviction.go | 10 +- internal/imgcache/metacache_internal_test.go | 349 +++++++++++++++++++ internal/imgcache/storage.go | 36 +- internal/imgcache/testutil_internal_test.go | 1 + 9 files changed, 460 insertions(+), 28 deletions(-) create mode 100644 internal/imgcache/metacache_internal_test.go diff --git a/README.md b/README.md index 808d192..9321291 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,10 @@ prevent abuse, and allowlisted source hosts for open access. Multiple source paths may reference the same content blob; the database tracks references rather than using filesystem refcounting. -In-process caching of request-to-output mappings targets 1-5k r/s. +Toward a target of 1-5k r/s, pixa keeps in memory the content types of +the 10,000 transformed images most recently cached or served, so a +cache hit on one of them reads only the image file from disk and not +the metadata file stored beside it. ### Routes diff --git a/TODO.md b/TODO.md index 4674052..bb548f5 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,17 @@ P2: security: referer blacklist # Completed Steps +- 2026-09-29 variant content types kept in memory (closes #70): + `Cache.metaCache` holds the content types of up to 10,000 variants in an LRU + (`github.com/hashicorp/golang-lru/v2`), filled by `StoreVariant` and by + `GetVariant` after it reads a `.meta` file, where a type `StoreVariant` added + meanwhile is kept over the one read, and never with the + `application/octet-stream` served for a variant without one; for a variant it + holds, `GetVariant` skips the `.meta` read, still opening the variant file and + taking the size from it; eviction removes the entry before deleting the files, + and `GetVariant` removes it when the file will not open; the cap is a + constant, not a setting; the unused `variantMeta` type is gone; `README.md` + describes it. - 2026-09-29 maintenance mode refuses image requests (closes #71): while `maintenance_mode` is on, `/v1/image/` and `/v1/e/` answer 503 with a `Retry-After` header and the JSON error body, from one middleware in diff --git a/go.mod b/go.mod index f5c4c98..4a13f35 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/go-chi/httprate v0.16.0 github.com/gorilla/csrf v1.7.3 github.com/gorilla/securecookie v1.1.2 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/prometheus/client_golang v1.23.2 github.com/slok/go-http-metrics v0.13.0 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index f9a4f3d..c4542f6 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index ddac1b6..a4faab4 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -14,6 +14,7 @@ import ( "sync" "time" + lru "github.com/hashicorp/golang-lru/v2" "sneak.berlin/go/pixa/internal/httpfetcher" ) @@ -26,6 +27,10 @@ var ( // HTTP status code for successful fetch. const httpStatusOK = 200 +// metaCacheSize is how many variants' content types metaCache holds. A +// variant not among them is served as before, reading its .meta file. +const metaCacheSize = 10000 + // CacheConfig holds cache configuration. type CacheConfig struct { StateDir string @@ -49,12 +54,6 @@ type CacheConfig struct { Logger *slog.Logger } -// variantMeta stores content type for fast cache hits without reading .meta file. -type variantMeta struct { - ContentType string - Size int64 -} - // Cache implements the caching layer for the image proxy. type Cache struct { db *sql.DB @@ -76,9 +75,10 @@ type Cache struct { evictionStarted bool evictionStopOnce sync.Once - // In-memory cache of variant metadata (content type, size) to avoid - // reading .meta files - metaCache map[VariantKey]variantMeta + // metaCache holds the content types of the variants most recently + // stored or served, so a hit does not read the variant's .meta file. + // It never stands in for the variant file, which is always opened. + metaCache *lru.Cache[VariantKey, string] // contentLocks serializes StoreSource and evictSourceBlob per // content hash, closing the race window between an eviction's row @@ -101,6 +101,11 @@ func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { log = slog.Default() } + metaCache, err := lru.New[VariantKey, string](metaCacheSize) + if err != nil { + return nil, fmt.Errorf("failed to create variant content type cache: %w", err) + } + c := &Cache{ db: db, config: config, @@ -109,7 +114,7 @@ func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { evictionPressure: make(chan struct{}, 1), evictionStop: make(chan struct{}), evictionDone: make(chan struct{}), - metaCache: make(map[VariantKey]variantMeta), + metaCache: metaCache, contentLocks: newContentLock(), } @@ -177,13 +182,30 @@ func (c *Cache) Lookup(ctx context.Context, req *ImageRequest) (*LookupResult, e }, nil } -// GetVariant returns a reader, size, and content type for a cached variant. +// GetVariant returns a reader, size, and content type for a cached +// variant. The content type comes from metaCache, or else from the +// variant's .meta file and is then kept in metaCache. A variant with +// no .meta file is served as application/octet-stream, which is not +// kept. func (c *Cache) GetVariant(cacheKey VariantKey) (io.ReadCloser, int64, string, error) { if c.disabled { return nil, 0, "", ErrNotFound } - return c.variants.LoadWithMeta(cacheKey) + contentType, known := c.metaCache.Get(cacheKey) + if !known { + return c.loadVariantWithMeta(cacheKey) + } + + reader, size, err := c.variants.LoadWithSize(cacheKey) + if err != nil { + // The file is gone, e.g. deleted outside pixa + c.metaCache.Remove(cacheKey) + + return nil, 0, "", err + } + + return reader, size, contentType, nil } // StoreSource stores fetched source content and metadata. On a @@ -286,6 +308,8 @@ func (c *Cache) StoreVariant( return err } + c.metaCache.Add(cacheKey, contentType) + _, err = c.db.ExecContext(ctx, ` INSERT INTO variant_content (cache_key, size_bytes, content_type) VALUES (?, ?, ?) @@ -499,6 +523,31 @@ func (c *Cache) IncrementTransformCount(ctx context.Context) { } } +// loadVariantWithMeta is GetVariant for a variant metaCache does not +// hold: it reads the content type from the variant's .meta file and +// keeps it in metaCache, unless a StoreVariant has put one there +// meanwhile, as the store's is newer. A read that finds no .meta file, +// as one can between a store's writing of the variant file and of its +// .meta file, serves application/octet-stream and keeps nothing, so +// metaCache only ever holds a type read from a .meta file or passed to +// StoreVariant. +func (c *Cache) loadVariantWithMeta( + cacheKey VariantKey, +) (io.ReadCloser, int64, string, error) { + reader, size, contentType, err := c.variants.LoadWithMeta(cacheKey) + if err != nil { + return nil, 0, "", err + } + + if contentType == "" { + return reader, size, fallbackContentType, nil + } + + c.metaCache.ContainsOrAdd(cacheKey, contentType) + + return reader, size, contentType, nil +} + // writeMetadataSidecar writes the JSON metadata sidecar of a stored source. // A failure is logged and is otherwise non-fatal; the metadata is in the // database. diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 8bf6edb..5d185ea 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -39,8 +39,8 @@ const tempFilePrefix = ".tmp-" // to each variant file. const variantMetaSuffix = ".meta" -// fallbackContentType is recorded when a reconciled variant file has -// no readable .meta sidecar. +// fallbackContentType is the content type given to a variant file that +// has no readable .meta sidecar, when it is served or reconciled. const fallbackContentType = "application/octet-stream" // UsageBytes returns the total number of bytes of cache content @@ -271,7 +271,9 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro // evictVariant removes one variant: accounting row first, then the // content and .meta files, so the database never references a deleted -// file. +// file. The metaCache entry goes before the files; a GetVariant that +// read them just before may put it back, and the next GetVariant then +// fails to open the file and removes it again. func (c *Cache) evictVariant(ctx context.Context, cacheKey VariantKey) error { _, err := c.db.ExecContext(ctx, `DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey)) @@ -279,6 +281,8 @@ func (c *Cache) evictVariant(ctx context.Context, cacheKey VariantKey) error { return fmt.Errorf("failed to delete variant accounting row: %w", err) } + c.metaCache.Remove(cacheKey) + err = c.variants.DeleteWithMeta(cacheKey) if err != nil { return err diff --git a/internal/imgcache/metacache_internal_test.go b/internal/imgcache/metacache_internal_test.go new file mode 100644 index 0000000..bade726 --- /dev/null +++ b/internal/imgcache/metacache_internal_test.go @@ -0,0 +1,349 @@ +package imgcache + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sync" + "testing" + "time" +) + +// webpRequest returns a request for a 100x100 WebP variant of path. +func webpRequest(path string) *ImageRequest { + return &ImageRequest{ + SourceHost: testHostCDN, + SourcePath: path, + Size: Size{Width: 100, Height: 100}, + Format: FormatWebP, + Quality: 85, + FitMode: FitCover, + } +} + +// assertVariantServed checks that GetVariant serves key with the given +// content and the image/webp content type storeEvictionTestVariant stores. +func assertVariantServed(t *testing.T, cache *Cache, key VariantKey, content []byte) { + t.Helper() + + reader, size, contentType, err := cache.GetVariant(key) + if err != nil { + t.Fatalf("GetVariant(%s) error = %v", key, err) + } + + defer func() { _ = reader.Close() }() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("reading variant %s: %v", key, err) + } + + if !bytes.Equal(got, content) { + t.Errorf("GetVariant(%s) content = %q, want %q", key, got, content) + } + + if size != int64(len(content)) { + t.Errorf("GetVariant(%s) size = %d, want %d", key, size, len(content)) + } + + if contentType != testContentTypeWebP { + t.Errorf("GetVariant(%s) content type = %q, want %q", + key, contentType, testContentTypeWebP) + } +} + +// assertVariantNotFound checks that GetVariant refuses key with +// ErrNotFound. +func assertVariantNotFound(t *testing.T, cache *Cache, key VariantKey) { + t.Helper() + + reader, _, _, err := cache.GetVariant(key) + if err == nil { + _ = reader.Close() + } + + if !errors.Is(err, ErrNotFound) { + t.Errorf("GetVariant(%s) error = %v, want ErrNotFound", key, err) + } +} + +// assertLookupMisses checks that Lookup reports request as a miss. +func assertLookupMisses(t *testing.T, cache *Cache, request *ImageRequest) { + t.Helper() + + lookup, err := cache.Lookup(t.Context(), request) + if err != nil { + t.Fatalf("Lookup(%s) error = %v", request.SourcePath, err) + } + + if lookup.Hit { + t.Errorf("Lookup(%s) is a hit, want a miss", request.SourcePath) + } +} + +// renameFile renames the file at from to to. +func renameFile(t *testing.T, from, to string) { + t.Helper() + + err := os.Rename(from, to) + if err != nil { + t.Fatalf("renaming %s: %v", from, err) + } +} + +// TestSecondHitDoesNotReadMetaFile checks that once a variant has been +// stored or read, a hit takes its content type from memory: with the +// .meta file deleted, GetVariant must still return the stored content +// type rather than the application/octet-stream it uses without one. +func TestSecondHitDoesNotReadMetaFile(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<20) + content := []byte("webp variant bytes") + + storeEvictionTestVariant(t, cache, testVariantKeyOne, content) + + // A second Cache on the same state directory starts with nothing in + // memory, as pixad does after a restart, so its first read uses the + // .meta file. + restarted, err := NewCache(cache.db, cache.config) + if err != nil { + t.Fatalf("NewCache() error = %v", err) + } + + assertVariantServed(t, restarted, testVariantKeyOne, content) + + err = os.Remove(cache.variants.keyToPath(testVariantKeyOne) + ".meta") + if err != nil { + t.Fatalf("removing .meta file: %v", err) + } + + assertVariantServed(t, cache, testVariantKeyOne, content) + assertVariantServed(t, restarted, testVariantKeyOne, content) +} + +// TestReadDuringStoreKeepsStoredContentType checks that a GetVariant +// which began before StoreVariant finished cannot replace the content +// type the store kept in memory. Such a read can find the variant file +// but not yet its .meta file, and so gets application/octet-stream. The +// test deletes the .meta file after the store, then runs the part of +// GetVariant that comes after its check of memory. +func TestReadDuringStoreKeepsStoredContentType(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<20) + content := []byte("webp variant bytes") + + storeEvictionTestVariant(t, cache, testVariantKeyOne, content) + + err := os.Remove(cache.variants.keyToPath(testVariantKeyOne) + ".meta") + if err != nil { + t.Fatalf("removing .meta file: %v", err) + } + + reader, _, contentType, err := cache.loadVariantWithMeta(testVariantKeyOne) + if err != nil { + t.Fatalf("loadVariantWithMeta(%s) error = %v", testVariantKeyOne, err) + } + + _ = reader.Close() + + t.Logf("the read without a .meta file got content type %q", contentType) + + assertVariantServed(t, cache, testVariantKeyOne, content) +} + +// TestReadOfOlderMetaFileKeepsStoredContentType checks that a read +// which got its content type from a .meta file that StoreVariant had +// not yet rewritten cannot replace the type the store kept in memory. +// The test writes such a .meta file, with a different content type, +// after the store, then runs the part of GetVariant that comes after +// its check of memory. +func TestReadOfOlderMetaFileKeepsStoredContentType(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<20) + content := []byte("webp variant bytes") + + storeEvictionTestVariant(t, cache, testVariantKeyOne, content) + + olderMeta, err := json.Marshal(VariantMeta{ + ContentType: testContentTypeJPEG, + Size: int64(len(content)), + }) + if err != nil { + t.Fatalf("encoding .meta file: %v", err) + } + + metaPath := cache.variants.keyToPath(testVariantKeyOne) + ".meta" + + err = os.WriteFile(metaPath, olderMeta, StorageFilePerm) + if err != nil { + t.Fatalf("writing .meta file: %v", err) + } + + reader, _, contentType, err := cache.loadVariantWithMeta(testVariantKeyOne) + if err != nil { + t.Fatalf("loadVariantWithMeta(%s) error = %v", testVariantKeyOne, err) + } + + _ = reader.Close() + + if contentType != testContentTypeJPEG { + t.Fatalf("loadVariantWithMeta(%s) content type = %q, want %q from the .meta file", + testVariantKeyOne, contentType, testContentTypeJPEG) + } + + kept, _ := cache.metaCache.Get(testVariantKeyOne) + if kept != testContentTypeWebP { + t.Errorf("content type in memory = %q, want the stored %q", + kept, testContentTypeWebP) + } +} + +// TestFailedReadDuringStoreKeepsStoredContentType checks that a read +// which found no .meta file cannot leave application/octet-stream in +// memory, even when another read has removed the content type +// StoreVariant kept there. In this order: the store; a read that finds +// the variant in memory but cannot open its file, and so removes it +// from memory; a read that opened the variant file before the store +// wrote its .meta file. Later hits must get the stored content type. +func TestFailedReadDuringStoreKeepsStoredContentType(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<20) + content := []byte("webp variant bytes") + variantPath := cache.variants.keyToPath(testVariantKeyOne) + metaPath := variantPath + ".meta" + + storeEvictionTestVariant(t, cache, testVariantKeyOne, content) + + renameFile(t, variantPath, variantPath+".hidden") + assertVariantNotFound(t, cache, testVariantKeyOne) + renameFile(t, variantPath+".hidden", variantPath) + + renameFile(t, metaPath, metaPath+".hidden") + + reader, _, contentType, err := cache.GetVariant(testVariantKeyOne) + if err != nil { + t.Fatalf("GetVariant(%s) error = %v", testVariantKeyOne, err) + } + + _ = reader.Close() + + t.Logf("the read without a .meta file got content type %q", contentType) + + renameFile(t, metaPath+".hidden", metaPath) + + assertVariantServed(t, cache, testVariantKeyOne, content) +} + +// TestEvictedVariantIsNotServed checks that a variant the evictor +// removed is a miss and cannot be read, although it had been stored +// and served before. +func TestEvictedVariantIsNotServed(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1500) + + oldRequest := webpRequest("/old.jpg") + newRequest := webpRequest("/new.jpg") + oldKey := CacheKey(oldRequest) + newKey := CacheKey(newRequest) + oldContent := bytes.Repeat([]byte{0x01}, 1000) + newContent := bytes.Repeat([]byte{0x02}, 1000) + + storeEvictionTestVariant(t, cache, oldKey, oldContent) + storeEvictionTestVariant(t, cache, newKey, newContent) + assertVariantServed(t, cache, oldKey, oldContent) + setVariantLastAccessed(t, cache, oldKey, time.Now().Add(-time.Hour)) + + err := cache.EvictToLimit(t.Context()) + if err != nil { + t.Fatalf("EvictToLimit() error = %v", err) + } + + assertLookupMisses(t, cache, oldRequest) + assertVariantNotFound(t, cache, oldKey) + assertVariantServed(t, cache, newKey, newContent) +} + +// TestVariantDeletedFromDiskIsNotServed checks that a variant whose +// file was deleted by something other than the evictor cannot be read, +// and is a miss afterwards. +func TestVariantDeletedFromDiskIsNotServed(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<20) + + request := webpRequest("/deleted.jpg") + key := CacheKey(request) + content := []byte("webp variant bytes") + + storeEvictionTestVariant(t, cache, key, content) + assertVariantServed(t, cache, key, content) + + err := os.Remove(cache.variants.keyToPath(key)) + if err != nil { + t.Fatalf("removing variant file: %v", err) + } + + assertVariantNotFound(t, cache, key) + assertLookupMisses(t, cache, request) +} + +// TestConcurrentVariantStoreReadAndEvict stores, reads and evicts +// variants from several goroutines at once, for the race detector. +func TestConcurrentVariantStoreReadAndEvict(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<20) + ctx := t.Context() + + var wg sync.WaitGroup + + for goroutine := range 8 { + wg.Go(func() { + key := VariantKey(fmt.Sprintf("aabbccdd01%02d", goroutine)) + content := []byte(key) + + for range 20 { + err := cache.StoreVariant( + ctx, key, bytes.NewReader(content), testContentTypeWebP) + if err != nil { + t.Errorf("StoreVariant(%s) error = %v", key, err) + + return + } + + reader, _, contentType, err := cache.GetVariant(key) + if err != nil { + t.Errorf("GetVariant(%s) error = %v", key, err) + + return + } + + _ = reader.Close() + + if contentType != testContentTypeWebP { + t.Errorf("GetVariant(%s) content type = %q, want %q", + key, contentType, testContentTypeWebP) + } + + err = cache.evictVariant(ctx, key) + if err != nil { + t.Errorf("evictVariant(%s) error = %v", key, err) + + return + } + + assertVariantNotFound(t, cache, key) + } + }) + } + + wg.Wait() +} diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index b9487ce..7e1cb38 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -506,32 +506,44 @@ func (s *VariantStorage) Load(key VariantKey) (io.ReadCloser, error) { return f, nil } -// LoadWithMeta returns a reader, size, and content type for the content at -// the given key. -func (s *VariantStorage) LoadWithMeta( - key VariantKey, -) (io.ReadCloser, int64, string, error) { +// LoadWithSize returns a reader and file size for the content at the +// given key. +func (s *VariantStorage) LoadWithSize(key VariantKey) (io.ReadCloser, int64, error) { path := s.keyToPath(key) - metaPath := path + ".meta" f, err := os.Open(path) //nolint:gosec // path derived from cache key if err != nil { if os.IsNotExist(err) { - return nil, 0, "", ErrNotFound + return nil, 0, ErrNotFound } - return nil, 0, "", fmt.Errorf("failed to open content: %w", err) + return nil, 0, fmt.Errorf("failed to open content: %w", err) } stat, err := f.Stat() if err != nil { _ = f.Close() - return nil, 0, "", fmt.Errorf("failed to stat content: %w", err) + return nil, 0, fmt.Errorf("failed to stat content: %w", err) } - // Load metadata for content type - contentType := "application/octet-stream" // fallback + return f, stat.Size(), nil +} + +// LoadWithMeta returns a reader, size, and content type for the content at +// the given key. The content type is read from the .meta file, and is +// empty when that file is missing or unreadable. +func (s *VariantStorage) LoadWithMeta( + key VariantKey, +) (io.ReadCloser, int64, string, error) { + f, size, err := s.LoadWithSize(key) + if err != nil { + return nil, 0, "", err + } + + var contentType string + + metaPath := s.keyToPath(key) + ".meta" metaData, err := os.ReadFile(metaPath) //nolint:gosec // path derived from cache key if err == nil { @@ -541,7 +553,7 @@ func (s *VariantStorage) LoadWithMeta( } } - return f, stat.Size(), contentType, nil + return f, size, contentType, nil } // Exists checks if content exists at the given key. diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index 1ca7c11..b29f750 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -24,6 +24,7 @@ const ( testHostExample = "example.com" testPathCat = "/photos/cat.jpg" testContentTypeJPEG = "image/jpeg" + testContentTypeWebP = "image/webp" testHeaderContentType = "Content-Type" ) From fd7d7ed205d78ab2e3cd028f55cde5e7b0d0ead6 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 29 Sep 2026 12:44:37 +0200 Subject: [PATCH 011/131] Container makes /var/lib/pixa usable before starting pixad (closes #159) The entrypoint now creates /var/lib/pixa if it is missing. When the directory or one of its top-level entries belongs to another user or group, it gives the whole tree to pixad (uid and gid 65532); it then sets the directory's mode to 750 and runs the server as pixad as before. Data left by a run under another uid is taken over this way. Only the top level is checked, so a normal start does not walk the cache; the tree is changed deepest first, so an interrupted start is finished by the next one. "Running under upaas" in README.md no longer tells the operator to create or chown the host directory. Model: opus-5-5 --- README.md | 9 ++------- TODO.md | 7 +++++++ deploy/docker-entrypoint.sh | 18 +++++++++++++----- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9321291..ed5b3b0 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,8 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: - **Port:** pixa listens on container port `8080`. - **Volume:** container path `/var/lib/pixa`, where pixa keeps its - database and cache. upaas bind-mounts the host path it is given and - does not create it, so the host directory must exist before the first - deploy. + database and cache. Creating the host directory when it is missing is + upaas's job, tracked in https://git.eeqj.de/sneak/upaas/issues/235. - **Environment variables:** - `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and login, 32+ characters, for example from @@ -58,10 +57,6 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: `healthy`. The probe uses the port from `PORT` (default `8080`), so a port changed only in a mounted config file is not seen by it: change the port with `PORT`. -- **First run:** create the host directory, owned by root or by uid - `65532` and gid `65532`. The server runs as the container's `pixad` - user, which has that uid and gid, and the container gives the - directory to `pixad` when it starts. ## Rationale diff --git a/TODO.md b/TODO.md index bb548f5..637e3a5 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ P2: security: referer blacklist # Completed Steps +- 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes + #159): `deploy/docker-entrypoint.sh` creates the directory if it is missing, + gives the directory and everything in it to `pixad` when the directory or one + of its top-level entries belongs to another user or group, sets its mode to + `750`, then runs the server as `pixad`; data left by an earlier run under + another uid is taken over this way; "Running under upaas" in `README.md` no + longer tells the operator to create or chown the host directory. - 2026-09-29 variant content types kept in memory (closes #70): `Cache.metaCache` holds the content types of up to 10,000 variants in an LRU (`github.com/hashicorp/golang-lru/v2`), filled by `StoreVariant` and by diff --git a/deploy/docker-entrypoint.sh b/deploy/docker-entrypoint.sh index 691ff2f..abb3f16 100755 --- a/deploy/docker-entrypoint.sh +++ b/deploy/docker-entrypoint.sh @@ -1,14 +1,22 @@ #!/bin/sh # deploy/docker-entrypoint.sh: the Docker image's ENTRYPOINT. It runs as -# root only to give /var/lib/pixa to pixad: a host directory -# bind-mounted there keeps its host owner, often root, and pixad could -# not write to it. The server itself always runs as pixad. +# root only to make /var/lib/pixa usable by pixad: a host directory +# bind-mounted there keeps its host owner, often root, and data from an +# earlier run may belong to another uid. The server itself always runs +# as pixad. set -eu main() { - if [ "$(stat -c %U /var/lib/pixa)" != pixad ]; then - chown pixad:pixad /var/lib/pixa + mkdir -p /var/lib/pixa + # Only the directory and its top-level entries are checked, so a + # normal start does not walk the cache. -depth gives each directory + # to pixad after its contents, so a start stopped part way leaves + # something at the top for the next start to find; -h changes a + # symlink itself, never the file it points to. + if [ -n "$(find /var/lib/pixa -maxdepth 1 \( ! -user pixad -o ! -group pixad \))" ]; then + find /var/lib/pixa -depth -exec chown -h pixad:pixad {} + fi + chmod 750 /var/lib/pixa exec su-exec pixad /usr/local/bin/pixad "$@" } From 3a2eb1f4d2141d1814fe1d377de95a027a1f2b7d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 2 Oct 2026 01:17:33 +0200 Subject: [PATCH 012/131] Read the architecture at run time, not via a Buildarch ldflag (closes #164) CONVENTIONS.md no longer passes the build architecture in through the Makefile. The Buildarch variable, globals field and BUILDARCH Makefile lines are removed from every example, and the logger's Identify example logs "arch", runtime.GOARCH, as internal/logger already does. Wording matches the HTTP server conventions in sneak/prompts. Model: opus-5-5 --- CONVENTIONS.md | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/CONVENTIONS.md b/CONVENTIONS.md index 6a62434..9f8c1d3 100644 --- a/CONVENTIONS.md +++ b/CONVENTIONS.md @@ -112,15 +112,13 @@ import ( ) var ( - Appname string = "CHANGEME" - Version string - Buildarch string + Appname string = "CHANGEME" + Version string ) func main() { globals.Appname = Appname globals.Version = Version - globals.Buildarch = Buildarch fx.New( fx.Provide( @@ -859,7 +857,7 @@ func (l *Logger) Identify() { l.log.Info("starting", "appname", l.params.Globals.Appname, "version", l.params.Globals.Version, - "buildarch", l.params.Globals.Buildarch, + "arch", runtime.GOARCH, ) } ``` @@ -979,23 +977,20 @@ import "go.uber.org/fx" // Package-level variables (set from main) var ( - Appname string - Version string - Buildarch string + Appname string + Version string ) // Struct for DI type Globals struct { - Appname string - Version string - Buildarch string + Appname string + Version string } func New(lc fx.Lifecycle) (*Globals, error) { n := &Globals{ - Appname: Appname, - Buildarch: Buildarch, - Version: Version, + Appname: Appname, + Version: Version, } return n, nil } @@ -1006,15 +1001,13 @@ func New(lc fx.Lifecycle) (*Globals, error) { ```go // cmd/httpd/main.go var ( - Appname string = "CHANGEME" // Default, overridden by build - Version string // Set at build time - Buildarch string // Set at build time + Appname string = "CHANGEME" // Default, overridden by build + Version string // Set at build time ) func main() { globals.Appname = Appname globals.Version = Version - globals.Buildarch = Buildarch // ... } ``` @@ -1025,10 +1018,9 @@ Use ldflags to inject version information at build time: ```makefile VERSION := $(shell git describe --tags --always) -BUILDARCH := $(shell go env GOARCH) build: - go build -ldflags "-X main.Version=$(VERSION) -X main.Buildarch=$(BUILDARCH)" ./cmd/httpd + go build -ldflags "-X main.Version=$(VERSION)" ./cmd/httpd ``` --- From 869b5ba67ff06a4aa8340e8c7c6005d56f205390 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 2 Oct 2026 06:01:40 +0200 Subject: [PATCH 013/131] Stamp the tag or short commit in a plain docker build (closes #166) .dockerignore now lets .git into the build context, without .git/config, which can hold a remote URL with a credential. ARG VERSION has no default: given none, the build stage takes the version from git describe --tags --always, and fails if the context carries .git and no version comes out. pixad now logs its name, version and architecture as its first log line, through the existing Logger.Identify, which nothing called. Model: opus-5-5 --- .dockerignore | 6 +++++- Dockerfile | 25 ++++++++++++++++++------- TODO.md | 7 +++++++ cmd/pixad/main.go | 5 ++++- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8fa00fc..549cdb4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,8 @@ -.git +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a +# password in a remote URL or the token the CI checkout step stores there. +.git/config .gitignore .DS_Store .env* diff --git a/Dockerfile b/Dockerfile index 76d9bac..91e4731 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,13 +43,24 @@ COPY . . RUN make test # VERSION is declared here, not earlier: a new value reruns only the -# build, not script/bootstrap or the tests. CGO stays enabled for -# govips; -trimpath keeps build paths out of the binary, and -s -w -# leave out the symbol table and debug information. -ARG VERSION=dev -RUN CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \ - -ldflags "-s -w -X main.Version=${VERSION}" \ - -o /pixad ./cmd/pixad +# build, not script/bootstrap or the tests. Given none, the version is +# `git describe --tags --always` of the .git in the build context (git +# comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH +# after one, the short commit when no tag is reachable. A context that +# carries .git and still yields no version fails the build; one without +# .git, as from a source tarball, stamps an empty version. CGO stays +# enabled for govips; -trimpath keeps build paths out of the binary, and +# -s -w leave out the symbol table and debug information. +ARG VERSION +RUN version="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ + [ "$version" = unknown ]; }; then \ + echo "the build context carries .git but yields no version" >&2; \ + exit 1; \ + fi; \ + CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \ + -ldflags "-s -w -X main.Version=${version}" \ + -o /pixad ./cmd/pixad # Runtime stage # alpine:3.21, 2026-02-25 diff --git a/TODO.md b/TODO.md index 637e3a5..ef4bd5c 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-02 a plain `docker build .` stamps the tag or short commit, not + `dev` (closes #166): `.dockerignore` lets `.git` into the build context, + without `.git/config`; with no `VERSION` build argument the `Dockerfile` + takes the version from `git describe --tags --always`, and fails the build if + the context carries `.git` and no version comes out; `ARG VERSION` has no + default; pixad logs its version, with its name and architecture, as its first + log line at startup. - 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes #159): `deploy/docker-entrypoint.sh` creates the directory if it is missing, gives the directory and everything in it to `pixad` when the directory or one diff --git a/cmd/pixad/main.go b/cmd/pixad/main.go index 601c1ee..96442dc 100644 --- a/cmd/pixad/main.go +++ b/cmd/pixad/main.go @@ -56,6 +56,9 @@ func run(_ *cobra.Command, _ []string) { middleware.New, healthcheck.New, ), - fx.Invoke(func(*server.Server) {}), + fx.Invoke( + func(log *logger.Logger) { log.Identify() }, + func(*server.Server) {}, + ), ).Run() } From 021516e09928d67fc37c9f92f489608e831a9928 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sat, 3 Oct 2026 17:49:43 +0200 Subject: [PATCH 014/131] Send CORS headers only from the image routes (closes #98) The CORS middleware with the `access_control_allow_origin` origin wrapped every route from the router root, so the login and URL generator pages and `/metrics` sent `Access-Control-Allow-Origin` too. It now wraps only `/v1/image/` and `/v1/e/`, which form a `/v1` subrouter so that a browser's preflight `OPTIONS` request still gets its answer; the maintenance mode group moved inside it unchanged. Every other route sends no CORS headers. A new test checks both image routes, a preflight included, and the login and URL generator pages. `README.md` and `config.example.yml` say the setting covers the image routes only. Unverified by test: `/metrics`, whose middleware registers with the process-wide Prometheus registry. Model: opus-5-5 --- README.md | 7 ++- TODO.md | 6 ++ config.example.yml | 5 +- internal/server/cors_internal_test.go | 64 ++++++++++++++++++++ internal/server/maintenance_internal_test.go | 8 ++- internal/server/routes.go | 38 +++++++----- 6 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 internal/server/cors_internal_test.go diff --git a/README.md b/README.md index ed5b3b0..6129f36 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ variables set by the file's `env:` section are checked the same way. | `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` | | `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB | | `PIXA_DOWNSTREAM_TIMEOUT` | `downstream_timeout` | Time allowed for answering one client request; default `60s` | -| `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN` | `access_control_allow_origin` | CORS origin allowed to read responses: `*` or one origin; default `*` | +| `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN` | `access_control_allow_origin` | CORS origin allowed to read image responses: `*` or one origin; default `*` | | `PIXA_METRICS_USERNAME` | `metrics.username` | Username for `/metrics`, which is served only when both are set | | `PIXA_METRICS_PASSWORD` | `metrics.password` | Password for `/metrics`; set together with the username | | `PIXA_SENTRY_DSN` | `sentry_dsn` | Sentry DSN for error reporting; empty disables it | @@ -247,8 +247,9 @@ variables set by the file's `env:` section are checked the same way. Key settings in more detail: -- `access_control_allow_origin` — the origin a browser lets read pixa's - responses, sent as the CORS `Access-Control-Allow-Origin` header: `*`, the +- `access_control_allow_origin` — the origin a browser lets read the responses + of the image routes, `/v1/image/` and `/v1/e/`, sent as the CORS + `Access-Control-Allow-Origin` header; no other route sends it. `*`, the default, is any site; otherwise one `http` or `https` origin such as `https://example.com`, whose host is a lowercase host name (letters, digits, hyphens and dots, with a letter in its last part) or an IP address diff --git a/TODO.md b/TODO.md index ef4bd5c..539e37e 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,12 @@ P2: security: referer blacklist # Completed Steps +- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS + middleware, with the `access_control_allow_origin` origin, moved from the + router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it + still answers a preflight `OPTIONS` request; the login and URL generator + pages, `/metrics` and the other routes send no `Access-Control-Allow-Origin`; + documented in `README.md` and `config.example.yml`. - 2026-10-02 a plain `docker build .` stamps the tag or short commit, not `dev` (closes #166): `.dockerignore` lets `.git` into the build context, without `.git/config`; with no `VERSION` build argument the `Dockerfile` diff --git a/config.example.yml b/config.example.yml index ebbc833..ac32c98 100644 --- a/config.example.yml +++ b/config.example.yml @@ -103,8 +103,9 @@ upstream_max_response_size: 52428800 # longer than upstream_fetch_timeout plus 20 seconds. downstream_timeout: 60s -# The origin a browser lets read pixa's responses, sent as the CORS -# Access-Control-Allow-Origin header: "*" (the default) is any site; +# The origin a browser lets read the responses of the image routes, +# /v1/image/ and /v1/e/, sent as the CORS Access-Control-Allow-Origin +# header; no other route sends it. "*" (the default) is any site; # otherwise one http or https origin such as https://example.com, whose # host is a lowercase host name (letters, digits, hyphens and dots, with a # letter in its last part) or an IP address (IPv6 in brackets, in its diff --git a/internal/server/cors_internal_test.go b/internal/server/cors_internal_test.go new file mode 100644 index 0000000..6232588 --- /dev/null +++ b/internal/server/cors_internal_test.go @@ -0,0 +1,64 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// TestCORSOnlyOnImageRoutes verifies that the image routes answer with the +// configured access_control_allow_origin, a preflight request included, and +// that the login and URL generator pages send no Access-Control-Allow-Origin, +// so no other site can read them. /metrics is left out: its middleware +// registers with the process-wide Prometheus registry, which only one test +// in this package can do. +func TestCORSOnlyOnImageRoutes(t *testing.T) { + t.Parallel() + + const appOrigin = "https://app.example.com" + + s := newTestServer(t) + s.config.AccessControlAllowOrigin = appOrigin + s.SetupRoutes() + + requests := []struct { + method string + path string + want string + }{ + {http.MethodGet, unsignedImagePath, appOrigin}, + {http.MethodHead, unsignedImagePath, appOrigin}, + {http.MethodOptions, unsignedImagePath, appOrigin}, + {http.MethodGet, encryptedImagePath, appOrigin}, + {http.MethodGet, "/", ""}, + {http.MethodOptions, "/", ""}, + {http.MethodPost, "/generate", ""}, + {http.MethodGet, "/logout", ""}, + } + + for _, tc := range requests { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequestWithContext( + t.Context(), tc.method, tc.path, nil) + req.Header.Set("Origin", appOrigin) + + // An OPTIONS request naming the method it asks about is the + // preflight a browser sends before some cross-origin requests. + if tc.method == http.MethodOptions { + req.Header.Set("Access-Control-Request-Method", http.MethodGet) + } + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + t.Logf("status %d", rec.Code) + + got := rec.Header().Get("Access-Control-Allow-Origin") + if got != tc.want { + t.Errorf("Access-Control-Allow-Origin = %q, want %q", + got, tc.want) + } + }) + } +} diff --git a/internal/server/maintenance_internal_test.go b/internal/server/maintenance_internal_test.go index 96b7272..f800005 100644 --- a/internal/server/maintenance_internal_test.go +++ b/internal/server/maintenance_internal_test.go @@ -13,6 +13,10 @@ import ( // unsignedImagePath is an image URL that carries no signature. const unsignedImagePath = "/v1/image/cdn.example.com/cat.jpg/100x100.jpeg" +// encryptedImagePath is an encrypted image URL whose token cannot be +// decrypted. +const encryptedImagePath = "/v1/e/token/cat.jpg" + // TestMaintenanceModeRefusesImageRequests verifies that while maintenance // mode is on, both image routes answer 503 Service Unavailable with a // Retry-After header and the JSON error body the image handlers send. @@ -28,7 +32,7 @@ func TestMaintenanceModeRefusesImageRequests(t *testing.T) { }{ {http.MethodGet, unsignedImagePath}, {http.MethodHead, unsignedImagePath}, - {http.MethodGet, "/v1/e/token/cat.jpg"}, + {http.MethodGet, encryptedImagePath}, } for _, tc := range requests { @@ -95,7 +99,7 @@ func TestImageRequestsServedWithoutMaintenanceMode(t *testing.T) { }{ {http.MethodGet, unsignedImagePath, http.StatusUnauthorized}, {http.MethodHead, unsignedImagePath, http.StatusUnauthorized}, - {http.MethodGet, "/v1/e/token/cat.jpg", http.StatusBadRequest}, + {http.MethodGet, encryptedImagePath, http.StatusBadRequest}, } for _, tc := range requests { diff --git a/internal/server/routes.go b/internal/server/routes.go index ade30ee..f341e8a 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -38,7 +38,6 @@ func (s *Server) SetupRoutes() { s.router.Use(s.mw.Metrics()) } - s.router.Use(s.mw.CORS()) s.router.Use(middleware.Timeout(s.config.DownstreamTimeout)) if s.sentryEnabled { @@ -74,22 +73,31 @@ func (s *Server) SetupRoutes() { s.router.Get("/logout", s.h.HandleLogout()) - // Image routes, refused while maintenance mode is on. Only these: the - // image's Docker HEALTHCHECK requests the health check, a 503 there - // would make the container unhealthy, and upaas marks a deploy failed - // when its container is unhealthy. - s.router.Group(func(r chi.Router) { - r.Use(s.refuseDuringMaintenance) + // Image routes, the only ones that send CORS headers, as pages on other + // sites read them. They are a subrouter rather than a group: a group's + // middleware runs only for a request that matches one of its routes, + // and a browser's preflight OPTIONS request matches none, so the CORS + // middleware could not answer it. + s.router.Route("/v1", func(r chi.Router) { + r.Use(s.mw.CORS()) - // Main image proxy route - // /v1/image///x. - r.Get("/v1/image/*", s.h.HandleImage()) - r.Head("/v1/image/*", s.h.HandleImage()) + // Refused while maintenance mode is on. Only these: the image's + // Docker HEALTHCHECK requests the health check, a 503 there would + // make the container unhealthy, and upaas marks a deploy failed + // when its container is unhealthy. + r.Group(func(r chi.Router) { + r.Use(s.refuseDuringMaintenance) - // Encrypted image URL route - // The trailing filename (e.g., /img.jpg) is ignored but helps - // browsers with content type - r.Get("/v1/e/{token}/*", s.h.HandleImageEnc()) + // Main image proxy route + // /v1/image///x. + r.Get("/image/*", s.h.HandleImage()) + r.Head("/image/*", s.h.HandleImage()) + + // Encrypted image URL route + // The trailing filename (e.g., /img.jpg) is ignored but helps + // browsers with content type + r.Get("/e/{token}/*", s.h.HandleImageEnc()) + }) }) // Metrics endpoint with auth From 697c14633b0ebc394c89d84ce902ca19013f17f6 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 01:58:48 +0200 Subject: [PATCH 015/131] Share one fetch and transcode among concurrent misses (closes #65) Requests that missed the same variant at once each fetched and transcoded it. They now share one call through golang.org/x/sync/singleflight, keyed on the variant cache key. The first request processes the variant with a context that ignores its cancellation but keeps its deadline; the others wait for its result, holding no connection or processing slot, and return as soon as their own context ends. The processing request waits even then, up to its deadline. Each request counts one miss; the processing counts its fetch and transcode once. A panic while processing is reported through the request's Sentry hub, when there is one, and becomes an error instead of stopping pixad. Model: opus-5-5 --- README.md | 8 + TODO.md | 12 + go.mod | 2 +- internal/imgcache/cache.go | 35 +- .../concurrent_misses_internal_test.go | 509 ++++++++++++++++++ internal/imgcache/service.go | 152 +++++- 6 files changed, 677 insertions(+), 41 deletions(-) create mode 100644 internal/imgcache/concurrent_misses_internal_test.go diff --git a/README.md b/README.md index 6129f36..ba1def6 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,14 @@ or proxy cache keeps the image after pixa would refuse the URL. A URL with no expiry gets one year. `immutable` only stops a client revalidating while its copy is fresh. +When several requests for the same image, size, format, quality and fit miss +the cache at once, they share one upstream fetch (or one read of the cached +source) and one transcode: the first request does the work, and the others wait +for its image or its error, holding no upstream connection or processing slot +of their own. A waiting request stops waiting when its own client goes away. +The work goes on for the others even if the first request's client goes away, +until that request's `downstream_timeout` ends. + The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is refused with 429 and a `Retry-After` header. Behind a reverse proxy the client diff --git a/TODO.md b/TODO.md index 539e37e..2400fe7 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,18 @@ P2: security: referer blacklist # Completed Steps +- 2026-09-29 share concurrent misses (closes #65): requests that miss the same + variant at once (the same cache key, so quality and fit included) share one + upstream fetch or cached source read and one transcode through + `golang.org/x/sync/singleflight`; the first request's processing ignores its + cancellation but keeps its deadline, and the others wait for its image or + error holding no upstream connection or processing slot, and stop waiting + when their own context ends; the request doing the processing waits for it + even then, up to its deadline; a request whose context has already ended + starts nothing; each request counts one miss, and the processing counts its + fetch and transcode once; a panic while processing is reported to Sentry when + `sentry_dsn` is set and becomes an error for every waiting request instead of + stopping pixad; documented in `README.md`. - 2026-09-29 only the image routes send CORS headers (closes #98): the CORS middleware, with the `access_control_allow_origin` origin, moved from the router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it diff --git a/go.mod b/go.mod index 4a13f35..ae8006a 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 go.uber.org/fx v1.24.0 golang.org/x/crypto v0.41.0 + golang.org/x/sync v0.19.0 modernc.org/sqlite v1.42.2 ) @@ -134,7 +135,6 @@ require ( golang.org/x/image v0.34.0 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.36.0 // indirect golang.org/x/term v0.34.0 // indirect golang.org/x/text v0.32.0 // indirect diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index a4faab4..a2d2d61 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -471,7 +471,8 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { return &stats, nil } -// IncrementStats increments cache statistics. +// IncrementStats counts a cache hit or miss, and an upstream fetch that read +// fetchBytes bytes, as IncrementUpstreamFetch does. func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) { var err error @@ -495,18 +496,26 @@ func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) c.log.Warn("failed to count cache hit or miss", "hit", hit, "error", err) } - if fetchBytes > 0 { - _, err = c.db.ExecContext(ctx, ` - UPDATE cache_stats - SET upstream_fetch_count = upstream_fetch_count + 1, - upstream_fetch_bytes = upstream_fetch_bytes + ?, - last_updated_at = CURRENT_TIMESTAMP - WHERE id = 1 - `, fetchBytes) - if err != nil { - c.log.Warn("failed to count upstream fetch", - "fetch_bytes", fetchBytes, "error", err) - } + c.IncrementUpstreamFetch(ctx, fetchBytes) +} + +// IncrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes. +// A fetch that read no bytes is not counted. +func (c *Cache) IncrementUpstreamFetch(ctx context.Context, fetchBytes int64) { + if fetchBytes <= 0 { + return + } + + _, err := c.db.ExecContext(ctx, ` + UPDATE cache_stats + SET upstream_fetch_count = upstream_fetch_count + 1, + upstream_fetch_bytes = upstream_fetch_bytes + ?, + last_updated_at = CURRENT_TIMESTAMP + WHERE id = 1 + `, fetchBytes) + if err != nil { + c.log.Warn("failed to count upstream fetch", + "fetch_bytes", fetchBytes, "error", err) } } diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go new file mode 100644 index 0000000..ad0e1b2 --- /dev/null +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -0,0 +1,509 @@ +package imgcache + +import ( + "bytes" + "context" + "errors" + "image/jpeg" + "io" + "io/fs" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/getsentry/sentry-go" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/magic" +) + +// arrivalWait is how long a test gives requests it has started to reach the +// point where they wait for a held fetch. +const arrivalWait = 100 * time.Millisecond + +// heldFetcher counts the fetches it is asked for and holds each one until +// releaseFetches is called, so that a test can have requests arrive while a +// fetch is in progress. started receives once for every fetch. +type heldFetcher struct { + upstream httpfetcher.Fetcher + fetches atomic.Int32 + started chan struct{} + release chan struct{} + releaseOnce sync.Once +} + +func (f *heldFetcher) Fetch( + ctx context.Context, url string, +) (*httpfetcher.FetchResult, error) { + f.fetches.Add(1) + + f.started <- struct{}{} + + select { + case <-f.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + + return f.upstream.Fetch(ctx, url) +} + +// releaseFetches lets every held fetch, and every later one, go on. +func (f *heldFetcher) releaseFetches() { + f.releaseOnce.Do(func() { close(f.release) }) +} + +// setupHeldFetchService returns a test service whose fetches go through a +// heldFetcher. Its database is limited to one connection: each connection to +// an in-memory SQLite database opens a new, empty one, so requests running at +// once must share the connection that holds the schema. +func setupHeldFetchService(t *testing.T) (*Service, *TestFixtures, *heldFetcher) { + t.Helper() + + svc, fixtures := SetupTestService(t) + svc.cache.db.SetMaxOpenConns(1) + + fetcher := &heldFetcher{ + upstream: svc.fetcher, + started: make(chan struct{}, 100), + release: make(chan struct{}), + } + svc.fetcher = fetcher + + t.Cleanup(fetcher.releaseFetches) + + return svc, fixtures, fetcher +} + +// photoVariant asks for the test photo, 100x100, at 50x25 as a JPEG of the +// given quality and fit mode. Each call returns a new request, as Get writes +// to the request it is given. +func photoVariant(fixtures *TestFixtures, quality int, fit FitMode) *ImageRequest { + return &ImageRequest{ + SourceHost: fixtures.GoodHost, + SourcePath: testPathPhoto, + Size: Size{Width: 50, Height: 25}, + Format: FormatJPEG, + Quality: quality, + FitMode: fit, + } +} + +// getResult is what one Get call returned, with the image read out. +type getResult struct { + image []byte + err error +} + +// startGet calls Get in a goroutine of its own and delivers what it returned +// on the channel. +func startGet( + ctx context.Context, svc *Service, req *ImageRequest, +) <-chan getResult { + results := make(chan getResult, 1) + + go func() { + resp, err := svc.Get(ctx, req) + if err != nil { + results <- getResult{err: err} + + return + } + + defer func() { _ = resp.Content.Close() }() + + image, err := io.ReadAll(resp.Content) + results <- getResult{image: image, err: err} + }() + + return results +} + +// jpegSize returns the width and height of the JPEG image in data, or 0 and 0 +// if data is not one. +func jpegSize(data []byte) (int, int) { + config, err := jpeg.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return 0, 0 + } + + return config.Width, config.Height +} + +// TestService_Get_ConcurrentMissesShareOneFetch starts several requests for +// one uncached variant while the first one's fetch is held. Between them they +// must fetch the source once and transcode it once, every one must be answered +// with the same 50x25 JPEG, and each must count one miss. +func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + const requests = 8 + + pending := make([]<-chan getResult, 0, requests) + for range requests { + pending = append(pending, + startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover))) + } + + <-fetcher.started + time.Sleep(arrivalWait) + fetcher.releaseFetches() + + var first []byte + + for i, results := range pending { + got := <-results + if got.err != nil { + t.Fatalf("request %d: Get() error = %v", i, got.err) + } + + if width, height := jpegSize(got.image); width != 50 || height != 25 { + t.Errorf("request %d: image is %dx%d, want a 50x25 JPEG", i, width, height) + } + + if first == nil { + first = got.image + } else if !bytes.Equal(got.image, first) { + t.Errorf("request %d: image differs from request 0's", i) + } + } + + if fetches := fetcher.fetches.Load(); fetches != 1 { + t.Errorf("%d requests made %d upstream fetches, want 1", requests, fetches) + } + + // NewTestFS builds the same files the test service's fetcher serves. + testFS, _ := NewTestFS(t) + + photo, err := fs.ReadFile(testFS, fixtures.GoodHostJPEG) + if err != nil { + t.Fatal(err) + } + + want := cacheStatsCounters{0, requests, 1, int64(len(photo)), 1} + + if got := readCacheStatsCounters(t, svc.cache); got != want { + t.Errorf("counters = %+v, want %+v", got, want) + } +} + +// TestService_Get_ConcurrentVariantsStayApart requests three variants of the +// test photo at once that differ only in quality or fit. Each must be made by +// a fetch and a transcode of its own, and each answer must be its own variant. +func TestService_Get_ConcurrentVariantsStayApart(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + cover := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + lowQuality := startGet(t.Context(), svc, photoVariant(fixtures, 40, FitCover)) + contain := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitContain)) + + for range 3 { + select { + case <-fetcher.started: + case <-time.After(5 * time.Second): + t.Fatal("fewer fetches started than variants requested: " + + "variants differing in quality or fit were merged") + } + } + + fetcher.releaseFetches() + + images := make(map[string][]byte) + + for _, variant := range []struct { + name string + results <-chan getResult + width, height int + }{ + {"q=85 fit=cover", cover, 50, 25}, + {"q=40 fit=cover", lowQuality, 50, 25}, + {"q=85 fit=contain", contain, 25, 25}, + } { + got := <-variant.results + if got.err != nil { + t.Fatalf("%s: Get() error = %v", variant.name, got.err) + } + + width, height := jpegSize(got.image) + if width != variant.width || height != variant.height { + t.Errorf("%s: image is %dx%d, want a %dx%d JPEG", variant.name, + width, height, variant.width, variant.height) + } + + images[variant.name] = got.image + } + + if bytes.Equal(images["q=85 fit=cover"], images["q=40 fit=cover"]) { + t.Error("q=40 was answered with the q=85 image") + } +} + +// TestService_Get_WaiterStopsWhenItsContextEnds has a second request for a +// variant join the first one's held fetch, then ends the second request's +// context. The second request must return at once with the context's error, +// while the fetch is still held, and the first must still be answered. +func TestService_Get_WaiterStopsWhenItsContextEnds(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + first := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + + <-fetcher.started + + waiterCtx, cancelWaiter := context.WithCancel(t.Context()) + waiter := startGet(waiterCtx, svc, photoVariant(fixtures, 85, FitCover)) + + time.Sleep(arrivalWait) + cancelWaiter() + + select { + case got := <-waiter: + if !errors.Is(got.err, context.Canceled) { + t.Errorf("waiting request: Get() error = %v, want %v", + got.err, context.Canceled) + } + case <-time.After(time.Second): + t.Fatal("waiting request did not return when its context ended") + } + + fetcher.releaseFetches() + + got := <-first + if got.err != nil { + t.Fatalf("first request: Get() error = %v", got.err) + } + + if width, height := jpegSize(got.image); width != 50 || height != 25 { + t.Errorf("first request: image is %dx%d, want a 50x25 JPEG", width, height) + } + + if fetches := fetcher.fetches.Load(); fetches != 1 { + t.Errorf("upstream fetches = %d, want 1", fetches) + } +} + +// TestService_Get_FirstRequestLeavingKeepsTheWork ends the context of the +// request whose fetch is held, after a second request has joined it. The fetch +// and transcode must go on and answer the second request. The first request +// waits for its own work, as every request did before misses were shared, and +// is answered too. +func TestService_Get_FirstRequestLeavingKeepsTheWork(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + firstCtx, cancelFirst := context.WithCancel(t.Context()) + first := startGet(firstCtx, svc, photoVariant(fixtures, 85, FitCover)) + + <-fetcher.started + + second := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + + time.Sleep(arrivalWait) + cancelFirst() + fetcher.releaseFetches() + + for name, results := range map[string]<-chan getResult{ + "first request": first, "second request": second, + } { + got := <-results + if got.err != nil { + t.Fatalf("%s: Get() error = %v", name, got.err) + } + + if width, height := jpegSize(got.image); width != 50 || height != 25 { + t.Errorf("%s: image is %dx%d, want a 50x25 JPEG", name, width, height) + } + } + + if fetches := fetcher.fetches.Load(); fetches != 1 { + t.Errorf("upstream fetches = %d, want 1", fetches) + } +} + +// TestService_Get_ConcurrentMissesShareAFailure has several requests for an +// image that cannot be served arrive while its fetch is held: the one fetch +// answers all of them with its error. The request after them is answered from +// the negative cache when the failure is kept there, and fetches again when it +// is not. +func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantErr error // what every request at once gets + wantNextErr error // what the request after them gets + wantFetches int32 // fetches once the request after them is answered + }{ + {"upstream answers 404, kept in the negative cache", + "/images/missing.jpg", httpfetcher.ErrUpstreamError, + ErrNegativeCached, 1}, + {"source fails the magic byte check, not kept", + "/images/text.png", magic.ErrUnknownFormat, magic.ErrUnknownFormat, 2}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + request := func() *ImageRequest { + req := photoVariant(fixtures, 85, FitCover) + req.SourcePath = tc.path + + return req + } + + pending := make([]<-chan getResult, 0, 4) + for range 4 { + pending = append(pending, startGet(t.Context(), svc, request())) + } + + <-fetcher.started + time.Sleep(arrivalWait) + fetcher.releaseFetches() + + for i, results := range pending { + if got := <-results; !errors.Is(got.err, tc.wantErr) { + t.Errorf("request %d: Get() error = %v, want %v", i, got.err, tc.wantErr) + } + } + + _, err := svc.Get(t.Context(), request()) + if !errors.Is(err, tc.wantNextErr) { + t.Errorf("next request: Get() error = %v, want %v", err, tc.wantNextErr) + } + + if fetches := fetcher.fetches.Load(); fetches != tc.wantFetches { + t.Errorf("upstream fetches = %d, want %d", fetches, tc.wantFetches) + } + }) + } +} + +// TestService_Get_EndedRequestFetchesNothing checks that a request whose +// context has already ended when it misses the cache starts no fetch. +func TestService_Get_EndedRequestFetchesNothing(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := svc.Get(ctx, photoVariant(fixtures, 85, FitCover)) + if !errors.Is(err, context.Canceled) { + t.Errorf("Get() error = %v, want %v", err, context.Canceled) + } + + if fetches := fetcher.fetches.Load(); fetches != 0 { + t.Errorf("upstream fetches = %d, want 0", fetches) + } +} + +// TestService_Get_ReturnsByItsDeadline gives a request a deadline and holds +// its fetch until the fetch's context ends, as the fetcher does while it waits +// for a free connection to the host. The request must return by its deadline +// with the deadline's error. +func TestService_Get_ReturnsByItsDeadline(t *testing.T) { + t.Parallel() + + svc, fixtures, _ := setupHeldFetchService(t) + + const timeout = 200 * time.Millisecond + + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + + results := startGet(ctx, svc, photoVariant(fixtures, 85, FitCover)) + + select { + case got := <-results: + if !errors.Is(got.err, context.DeadlineExceeded) { + t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded) + } + case <-time.After(timeout + time.Second): + t.Fatal("request did not return by its deadline") + } +} + +// panickingFetcher panics on every fetch. +type panickingFetcher struct{} + +func (panickingFetcher) Fetch( + context.Context, string, +) (*httpfetcher.FetchResult, error) { + panic("upstream fetcher panicked") +} + +// TestService_Get_PanicBecomesAnError checks that a panic while a variant is +// being made reaches its request as an error naming the panic, instead of +// being raised again. +func TestService_Get_PanicBecomesAnError(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + svc.fetcher = panickingFetcher{} + + var err error + + func() { + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Get() panicked: %v", recovered) + } + }() + + _, err = svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover)) + }() + + if err == nil || !strings.Contains(err.Error(), "upstream fetcher panicked") { + t.Errorf("Get() error = %v, want one naming the panic", err) + } +} + +// TestService_Get_PanicIsReportedToSentry checks that a panic while a variant +// is being made is reported through the Sentry hub on the request's context, +// where the Sentry middleware puts one when sentry_dsn is set. +func TestService_Get_PanicIsReportedToSentry(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + svc.fetcher = panickingFetcher{} + + transport := &sentry.MockTransport{} + + client, err := sentry.NewClient(sentry.ClientOptions{ + Dsn: "https://abc123@sentry.example.com/42", + Transport: transport, + }) + if err != nil { + t.Fatal(err) + } + + ctx := sentry.SetHubOnContext(t.Context(), + sentry.NewHub(client, sentry.NewScope())) + + func() { + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Get() panicked: %v", recovered) + } + }() + + _, _ = svc.Get(ctx, photoVariant(fixtures, 85, FitCover)) + }() + + events := transport.Events() + if len(events) != 1 || events[0].Message != "upstream fetcher panicked" { + t.Errorf("Sentry events = %+v, want one for the panic", events) + } +} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index e1152dc..90acf0b 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -8,9 +8,12 @@ import ( "io" "log/slog" "net/url" + "runtime/debug" "time" "github.com/dustin/go-humanize" + "github.com/getsentry/sentry-go" + "golang.org/x/sync/singleflight" "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" @@ -29,6 +32,9 @@ type Service struct { log *slog.Logger allowHTTP bool maxResponseSize int64 + // variantsInProgress lets the requests that miss the same variant at the + // same time share one fetch and one transcode. + variantsInProgress singleflight.Group } // ServiceConfig holds configuration for the image service. @@ -160,14 +166,12 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e } } - // Cache miss - process the cached source or fetch it, then count the - // miss with the bytes it fetched from upstream, also when it failed or - // the request context has ended meanwhile - cacheKey := CacheKey(req) + // Cache miss - get the variant, processed once for all the requests that + // miss it at the same time, then count this request's miss, also when it + // failed or the request context has ended meanwhile + response, err := s.processOrWait(ctx, req) - response, fetchedBytes, err := s.processFromSourceOrFetch(ctx, req, cacheKey) - - s.cache.IncrementStats(context.WithoutCancel(ctx), false, fetchedBytes) + s.cache.IncrementStats(context.WithoutCancel(ctx), false, 0) if err != nil { return nil, err @@ -241,6 +245,96 @@ func (s *Service) GenerateSignedURL( baseURL, path, sig, exp, req.Quality, req.FitMode), nil } +// errPanicked is returned when processing a variant panicked. +var errPanicked = errors.New("panic while processing image") + +// processOrWait returns the variant req asks for. The first of the requests +// that miss a variant at the same time processes it, and singleflight hands +// its result, or its error, to the others: they fetch nothing, read no source +// and take no upstream connection or processing slot. The processing ignores +// the first request's cancellation, so the others are still served if that +// client goes away, but keeps its deadline. +func (s *Service) processOrWait( + ctx context.Context, req *ImageRequest, +) (*ImageResponse, error) { + // A request that has already ended starts no processing + if ctx.Err() != nil { + return nil, ctx.Err() + } + + cacheKey := CacheKey(req) + + // Closed when this request's own function runs, which singleflight does + // only when no other request is processing the variant + processing := make(chan struct{}) + + results := s.variantsInProgress.DoChan(string(cacheKey), + func() (_ any, err error) { + close(processing) + + // singleflight would raise a panic again in a goroutine of its + // own, where no handler recovers it, and stop pixad. It is + // reported through the Sentry hub that the Sentry middleware puts + // on the request's context when sentry_dsn is set. + defer func() { + recovered := recover() + if recovered != nil { + s.log.Error("panic while processing image", + "host", req.SourceHost, "path", req.SourcePath, + "panic", recovered, "stack", string(debug.Stack())) + + if hub := sentry.GetHubFromContext(ctx); hub != nil { + hub.RecoverWithContext(ctx, recovered) + } + + err = fmt.Errorf("%w: %v", errPanicked, recovered) + } + }() + + processingCtx := context.WithoutCancel(ctx) + + if deadline, ok := ctx.Deadline(); ok { + var cancel context.CancelFunc + + processingCtx, cancel = context.WithDeadline(processingCtx, deadline) + defer cancel() + } + + return s.processFromSourceOrFetch(processingCtx, req, cacheKey) + }) + + var result singleflight.Result + + select { + case result = <-results: + case <-ctx.Done(): + select { + case <-processing: + // This request is processing the variant: it waits for the + // result, as every request did before misses were shared + result = <-results + default: + // Another request is processing the variant, or this request's + // function has not started yet; the processing goes on without it + return nil, ctx.Err() + } + } + + if result.Err != nil { + return nil, result.Err + } + + variant, _ := result.Val.(*processedVariant) + + return &ImageResponse{ + Content: io.NopCloser(bytes.NewReader(variant.data)), + ContentLength: int64(len(variant.data)), + ContentType: variant.contentType, + FetchedBytes: variant.fetchedBytes, + ETag: formatETag(cacheKey), + }, nil +} + // loadCachedSource opens source content from cache, without reading it, and // returns it with its size; nil if the cached data is unavailable, empty or // exceeds maxResponseSize. @@ -275,13 +369,12 @@ func (s *Service) loadCachedSource( } // processFromSourceOrFetch processes an image, using cached source content -// if available. It also returns the number of bytes fetched from upstream, -// as fetchAndProcess does, or 0 when the cached source was used. +// if available. func (s *Service) processFromSourceOrFetch( ctx context.Context, req *ImageRequest, cacheKey VariantKey, -) (*ImageResponse, int64, error) { +) (*processedVariant, error) { // Check if we have cached source content contentHash, _, err := s.cache.LookupSource(ctx, req) if err != nil { @@ -308,19 +401,17 @@ func (s *Service) processFromSourceOrFetch( // Process using cached source; nothing was fetched from upstream. The // image processor reads the source only once it has a processing slot, // so a request waiting for one holds none of it in memory. - resp, err := s.processAndStore(ctx, req, cacheKey, source, sourceSize) - - return resp, 0, err + return s.processAndStore(ctx, req, cacheKey, source, sourceSize) } // fetchAndProcess fetches from upstream, processes, and caches the result. -// It also returns the number of bytes read from upstream, including when +// It counts the fetch with the bytes read from upstream, including when // reading the response or a later step fails. func (s *Service) fetchAndProcess( ctx context.Context, req *ImageRequest, cacheKey VariantKey, -) (*ImageResponse, int64, error) { +) (*processedVariant, error) { // Fetch from upstream sourceURL := req.SourceURL() @@ -339,7 +430,7 @@ func (s *Service) fetchAndProcess( } } - return nil, 0, fmt.Errorf("upstream fetch failed: %w", err) + return nil, fmt.Errorf("upstream fetch failed: %w", err) } // Closing the body frees the upstream connection. It is closed only @@ -352,8 +443,11 @@ func (s *Service) fetchAndProcess( sourceData, err := io.ReadAll(fetchResult.Content) fetchBytes := int64(len(sourceData)) + // Counted also when the request context has ended meanwhile + s.cache.IncrementUpstreamFetch(context.WithoutCancel(ctx), fetchBytes) + if err != nil { - return nil, fetchBytes, fmt.Errorf("failed to read upstream response: %w", err) + return nil, fmt.Errorf("failed to read upstream response: %w", err) } // Calculate download bitrate @@ -381,7 +475,7 @@ func (s *Service) fetchAndProcess( // Validate magic bytes match content type err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) if err != nil { - return nil, fetchBytes, fmt.Errorf("content validation failed: %w", err) + return nil, fmt.Errorf("content validation failed: %w", err) } // Store source content @@ -391,11 +485,17 @@ func (s *Service) fetchAndProcess( // Continue even if caching fails } - resp, err := s.processAndStore( + return s.processAndStore( ctx, req, cacheKey, bytes.NewReader(sourceData), fetchBytes, ) +} - return resp, fetchBytes, err +// processedVariant is a variant as processAndStore made it. Each request that +// shared its processing serves it through a reader of its own. +type processedVariant struct { + data []byte + contentType string + fetchedBytes int64 } // processAndStore processes the image read from source and stores the @@ -406,7 +506,7 @@ func (s *Service) processAndStore( cacheKey VariantKey, source io.Reader, fetchBytes int64, -) (*ImageResponse, error) { +) (*processedVariant, error) { // Process the image processStart := time.Now() @@ -470,12 +570,10 @@ func (s *Service) processAndStore( // Continue even if caching fails } - return &ImageResponse{ - Content: io.NopCloser(bytes.NewReader(processedData)), - ContentLength: outputSize, - ContentType: processResult.ContentType, - FetchedBytes: fetchBytes, - ETag: formatETag(cacheKey), + return &processedVariant{ + data: processedData, + contentType: processResult.ContentType, + fetchedBytes: fetchBytes, }, nil } From b402eaf88c32fff5dd72181ebee0da0443bb3f7b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 03:58:35 +0200 Subject: [PATCH 016/131] Run the checks on every script/cibuild and script/docker build (closes #101) Both scripts pass a new CHECK_EPOCH, which the Dockerfile's make fmt-check, make lint and make test steps name in their commands. On an unchanged tree Docker used to serve those steps from its build cache, so a run could pass without checking anything. The script/bootstrap steps stay cached. A plain docker build . still works, as upaas builds the image that way: it leaves CHECK_EPOCH empty and reuses the check steps only for an identical build context. Model: opus-5-5 --- Dockerfile | 16 +++++++++++----- README.md | 5 +++-- TODO.md | 9 +++++++++ script/cibuild | 11 +++++++---- script/docker | 10 +++++++--- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91e4731..424ee87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,14 @@ COPY . . # Tells script/lint it is inside a container, so it runs the linter. ENV container=docker -# Run formatting check and linter -RUN make fmt-check -RUN make lint +# Run formatting check and linter. script/cibuild and script/docker pass +# a new CHECK_EPOCH on every run, and each check step names it in its +# command, so a new value reruns the step instead of reusing a cached +# success that checked nothing. A plain `docker build .` leaves it empty +# and reuses the check steps only for an identical build context. +ARG CHECK_EPOCH +RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check +RUN echo "check epoch: ${CHECK_EPOCH}" && make lint # Build stage # golang:1.25.4-alpine, 2026-02-25 @@ -39,8 +44,9 @@ RUN script/bootstrap # Copy source code COPY . . -# Run tests -RUN make test +# Run tests; a new CHECK_EPOCH reruns them, as in the lint stage. +ARG CHECK_EPOCH +RUN echo "check epoch: ${CHECK_EPOCH}" && make test # VERSION is declared here, not earlier: a new value reruns only the # build, not script/bootstrap or the tests. Given none, the version is diff --git a/README.md b/README.md index ba1def6..76a1ef3 100644 --- a/README.md +++ b/README.md @@ -353,8 +353,9 @@ them. We provide: - `script/check` — run test, lint, and fmt-check - `script/docker` — build the Docker image tagged via `script/projectname` - `script/docker-smoke` — build the image, start it, wait for it to be healthy -- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile - runs the checks, so a green build implies a green repo) +- `script/cibuild` — CI entrypoint: `docker build .` with a new + `CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of + coming from the build cache, and a green run implies a green repo - `script/precommit` — pre-commit checks (`go mod tidy` guard, then `script/check`) - `script/install-precommit` — install the git pre-commit hook that diff --git a/TODO.md b/TODO.md index 2400fe7..cebef13 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,15 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-03 every `script/cibuild` and `script/docker` run executes the checks + (closes #101): the `Dockerfile` declares `CHECK_EPOCH` above `make fmt-check` + and `make lint` in the lint stage and above `make test` in the build stage, + and each of those steps names it in its command; both scripts pass a new value + on every run, so Docker runs the checks instead of reusing cached results, + while the `script/bootstrap` steps stay cached; a plain `docker build .` still + works, leaves it empty, and reuses the check steps only for an identical build + context; the `script/cibuild` comment and `README.md` no longer say that any + successful build implies a green repo. - 2026-09-29 share concurrent misses (closes #65): requests that miss the same variant at once (the same cache key, so quality and fit included) share one upstream fetch or cached source read and one transcode through diff --git a/script/cibuild b/script/cibuild index e20e047..948134c 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,15 +1,18 @@ #!/bin/sh # script/cibuild: run the CI build. The Dockerfile runs the checks -# (make fmt-check, lint, test), so a successful build implies a green -# repo. Generic: needs no adaptation. The Gitea workflow runs this on -# push. +# (make fmt-check, lint, test) as build steps. This script passes a new +# CHECK_EPOCH on every run, so Docker runs those steps instead of +# reusing cached results: a successful run means the checks ran and +# passed on this tree. Generic: needs no adaptation. The Gitea workflow +# runs this on push. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - docker build . + epoch="$(date +%s)$$" + docker build --build-arg CHECK_EPOCH="$epoch" . } main "$@" diff --git a/script/docker b/script/docker index 2884e41..b9907d8 100755 --- a/script/docker +++ b/script/docker @@ -1,7 +1,9 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. -# Identical in all repos; the tag comes from script/projectname. -# Generic: needs no adaptation. +# Identical in all repos; the tag comes from script/projectname. Like +# script/cibuild, it passes a new CHECK_EPOCH, so the build runs the +# checks instead of reusing cached results. Generic: needs no +# adaptation. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -9,7 +11,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build -t "$("$SCRIPT_DIR/projectname")" . + epoch="$(date +%s)$$" + docker build --build-arg CHECK_EPOCH="$epoch" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@" From 00da62db2cdd4b20512e0638e331ab14d31b8c49 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 04:59:30 +0200 Subject: [PATCH 017/131] Exit with the shutdown's code and wait for image processing (closes #86) fx alone handles SIGINT and SIGTERM; the server's own handler, which only cancelled a context that fx's stop did not wait for, is gone. fx's Run exits with the shutdown's code: the one a shutdown request carries, 0 for a signal, 1 when the app fails to start or stop. A listen error asks fx to shut down with exit code 1. A Sentry DSN that cannot be used fails the server's start hook, so fx stops what had already started. The server's stop hook stops the HTTP server, then waits for the images still being processed, both within ShutdownTimeout; images still being processed after that are logged and fail the stop, so the exit code is 1. Model: opus-5-5 --- TODO.md | 11 ++ cmd/pixad/main.go | 5 + internal/handlers/handlers.go | 6 + internal/imageprocessor/imageprocessor.go | 25 ++++ ...max_concurrent_processing_internal_test.go | 66 ++++++++++ internal/imgcache/service.go | 6 + internal/server/http.go | 14 ++- internal/server/server.go | 115 ++++++++---------- internal/server/shutdown_internal_test.go | 96 +++++++++++++++ 9 files changed, 275 insertions(+), 69 deletions(-) create mode 100644 internal/server/shutdown_internal_test.go diff --git a/TODO.md b/TODO.md index cebef13..ed8431c 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,17 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-03 shutdown sets the exit code and waits for image processing + (closes #86): fx alone handles SIGINT and SIGTERM, and the server's own + signal handler is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's + code: 0 for a signal, 1 when the HTTP server cannot listen or the app fails + to start or to stop; the server's stop hook, which fx waits for, stops the + HTTP server, waits for the images still being processed, both within 5 + seconds, then flushes Sentry; images still being processed after that are + logged with their count and make the exit code 1; a Sentry DSN that cannot be + used fails startup, so the stop hooks of what had already started run, + instead of exiting the process from a goroutine; the eviction loop is left to + #102. - 2026-10-03 every `script/cibuild` and `script/docker` run executes the checks (closes #101): the `Dockerfile` declares `CHECK_EPOCH` above `make fmt-check` and `make lint` in the lint stage and above `make test` in the build stage, diff --git a/cmd/pixad/main.go b/cmd/pixad/main.go index 96442dc..be56d5f 100644 --- a/cmd/pixad/main.go +++ b/cmd/pixad/main.go @@ -4,6 +4,8 @@ package main import ( "fmt" "os" + "os/signal" + "syscall" "github.com/spf13/cobra" "go.uber.org/fx" @@ -45,6 +47,9 @@ func run(_ *cobra.Command, _ []string) { _ = os.Setenv("PIXA_CONFIG_PATH", configPath) } + // A write to a closed stdout or stderr must not end the process. + signal.Ignore(syscall.SIGPIPE) + fx.New( fx.Provide( config.New, diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index b350595..fafb27b 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -81,6 +81,12 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { return s, nil } +// WaitForProcessing waits until no image is being processed, or until ctx +// ends, and returns how many images were still being processed then. +func (s *Handlers) WaitForProcessing(ctx context.Context) int { + return s.imgSvc.WaitForProcessing(ctx) +} + // initImageService initializes the image cache and service. func (s *Handlers) initImageService() error { // Create the cache. cache_max_bytes: 0 disables the disk cache diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 6a9a92d..f26ed57 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -331,6 +331,31 @@ func FormatToMIME(format Format) string { } } +// WaitForProcessing waits until no image is being processed, or until ctx +// ends, and returns how many images were still being processed then. It +// waits by taking each slot in processingSemaphore as it frees up until it +// holds them all, or until ctx ends, then gives back the slots it took. +func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int { + taken := 0 + + defer func() { + for range taken { + <-p.processingSemaphore + } + }() + + for taken < cap(p.processingSemaphore) { + select { + case p.processingSemaphore <- struct{}{}: + taken++ + case <-ctx.Done(): + return len(p.processingSemaphore) - taken + } + } + + return 0 +} + // acquireSlot takes a slot in processingSemaphore, waiting at most // processingWaitTimeout for one to free up, and returns the func that gives // it back. A free slot is taken even when ctx has ended; only the wait for diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go index dc826c4..ba5388c 100644 --- a/internal/imageprocessor/max_concurrent_processing_internal_test.go +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -300,3 +300,69 @@ func TestProcessReleasesSlotOnError(t *testing.T) { }) } } + +// TestWaitForProcessing holds a processing slot with a Process call that +// cannot finish reading its input. WaitForProcessing must report that image +// when its context ends first, wait for it otherwise, return 0 once it has +// finished, and give back the slots it took while waiting. +func TestWaitForProcessing(t *testing.T) { + t.Parallel() + + proc := New(Params{MaxConcurrentProcessing: 2}) + + gate := make(chan struct{}) + entered := make(chan struct{}, 1) + results := make(chan error, 1) + + openGate := sync.OnceFunc(func() { close(gate) }) + t.Cleanup(openGate) + + processInBackground(proc, &gatedReader{ + data: bytes.NewReader(createTestJPEG(t, 10, 10)), gate: gate, + entered: entered, counter: &readingCounter{}, + }, results) + waitForEntries(t, entered, 1) + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + stillProcessing := proc.WaitForProcessing(ctx) + t.Logf("WaitForProcessing() after its context ended: %d", stillProcessing) + + if stillProcessing != 1 { + t.Errorf("WaitForProcessing() after its context ended = %d, want 1", + stillProcessing) + } + + waited := make(chan int, 1) + + go func() { waited <- proc.WaitForProcessing(t.Context()) }() + + select { + case got := <-waited: + t.Fatalf("WaitForProcessing() = %d while an image was being processed", + got) + case <-time.After(100 * time.Millisecond): + } + + openGate() + + err := <-results + if err != nil { + t.Errorf("Process() error = %v, want nil", err) + } + + select { + case got := <-waited: + if got != 0 { + t.Errorf("WaitForProcessing() once processing finished = %d, want 0", + got) + } + case <-time.After(5 * time.Second): + t.Fatal("WaitForProcessing() did not return once processing finished") + } + + if held := len(proc.processingSemaphore); held != 0 { + t.Errorf("%d slots still held after WaitForProcessing() returned", held) + } +} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 90acf0b..77c7ce9 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -199,6 +199,12 @@ func (s *Service) Stats(ctx context.Context) (*CacheStats, error) { return s.cache.Stats(ctx) } +// WaitForProcessing waits until no image is being processed, or until ctx +// ends, and returns how many images were still being processed then. +func (s *Service) WaitForProcessing(ctx context.Context) int { + return s.processor.WaitForProcessing(ctx) +} + // ValidateRequest validates the request signature if required. func (s *Service) ValidateRequest(req *ImageRequest) error { // Check if host is allowed (no signature required) diff --git a/internal/server/http.go b/internal/server/http.go index 13162e0..33ce153 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -5,6 +5,8 @@ import ( "fmt" "net/http" "time" + + "go.uber.org/fx" ) // HTTP server configuration constants. @@ -36,19 +38,19 @@ func (s *Server) newHTTPServer() *http.Server { } } +// serveUntilShutdown serves on s.httpServer until it is shut down. When it +// stops for any other reason, such as its port being in use, it asks fx to +// shut down with exit code 1. func (s *Server) serveUntilShutdown() { - s.httpServer = s.newHTTPServer() - - s.SetupRoutes() - s.log.Info("http begin listen", "listenaddr", s.httpServer.Addr) err := s.httpServer.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { s.log.Error("listen error", "error", err) - if s.cancelFunc != nil { - s.cancelFunc() + err = s.shutdowner.Shutdown(fx.ExitCode(1)) + if err != nil { + s.log.Error("shutdown request failed", "error", err) } } } diff --git a/internal/server/server.go b/internal/server/server.go index bbaf44f..7be7d67 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,12 +3,10 @@ package server import ( "context" + "errors" "fmt" "log/slog" "net/http" - "os" - "os/signal" - "syscall" "time" "github.com/getsentry/sentry-go" @@ -27,6 +25,10 @@ const ( SentryFlushTimeout = 2 * time.Second ) +// errStillProcessing is returned by the server's stop hook when images are +// still being processed once ShutdownTimeout has passed. +var errStillProcessing = errors.New("images still being processed at shutdown") + // Params defines dependencies for Server. type Params struct { fx.In @@ -36,6 +38,7 @@ type Params struct { Config *config.Config Middleware *middleware.Middleware Handlers *handlers.Handlers + Shutdowner fx.Shutdowner } // Server is the main HTTP server. @@ -45,59 +48,58 @@ type Server struct { globals *globals.Globals mw *middleware.Middleware h *handlers.Handlers + shutdowner fx.Shutdowner startupTime time.Time - exitCode int sentryEnabled bool - cancelFunc context.CancelFunc httpServer *http.Server router *chi.Mux } -// New creates a new Server instance. +// New creates a new Server instance. Its start hook starts Sentry and the +// HTTP server; its stop hook, which fx runs on SIGINT, SIGTERM or a +// shutdown request, shuts them down. func New(lc fx.Lifecycle, params Params) (*Server, error) { s := &Server{ - log: params.Logger.Get(), - config: params.Config, - globals: params.Globals, - mw: params.Middleware, - h: params.Handlers, + log: params.Logger.Get(), + config: params.Config, + globals: params.Globals, + mw: params.Middleware, + h: params.Handlers, + shutdowner: params.Shutdowner, } lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { s.startupTime = time.Now() - go s.Run(context.WithoutCancel(ctx)) - return nil - }, - OnStop: func(_ context.Context) error { - if s.cancelFunc != nil { - s.cancelFunc() + err := s.enableSentry() + if err != nil { + return err } + s.SetupRoutes() + s.httpServer = s.newHTTPServer() + + go s.serveUntilShutdown() + return nil }, + OnStop: s.cleanShutdown, }) return s, nil } -// Run starts the server. -func (s *Server) Run(ctx context.Context) { - s.enableSentry() - s.serve(ctx) -} - // MaintenanceMode returns whether maintenance mode is enabled. func (s *Server) MaintenanceMode() bool { return s.config.MaintenanceMode } -func (s *Server) enableSentry() { +func (s *Server) enableSentry() error { s.sentryEnabled = false if s.config.SentryDSN == "" { - return + return nil } err := sentry.Init(sentry.ClientOptions{ @@ -105,55 +107,42 @@ func (s *Server) enableSentry() { Release: fmt.Sprintf("%s-%s", s.globals.Appname, s.globals.Version), }) if err != nil { - s.log.Error("sentry init failure", "error", err) - os.Exit(1) + return fmt.Errorf("sentry init failure: %w", err) } s.log.Info("sentry error reporting activated") s.sentryEnabled = true + + return nil } -func (s *Server) serve(ctx context.Context) int { - ctx, cancelFunc := context.WithCancel(ctx) - s.cancelFunc = cancelFunc +// cleanShutdown stops the HTTP server, waits for the images still being +// processed, then flushes Sentry. The first two share ShutdownTimeout. It +// returns errStillProcessing when images are still being processed after +// that, as their work is abandoned. +func (s *Server) cleanShutdown(ctx context.Context) error { + s.log.Info("shutting down") - go func() { - c := make(chan os.Signal, 1) - - signal.Ignore(syscall.SIGPIPE) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - - sig := <-c - s.log.Info("signal received", "signal", sig) - - if s.cancelFunc != nil { - s.cancelFunc() - } - }() - - go s.serveUntilShutdown() - - <-ctx.Done() - s.cleanShutdown(ctx) - - return s.exitCode -} - -func (s *Server) cleanShutdown(ctx context.Context) { - s.exitCode = 0 - - ctxShutdown, shutdownCancel := context.WithTimeout( - context.WithoutCancel(ctx), ShutdownTimeout) + ctxShutdown, shutdownCancel := context.WithTimeout(ctx, ShutdownTimeout) defer shutdownCancel() - if s.httpServer != nil { - err := s.httpServer.Shutdown(ctxShutdown) - if err != nil { - s.log.Error("server clean shutdown failed", "error", err) - } + err := s.httpServer.Shutdown(ctxShutdown) + if err != nil { + s.log.Error("server clean shutdown failed", "error", err) } + stillProcessing := s.h.WaitForProcessing(ctxShutdown) + if s.sentryEnabled { sentry.Flush(SentryFlushTimeout) } + + if stillProcessing > 0 { + s.log.Error("images still being processed at shutdown", + "count", stillProcessing) + + return errStillProcessing + } + + return nil } diff --git a/internal/server/shutdown_internal_test.go b/internal/server/shutdown_internal_test.go new file mode 100644 index 0000000..4087bb1 --- /dev/null +++ b/internal/server/shutdown_internal_test.go @@ -0,0 +1,96 @@ +package server + +import ( + "log/slog" + "net" + "testing" + "time" + + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/logger" +) + +// shutdownRecorder is an fx.Shutdowner that sends the options of each +// shutdown request on requests. +type shutdownRecorder struct { + requests chan []fx.ShutdownOption +} + +func (r shutdownRecorder) Shutdown(opts ...fx.ShutdownOption) error { + r.requests <- opts + + return nil +} + +// TestSentryInitFailureFailsStartup checks that a Sentry DSN that cannot be +// used makes the server's start hook fail, so fx stops what has already +// started, instead of the process exiting from a goroutine. +func TestSentryInitFailureFailsStartup(t *testing.T) { + t.Parallel() + + lc := fxtest.NewLifecycle(t) + + log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}}) + if err != nil { + t.Fatalf("logger.New() error = %v", err) + } + + _, err = New(lc, Params{ + Logger: log, + Globals: &globals.Globals{Appname: "pixad"}, + Config: &config.Config{SentryDSN: "not-a-dsn"}, + }) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + err = lc.Start(t.Context()) + t.Logf("Start() error = %v", err) + + if err == nil { + t.Fatal("Start() error = nil, want the Sentry initialization error") + } +} + +// TestListenErrorRequestsShutdownWithExitCode1 occupies the server's port +// and checks that the listen error asks fx to shut down with exit code 1. +func TestListenErrorRequestsShutdownWithExitCode1(t *testing.T) { + t.Parallel() + + busy, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", ":0") + if err != nil { + t.Fatalf("Listen() error = %v", err) + } + + t.Cleanup(func() { _ = busy.Close() }) + + addr, ok := busy.Addr().(*net.TCPAddr) + if !ok { + t.Fatalf("listener address %v is not a TCP address", busy.Addr()) + } + + requests := make(chan []fx.ShutdownOption, 1) + s := &Server{ + log: slog.New(slog.DiscardHandler), + config: &config.Config{Port: addr.Port}, + shutdowner: shutdownRecorder{requests: requests}, + } + s.httpServer = s.newHTTPServer() + + go s.serveUntilShutdown() + + select { + case opts := <-requests: + t.Logf("shutdown options = %v", opts) + + if len(opts) != 1 || opts[0] != fx.ExitCode(1) { + t.Errorf("shutdown options = %v, want [fx.ExitCode(1)]", opts) + } + case <-time.After(5 * time.Second): + t.Fatal("no shutdown was requested after the listen error") + } +} From 3a274aaa4434d91e3c5e2bcab14e3b7615285f20 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 06:24:36 +0200 Subject: [PATCH 018/131] Make the README's storage, formats, TLS and metrics match the code (closes #74) "Storage" names the cache directories pixa uses (cache/sources, cache/metadata, cache/variants) and how files are named in each; the schema comments name the same paths, and the output_content comment says the table is not written. Routes and the signature section give one list of output formats, jpg and original included, and say how those two are signed. The TLS sentence names allow_http as its exception. "Metrics" says what exists: generic HTTP and Go runtime metrics at /metrics, measured and served only when the metrics username and password are set. Model: opus-5-5 --- README.md | 31 ++++++++++++++++++--------- TODO.md | 8 +++++++ internal/db/migrations/001_schema.sql | 7 +++--- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 76a1ef3..b6610e4 100644 --- a/README.md +++ b/README.md @@ -71,13 +71,17 @@ prevent abuse, and allowlisted source hosts for open access. ### Storage - **Source content**: - `/cache/src-content///` + `/cache/sources///` - **Source metadata**: - `/cache/src-metadata//.json` - (fetch time, original headers, request, content hash) -- **Database**: `/state.sqlite3` (SQLite) -- **Output documents**: - `/cache/dst-content///` + `/cache/metadata//.json` + (host, path and query, content hash, upstream status and headers, fetch time) +- **Database**: `/state.sqlite3` (SQLite) +- **Transformed images**: + `/cache/variants///`, + each with a `.meta` file beside it holding its content type + +`` and `` are the first and second pairs of characters of the file's +name. Multiple source paths may reference the same content blob; the database tracks references rather than using filesystem refcounting. @@ -92,12 +96,15 @@ the metadata file stored beside it. /v1/image///.?sig=&exp= ``` -Images are only fetched from origins using TLS with valid certificates. +Images are only fetched from origins using TLS with valid certificates, unless +`allow_http` is set: then pixa fetches every image over plain HTTP, which is for +testing only. A request whose query string cannot be decoded, or gives any parameter more than once, is refused with 400. -- ``: one of `orig`, `png`, `jpeg`, `webp` +- ``: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, + `avif`, `gif` - ``: `orig` or `x` (e.g. `800x600`) An image is served with `Cache-Control: public, max-age=, immutable`. @@ -173,7 +180,8 @@ Where: - `query` — source query string, empty string if none - `width` — requested width in pixels, `0` for original - `height` — requested height in pixels, `0` for original -- `format` — output format (jpeg, png, webp, avif, gif, orig) +- `format` — output format, one of those listed under Routes, with `original` + signed as `orig` and `jpg` as `jpeg` - `expiration` — the URL's `exp` query parameter, the Unix timestamp when the signature expires; a request whose `exp` is not a whole number, an empty `exp=` included, is refused with 400 @@ -330,7 +338,10 @@ See `config.example.yml` for all options with defaults. - **Image processing**: govips (CGO wrapper for libvips) - **Database**: SQLite via modernc.org/sqlite - **Static assets**: embedded via `//go:embed` -- **Metrics**: Prometheus +- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics + (duration, response size, requests in flight) and the Go runtime and process + metrics; requests are measured and `/metrics` is served only when + `metrics.username` and `metrics.password` are set - **Logging**: stdlib slog ## Entrypoints diff --git a/TODO.md b/TODO.md index ed8431c..c70f11b 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,14 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 `README.md` matches the code (closes #74): "Storage" names the + cache directories pixa uses (`cache/sources`, `cache/metadata`, + `cache/variants`) and how files are named in each, and the comments in + `001_schema.sql` name the same paths; the routes and the signature section + list the same output formats, `jpg` and `original` included; the TLS + sentence names `allow_http` as its exception; "Metrics" says only generic + HTTP and Go runtime metrics exist, measured and served only when the metrics + username and password are set. - 2026-10-03 shutdown sets the exit code and waits for image processing (closes #86): fx alone handles SIGINT and SIGTERM, and the server's own signal handler is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index c4ca863..41a0e02 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -2,7 +2,7 @@ -- Creates all tables for the pixa caching image proxy -- Source content blobs --- Files stored at: cache/src-content/// +-- Files stored at: cache/sources/// -- last_accessed_at is NULL until the first LRU touch; eviction falls -- back to fetched_at for rows that have never been touched. CREATE TABLE IF NOT EXISTS source_content ( @@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed ON source_content(last_accessed_at); -- Source URL metadata - maps URLs to content hashes --- JSON stored at: cache/src-metadata//.json +-- JSON stored at: cache/metadata//.json CREATE TABLE IF NOT EXISTS source_metadata ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_host TEXT NOT NULL, @@ -56,7 +56,8 @@ CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed ON variant_content(last_accessed_at); -- Output/transformed content blobs --- Files stored at: cache/dst-content/// +-- Not written: transformed images are stored in cache/variants and +-- tracked in variant_content above. CREATE TABLE IF NOT EXISTS output_content ( content_hash TEXT PRIMARY KEY, content_type TEXT NOT NULL, From 5b17d1f555fcc96afb79af19537a9d8e8d7cf1b0 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 07:11:31 +0200 Subject: [PATCH 019/131] Remove idle host semaphores and delete .meta with its variant (closes #87) Each upstream host's semaphore now counts the fetches holding or waiting for one of its slots, and is removed from hostSems when the last of them gives its slot back or stops waiting, so a long-running pixad no longer keeps one semaphore per host it ever fetched from. VariantStorage.Delete removes the variant's .meta file too, a missing one not being an error; DeleteWithMeta, which eviction called for that, is gone. Model: opus-5-5 --- TODO.md | 7 ++ internal/httpfetcher/fetch_internal_test.go | 18 +++- internal/httpfetcher/httpfetcher.go | 52 +++++++++-- .../max_connections_internal_test.go | 93 +++++++++++++++++++ internal/imgcache/eviction.go | 2 +- internal/imgcache/storage.go | 16 +--- internal/imgcache/storage_internal_test.go | 70 ++++++++++++++ 7 files changed, 235 insertions(+), 23 deletions(-) diff --git a/TODO.md b/TODO.md index c70f11b..404abd1 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,13 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 upstream host semaphores and variant `.meta` files no longer + outlive their use (closes #87): the fetcher counts the fetches holding or + waiting for a slot of each upstream host's semaphore and removes the host's + semaphore once none is left, so fetches from many hosts no longer leave one + semaphore each until restart; `VariantStorage.Delete` removes the variant's + `.meta` file along with it, a missing `.meta` file not being an error, and + `DeleteWithMeta`, which eviction called for that, is gone. - 2026-10-04 `README.md` matches the code (closes #74): "Storage" names the cache directories pixa uses (`cache/sources`, `cache/metadata`, `cache/variants`) and how files are named in each, and the comments in diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index c0d6e9d..83c78c0 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -191,7 +191,23 @@ func fetchBody(t *testing.T, f *HTTPFetcher, path string) string { // semLen reports how many per-host semaphore slots are currently held. func semLen(f *HTTPFetcher, host string) int { - return len(f.getHostSemaphore(host)) + f.hostSemMu.Lock() + defer f.hostSemMu.Unlock() + + sem, ok := f.hostSems[host] + if !ok { + return 0 + } + + return len(sem.slots) +} + +// hostSemCount reports how many hosts have a semaphore in hostSems. +func hostSemCount(f *HTTPFetcher) int { + f.hostSemMu.Lock() + defer f.hostSemMu.Unlock() + + return len(f.hostSems) } func TestFetchRedirectToPrivateIPBlocked(t *testing.T) { diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index d65f2da..3cfee1c 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -160,10 +160,13 @@ func DefaultConfig() *Config { // HTTPFetcher implements Fetcher with SSRF protection and connection limits // per host and for all hosts together. type HTTPFetcher struct { - client *http.Client - config *Config - hostSems map[string]chan struct{} // per-host semaphores - hostSemMu sync.Mutex // protects hostSems map + client *http.Client + config *Config + // hostSems holds the semaphore of each host with a fetch holding or + // waiting for one of its slots; the entry is removed when the host's + // last such fetch gives its slot back or stops waiting. + hostSems map[string]*hostSemaphore + hostSemMu sync.Mutex // protects hostSems and each entry's count // allHostsSemaphore has one slot per connection allowed to all hosts // together (config.MaxConnections). allHostsSemaphore chan struct{} @@ -171,6 +174,14 @@ type HTTPFetcher struct { connectionWaitTimeout time.Duration } +// hostSemaphore is one host's connection slots +// (config.MaxConnectionsPerHost) and the number of fetches holding or +// waiting for one of them. +type hostSemaphore struct { + slots chan struct{} + count int +} + // New creates a new HTTPFetcher with SSRF protection. func New(config *Config) *HTTPFetcher { if config == nil { @@ -211,7 +222,7 @@ func New(config *Config) *HTTPFetcher { return &HTTPFetcher{ client: client, config: config, - hostSems: make(map[string]chan struct{}), + hostSems: make(map[string]*hostSemaphore), allHostsSemaphore: make(chan struct{}, config.MaxConnections), connectionWaitTimeout: ConnectionWaitTimeout, } @@ -307,6 +318,8 @@ func (f *HTTPFetcher) acquireConnection( select { case hostSem <- struct{}{}: case <-ctx.Done(): + f.putHostSemaphore(host) + return nil, ctx.Err() } @@ -314,32 +327,55 @@ func (f *HTTPFetcher) acquireConnection( case f.allHostsSemaphore <- struct{}{}: case <-time.After(f.connectionWaitTimeout): <-hostSem + f.putHostSemaphore(host) return nil, ErrTooManyConnections case <-ctx.Done(): <-hostSem + f.putHostSemaphore(host) return nil, ctx.Err() } return func() { <-hostSem + f.putHostSemaphore(host) <-f.allHostsSemaphore }, nil } -// getHostSemaphore returns the semaphore for a host, creating it if necessary. +// getHostSemaphore returns the semaphore for a host, creating it if +// necessary, and counts the caller among the fetches using it. The caller +// calls putHostSemaphore once it holds no slot and waits for none. func (f *HTTPFetcher) getHostSemaphore(host string) chan struct{} { f.hostSemMu.Lock() defer f.hostSemMu.Unlock() sem, ok := f.hostSems[host] if !ok { - sem = make(chan struct{}, f.config.MaxConnectionsPerHost) + sem = &hostSemaphore{ + slots: make(chan struct{}, f.config.MaxConnectionsPerHost), + } f.hostSems[host] = sem } - return sem + sem.count++ + + return sem.slots +} + +// putHostSemaphore stops counting the caller among the fetches using the +// host's semaphore, and removes the semaphore when no fetch uses it. +func (f *HTTPFetcher) putHostSemaphore(host string) { + f.hostSemMu.Lock() + defer f.hostSemMu.Unlock() + + sem := f.hostSems[host] + + sem.count-- + if sem.count == 0 { + delete(f.hostSems, host) + } } // buildResult validates the upstream response and assembles a FetchResult diff --git a/internal/httpfetcher/max_connections_internal_test.go b/internal/httpfetcher/max_connections_internal_test.go index d23816c..a776460 100644 --- a/internal/httpfetcher/max_connections_internal_test.go +++ b/internal/httpfetcher/max_connections_internal_test.go @@ -5,6 +5,7 @@ import ( "errors" "net" "strconv" + "sync" "testing" "time" ) @@ -119,6 +120,98 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { } } +// TestFetchRemovesIdleHostSemaphores checks that a host's semaphore is +// removed once no fetch holds or waits for one of its slots: after 100 +// concurrent fetches from 50 hosts have all finished, no semaphore is left. +func TestFetchRemovesIdleHostSemaphores(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + f, _ := newServerFetcher(t, srv, nil) + ctx := testContext(t) + + var wg sync.WaitGroup + + for i := range 100 { + wg.Go(func() { + res, err := f.Fetch(ctx, imageURLOnPort(1+i%50)) + if err != nil { + t.Errorf("Fetch() error = %v", err) + + return + } + + _ = res.Content.Close() + }) + } + + wg.Wait() + + if n := hostSemCount(f); n != 0 { + t.Errorf("%d host semaphores left after every fetch finished, want 0", n) + } +} + +// TestFetchRemovesHostSemaphoreWhenNoConnection checks that a fetch that +// ends without a connection leaves no semaphore behind: when it is refused +// after waiting for a connection shared by all hosts, when its context ends +// while it waits for its host's slot, and when its context ends while it +// waits for a connection shared by all hosts, long before the 10 second +// wait timeout. +func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + + cfg := DefaultConfig() + cfg.MaxConnections = 1 + cfg.MaxConnectionsPerHost = 1 + + f, _ := newServerFetcher(t, srv, cfg) + f.connectionWaitTimeout = 100 * time.Millisecond + + open, err := f.Fetch(testContext(t), imageURLOnPort(81)) + if err != nil { + t.Fatalf("first Fetch() error = %v", err) + } + + _, err = f.Fetch(testContext(t), imageURLOnPort(82)) + if !errors.Is(err, ErrTooManyConnections) { + t.Fatalf("Fetch() from another host: error = %v, "+ + "want ErrTooManyConnections", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + _, err = f.Fetch(ctx, imageURLOnPort(81)) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Fetch() from the busy host: error = %v, "+ + "want context.DeadlineExceeded", err) + } + + // Back to the 10 second wait, so the next fetch's context ends first. + f.connectionWaitTimeout = ConnectionWaitTimeout + + ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + _, err = f.Fetch(ctx, imageURLOnPort(83)) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Fetch() from a host with nothing open: error = %v, "+ + "want context.DeadlineExceeded", err) + } + + err = open.Content.Close() + if err != nil { + t.Fatalf("close first body: %v", err) + } + + if n := hostSemCount(f); n != 0 { + t.Errorf("%d host semaphores left after every fetch finished, want 0", n) + } +} + // TestFetchReleasesConnectionOnError checks that a fetch that fails after // taking its connection gives it back: with MaxConnections at 1, the slot // must be free after the failure and the next fetch must succeed. diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 5d185ea..9f90b8f 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -283,7 +283,7 @@ func (c *Cache) evictVariant(ctx context.Context, cacheKey VariantKey) error { c.metaCache.Remove(cacheKey) - err = c.variants.DeleteWithMeta(cacheKey) + err = c.variants.Delete(cacheKey) if err != nil { return err } diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index 7e1cb38..395178d 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -564,7 +564,8 @@ func (s *VariantStorage) Exists(key VariantKey) bool { return err == nil } -// Delete removes content at the given key. +// Delete removes the content at the given key together with its .meta +// sidecar file. A missing file is not an error. func (s *VariantStorage) Delete(key VariantKey) error { path := s.keyToPath(key) @@ -573,18 +574,7 @@ func (s *VariantStorage) Delete(key VariantKey) error { return fmt.Errorf("failed to delete content: %w", err) } - return nil -} - -// DeleteWithMeta removes the content at the given key together with -// its .meta sidecar file. A missing file is not an error. -func (s *VariantStorage) DeleteWithMeta(key VariantKey) error { - err := s.Delete(key) - if err != nil { - return err - } - - metaPath := s.keyToPath(key) + ".meta" + metaPath := path + ".meta" err = os.Remove(metaPath) if err != nil && !os.IsNotExist(err) { diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go index 0465bb1..c57e1ee 100644 --- a/internal/imgcache/storage_internal_test.go +++ b/internal/imgcache/storage_internal_test.go @@ -438,3 +438,73 @@ func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { t.Errorf("log missing %s; got %q", want, logBuf.String()) } } + +// storeTestVariant stores one variant, with its .meta file, in a new +// VariantStorage and returns the storage and the variant's key. +func storeTestVariant(t *testing.T) (*VariantStorage, VariantKey) { + t.Helper() + + storage, err := NewVariantStorage(t.TempDir(), slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatalf("NewVariantStorage() error = %v", err) + } + + key := CacheKey(&ImageRequest{SourceHost: testHostCDN, SourcePath: testPathCat}) + + _, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp") + if err != nil { + t.Fatalf("Store() error = %v", err) + } + + return storage, key +} + +// TestVariantStorage_DeleteRemovesMeta verifies that Delete removes the +// variant's .meta file along with the variant file. +func TestVariantStorage_DeleteRemovesMeta(t *testing.T) { + t.Parallel() + + storage, key := storeTestVariant(t) + metaPath := storage.keyToPath(key) + ".meta" + + _, err := os.Stat(metaPath) + if err != nil { + t.Fatalf("Store() wrote no .meta file: %v", err) + } + + err = storage.Delete(key) + if err != nil { + t.Fatalf("Delete() error = %v", err) + } + + if storage.Exists(key) { + t.Error("Exists() = true after delete, want false") + } + + _, err = os.Stat(metaPath) + if !os.IsNotExist(err) { + t.Errorf(".meta file left after Delete() (stat err=%v)", err) + } +} + +// TestVariantStorage_DeleteWithoutMeta verifies that Delete succeeds for a +// variant whose .meta file is missing. +func TestVariantStorage_DeleteWithoutMeta(t *testing.T) { + t.Parallel() + + storage, key := storeTestVariant(t) + + err := os.Remove(storage.keyToPath(key) + ".meta") + if err != nil { + t.Fatalf("removing .meta file: %v", err) + } + + err = storage.Delete(key) + if err != nil { + t.Fatalf("Delete() error = %v, want nil", err) + } + + if storage.Exists(key) { + t.Error("Exists() = true after delete, want false") + } +} From 6830bdc5de40f60c2051568dbe6b1f2a08b449fd Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 08:41:37 +0200 Subject: [PATCH 020/131] Remove unimplemented Purge and three unused interfaces (closes #73) Purge only returned an error and nothing called it, so it is gone from the ImageCache interface and from Service, along with its error value. It can be added back when something needs it; eviction reclaims disk space meanwhile. The SignatureValidator, Allowlist and Storage interfaces in imgcache.go had no implementers and no users, and described types that look different from the real ones, so they are deleted. No behaviour changes. Model: opus-5-5 --- TODO.md | 5 +++++ internal/imgcache/imgcache.go | 30 ------------------------------ internal/imgcache/service.go | 12 +++--------- 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/TODO.md b/TODO.md index 404abd1..be3eb08 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,11 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 dead code in `internal/imgcache` is gone (closes #73): `Purge`, + which only returned an error and which nothing called, is no longer part of + the `ImageCache` interface or `Service`; the `SignatureValidator`, + `Allowlist` and `Storage` interfaces, which nothing implemented or used, are + deleted. Nothing else changes. - 2026-10-04 upstream host semaphores and variant `.meta` files no longer outlive their use (closes #87): the fetcher counts the fetches holding or waiting for a slot of each upstream host's semaphore and removes the host's diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index a74e783..1647658 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "net/url" "time" ) @@ -154,9 +153,6 @@ type ImageCache interface { // Warm pre-fetches and caches an image without returning it Warm(ctx context.Context, req *ImageRequest) error - // Purge removes a cached image - Purge(ctx context.Context, req *ImageRequest) error - // Stats returns cache statistics Stats(ctx context.Context) (*CacheStats, error) } @@ -176,29 +172,3 @@ type CacheStats struct { // HitRate is HitCount / (HitCount + MissCount) HitRate float64 } - -// SignatureValidator validates request signatures -type SignatureValidator interface { - // Validate checks if the signature is valid for the request - Validate(req *ImageRequest) error - // Generate creates a signature for a request - Generate(req *ImageRequest) string -} - -// Allowlist checks if a URL is allowlisted (no signature required) -type Allowlist interface { - // IsAllowlisted returns true if the URL doesn't require a signature - IsAllowlisted(u *url.URL) bool -} - -// Storage handles persistent storage of cached content -type Storage interface { - // Store saves content and returns its hash - Store(ctx context.Context, content io.Reader) (hash string, err error) - // Load retrieves content by hash - Load(ctx context.Context, hash string) (io.ReadCloser, error) - // Delete removes content by hash - Delete(ctx context.Context, hash string) error - // Exists checks if content exists - Exists(ctx context.Context, hash string) (bool, error) -} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 77c7ce9..968dd95 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -56,11 +56,10 @@ type ServiceConfig struct { Logger *slog.Logger } -// Static errors for service construction and unimplemented operations. +// Static errors for service construction. var ( - errCacheRequired = errors.New("cache is required") - errSigningKeyRequired = errors.New("signing key is required") - errPurgeNotImplemented = errors.New("purge not implemented") + errCacheRequired = errors.New("cache is required") + errSigningKeyRequired = errors.New("signing key is required") ) // NewService creates a new image service. @@ -189,11 +188,6 @@ func (s *Service) Warm(ctx context.Context, req *ImageRequest) error { return err } -// Purge removes a cached image. Purging is not implemented yet. -func (s *Service) Purge(_ context.Context, _ *ImageRequest) error { - return errPurgeNotImplemented -} - // Stats returns cache statistics. func (s *Service) Stats(ctx context.Context) (*CacheStats, error) { return s.cache.Stats(ctx) From 1616e91a6af0a3cb84f3b9d03209ef342e4d583a Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 09:24:42 +0200 Subject: [PATCH 021/131] Stop cache eviction in progress at shutdown (closes #102) StartEviction runs the eviction goroutine with its own context, which StopEviction cancels in place of the old stop channel, so a pass in progress stops at its next database call, file, row or eviction candidate instead of running to completion, and no new pass starts. StopEviction takes a context: when it ends before the goroutine exits, StopEviction stops waiting and returns an error wrapping it. The handlers' stop hook passes fx's stop context, so an eviction still running at fx's stop deadline fails the stop and the exit code is 1. A stop logs at most one warning. Model: opus-5-5 --- TODO.md | 9 + internal/handlers/handlers.go | 17 +- internal/imgcache/cache.go | 8 +- internal/imgcache/eviction.go | 87 +++++-- internal/imgcache/eviction_internal_test.go | 247 +++++++++++++++++++- 5 files changed, 324 insertions(+), 44 deletions(-) diff --git a/TODO.md b/TODO.md index be3eb08..a15e955 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,15 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 shutdown stops cache eviction in progress (closes #102): + `StartEviction` runs the eviction goroutine with its own context, which + `StopEviction` cancels, so a pass in progress stops at its next database + call, file, row or eviction candidate instead of running to completion, and + no pass starts after it, so a stop logs at most one warning; + `StopEviction` takes a context and, when that context ends before the + goroutine exits, stops waiting and returns its error; the handlers' stop hook + passes fx's stop context, so an eviction still running when fx's stop + deadline ends fails the stop and makes the exit code 1. - 2026-10-04 dead code in `internal/imgcache` is gone (closes #73): `Purge`, which only returned an error and which nothing called, is no longer part of the `ImageCache` interface or `Service`; the `SignatureValidator`, diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index fafb27b..cecb5d7 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -58,23 +58,16 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { } lc.Append(fx.Hook{ - // The eviction goroutine must outlive OnStart, so it cannot - // inherit this hook's context. It makes its own instead, which - // leaves it uncancellable: an in-flight pass runs to completion - // during OnStop regardless of the shutdown deadline. Making the - // loop cancellable changes shutdown semantics and is tracked - // separately in issue #102, rather than being folded into the - // lint-conformance change that surfaced it. - //nolint:contextcheck // see issue #102 + //nolint:contextcheck // the eviction loop outlives OnStart; OnStop cancels it OnStart: func(_ context.Context) error { return s.initImageService() }, - OnStop: func(_ context.Context) error { - if s.imgCache != nil { - s.imgCache.StopEviction() + OnStop: func(ctx context.Context) error { + if s.imgCache == nil { + return nil } - return nil + return s.imgCache.StopEviction(ctx) }, }) diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index a2d2d61..5cd47ae 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -11,7 +11,6 @@ import ( "io" "log/slog" "path/filepath" - "sync" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -69,11 +68,11 @@ type Cache struct { // Eviction machinery. The channels are created in NewCache so // stores can signal write pressure without racing StartEviction. + // evictionCancel, set by StartEviction, cancels the eviction + // goroutine's context. evictionPressure chan struct{} - evictionStop chan struct{} evictionDone chan struct{} - evictionStarted bool - evictionStopOnce sync.Once + evictionCancel context.CancelFunc // metaCache holds the content types of the variants most recently // stored or served, so a hit does not read the variant's .meta file. @@ -112,7 +111,6 @@ func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { log: log, disabled: config.DisableDiskCache, evictionPressure: make(chan struct{}, 1), - evictionStop: make(chan struct{}), evictionDone: make(chan struct{}), metaCache: metaCache, contentLocks: newContentLock(), diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 9f90b8f..29de851 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -117,7 +117,10 @@ func (c *Cache) EvictToLimit(ctx context.Context) error { // evictBatch fetches one batch of LRU candidates across variants and // source blobs and evicts them oldest-first until excessBytes are -// freed or the batch is exhausted. It returns the bytes freed. +// freed or the batch is exhausted. It returns the bytes freed. A +// candidate that fails once ctx is cancelled (every one started after +// that fails at its first database call) ends the batch with ctx's +// error, without a warning. func (c *Cache) evictBatch(ctx context.Context, excessBytes int64) (int64, error) { candidates, err := c.evictionCandidates(ctx) if err != nil { @@ -133,6 +136,10 @@ func (c *Cache) evictBatch(ctx context.Context, excessBytes int64) (int64, error err := c.evictCandidate(ctx, candidate) if err != nil { + if ctx.Err() != nil { + return freed, ctx.Err() + } + c.log.Warn("failed to evict cache entry", "cache_key", candidate.cacheKey, "content_hash", candidate.contentHash, @@ -424,37 +431,44 @@ func (c *Cache) notifyWritePressure() { // startup and again on every periodic tick thereafter, and evicts to // the configured limit on the given periodic interval and on // write-pressure notifications. It is a no-op on a disabled cache or -// when already started. +// when already started. The goroutine outlives the caller, so it runs +// with its own context, which StopEviction cancels. func (c *Cache) StartEviction(interval time.Duration) { - if c.disabled || c.evictionStarted { + if c.disabled || c.evictionCancel != nil { return } - c.evictionStarted = true + ctx, cancel := context.WithCancel(context.Background()) + c.evictionCancel = cancel - go c.evictionLoop(interval) + go c.evictionLoop(ctx, interval) } -// StopEviction stops the background eviction goroutine and waits for -// it to exit. It is safe to call when eviction was never started, and -// safe to call more than once. -func (c *Cache) StopEviction() { - if !c.evictionStarted { - return +// StopEviction cancels the background eviction goroutine, which +// interrupts a pass in progress, and waits for it to exit or for ctx to +// end, whichever comes first. In the second case it returns an error +// wrapping ctx's error. It is safe to call when eviction was never +// started, and safe to call more than once. +func (c *Cache) StopEviction(ctx context.Context) error { + if c.evictionCancel == nil { + return nil } - c.evictionStopOnce.Do(func() { - close(c.evictionStop) - <-c.evictionDone - }) + c.evictionCancel() + + select { + case <-c.evictionDone: + return nil + case <-ctx.Done(): + return fmt.Errorf("cache eviction still running: %w", ctx.Err()) + } } -// evictionLoop is the body of the background eviction goroutine. -func (c *Cache) evictionLoop(interval time.Duration) { +// evictionLoop is the body of the background eviction goroutine. It +// returns when ctx is cancelled, and starts no pass after that. +func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { defer close(c.evictionDone) - ctx := context.Background() - c.runReconciliationPass(ctx) c.runEvictionPass(ctx) @@ -463,7 +477,7 @@ func (c *Cache) evictionLoop(interval time.Duration) { for { select { - case <-c.evictionStop: + case <-ctx.Done(): return case <-ticker.C: // Reconciliation walks the cache directories, so it only @@ -484,8 +498,13 @@ func (c *Cache) evictionLoop(interval time.Duration) { } // runEvictionPass runs one eviction pass, logging failures instead of -// propagating them (the loop must keep running). +// propagating them (the loop must keep running). It does nothing once +// ctx is cancelled. func (c *Cache) runEvictionPass(ctx context.Context) { + if ctx.Err() != nil { + return + } + err := c.EvictToLimit(ctx) if err != nil { c.log.Warn("cache eviction pass failed", "error", err) @@ -493,8 +512,13 @@ func (c *Cache) runEvictionPass(ctx context.Context) { } // runReconciliationPass runs one reconciliation pass, logging failures -// instead of propagating them (the loop must keep running). +// instead of propagating them (the loop must keep running). It does +// nothing once ctx is cancelled. func (c *Cache) runReconciliationPass(ctx context.Context) { + if ctx.Err() != nil { + return + } + err := c.reconcileAccounting(ctx) if err != nil { c.log.Warn("cache accounting reconciliation failed", "error", err) @@ -511,7 +535,8 @@ func (c *Cache) runReconciliationPass(ctx context.Context) { // know (and rows whose files are gone), and sweeps stale temp files // left behind by crashed writes. Running it periodically, not just // once, bounds how long such drift can accumulate unaccounted for on a -// long-running process to one eviction interval. +// long-running process to one eviction interval. Once ctx is cancelled, +// it stops at the next file or row and returns ctx's error. func (c *Cache) reconcileAccounting(ctx context.Context) error { if c.disabled { return nil @@ -546,6 +571,10 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error { return filepath.WalkDir( c.variants.baseDir, func(path string, entry fs.DirEntry, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil || entry.IsDir() { return err } @@ -634,6 +663,10 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error { } for _, key := range keys { + if ctx.Err() != nil { + return ctx.Err() + } + if c.variants.Exists(key) { continue } @@ -699,6 +732,10 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error { return filepath.WalkDir( c.srcContent.baseDir, func(path string, entry fs.DirEntry, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil || entry.IsDir() { return err } @@ -760,6 +797,10 @@ func (c *Cache) reconcileSourceRows(ctx context.Context) error { } for _, hash := range hashes { + if ctx.Err() != nil { + return ctx.Err() + } + if c.srcContent.Exists(hash) { continue } diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index f7084d3..5b51f18 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -4,9 +4,12 @@ import ( "bytes" "context" "database/sql" + "errors" "io/fs" + "log/slog" "os" "path/filepath" + "strings" "testing" "time" @@ -657,7 +660,7 @@ func TestEvictionRunsUnderWritePressure(t *testing.T) { // An interval far longer than the test ensures only write // pressure can trigger eviction here. cache.StartEviction(time.Hour) - defer cache.StopEviction() + defer func() { _ = cache.StopEviction(t.Context()) }() keys := []VariantKey{ testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree, @@ -690,7 +693,7 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) { // write-pressure notification fires and only the periodic ticker // can trigger eviction. cache.StartEviction(100 * time.Millisecond) - defer cache.StopEviction() + defer func() { _ = cache.StopEviction(t.Context()) }() keys := []VariantKey{ testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree, @@ -751,7 +754,7 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { } cache.StartEviction(time.Hour) - defer cache.StopEviction() + defer func() { _ = cache.StopEviction(t.Context()) }() deadline := time.Now().Add(5 * time.Second) @@ -808,7 +811,7 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { const interval = 100 * time.Millisecond cache.StartEviction(interval) - defer cache.StopEviction() + defer func() { _ = cache.StopEviction(t.Context()) }() // Let startup reconciliation run and settle on an empty cache // before introducing the untracked file, so the adoption we assert @@ -862,6 +865,242 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { } } +// TestStopEvictionInterruptsPassInProgress holds the test database's +// only connection, so the startup reconciliation pass waits for it, and +// checks that StopEviction stops that pass instead of waiting for the +// connection to come free, and that the stop logs one warning: the +// interrupted reconciliation's, with no eviction pass started after it. +func TestStopEvictionInterruptsPassInProgress(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + var logBuf bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil)) + + conn, err := cache.db.Conn(t.Context()) + if err != nil { + t.Fatalf("failed to take the database connection: %v", err) + } + + defer func() { _ = conn.Close() }() + + cache.StartEviction(time.Hour) + + // The pass is in progress once it waits for the connection. + deadline := time.Now().Add(5 * time.Second) + + for cache.db.Stats().WaitCount == 0 { + if time.Now().After(deadline) { + t.Fatal("the reconciliation pass never waited for the database") + } + + time.Sleep(10 * time.Millisecond) + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + err = cache.StopEviction(ctx) + t.Logf("StopEviction() error = %v", err) + + if err != nil { + t.Fatalf("StopEviction() error = %v, want nil: the pass waiting for "+ + "the database did not stop", err) + } + + t.Logf("log output: %s", logBuf.String()) + + warnings := strings.Count(logBuf.String(), `"level":"WARN"`) + if warnings != 1 { + t.Errorf("the stop logged %d warnings, want 1", warnings) + } +} + +// TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context +// while the oldest of three source blobs is being evicted, and checks that +// EvictToLimit then returns context.Canceled without evicting the other +// two or logging a warning for either of them. +func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1) + + var logBuf bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil)) + + hashes := []ContentHash{ + storeEvictionTestSource(t, cache, "cancel.example.com", "/a.jpg", + bytes.Repeat([]byte{0x61}, 1000)), + storeEvictionTestSource(t, cache, "cancel.example.com", "/b.jpg", + bytes.Repeat([]byte{0x62}, 1000)), + storeEvictionTestSource(t, cache, "cancel.example.com", "/c.jpg", + bytes.Repeat([]byte{0x63}, 1000)), + } + + base := time.Now().Add(-time.Hour) + + for i, hash := range hashes { + setSourceLastAccessed(t, cache, hash, base.Add(time.Duration(i)*time.Minute)) + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + cache.evictSourceBlobTestHook = func(ContentHash) { cancel() } + + err := cache.EvictToLimit(ctx) + t.Logf("EvictToLimit() error = %v", err) + t.Logf("log output: %s", logBuf.String()) + + if !errors.Is(err, context.Canceled) { + t.Errorf("EvictToLimit() error = %v, want context.Canceled", err) + } + + if cache.srcContent.Exists(hashes[0]) { + t.Errorf("source blob %s, evicted when the context was cancelled, "+ + "is still on disk", hashes[0]) + } + + for _, hash := range hashes[1:] { + if !cache.srcContent.Exists(hash) { + t.Errorf("source blob %s was evicted after the context was cancelled", hash) + } + } + + if strings.Contains(logBuf.String(), `"level":"WARN"`) { + t.Errorf("EvictToLimit logged a warning after the context was cancelled") + } + + assertNoDanglingReferences(t, cache) +} + +// TestStopEvictionReturnsWhenItsContextEnds pauses an eviction pass where +// cancellation cannot reach it, after a source blob's rows are deleted and +// before its file is removed, and checks that StopEviction returns its +// context's error when that context ends instead of waiting for the pass. +// Once the pass goes on, the goroutine exits and no row points at a +// missing file. +func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1) + + paused := make(chan struct{}) + resume := make(chan struct{}) + + cache.evictSourceBlobTestHook = func(ContentHash) { + close(paused) + <-resume + } + + hash := storeEvictionTestSource(t, cache, "stop.example.com", "/a.jpg", + bytes.Repeat([]byte{0x61}, 1000)) + + cache.StartEviction(time.Hour) + + select { + case <-paused: + case <-time.After(5 * time.Second): + t.Fatal("the eviction pass never reached the source blob") + } + + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + + err := cache.StopEviction(ctx) + t.Logf("StopEviction() error = %v", err) + + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("StopEviction() error = %v, want context.DeadlineExceeded", err) + } + + close(resume) + + err = cache.StopEviction(t.Context()) + if err != nil { + t.Fatalf("second StopEviction() error = %v, want nil", err) + } + + assertNoDanglingReferences(t, cache) + + if cache.srcContent.Exists(hash) { + t.Errorf("source blob %s is still on disk after its rows were deleted", hash) + } +} + +// TestReconciliationWalksStopOnceCancelled checks that both directory +// walks of a reconciliation pass return the context's error once it is +// cancelled, leaving in place a stale temp file they would otherwise +// remove. +func TestReconciliationWalksStopOnceCancelled(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + staleTime := time.Now().Add(-2 * staleTempFileAge) + + walks := map[string]func(context.Context) error{ + cache.variants.baseDir: cache.reconcileVariantFiles, + cache.srcContent.baseDir: cache.reconcileSourceFiles, + } + + for dir, walk := range walks { + tempFile := filepath.Join(dir, tempFilePrefix+"stale") + + err := os.WriteFile(tempFile, []byte("partial"), 0o600) + if err != nil { + t.Fatalf("failed to write temp file: %v", err) + } + + err = os.Chtimes(tempFile, staleTime, staleTime) + if err != nil { + t.Fatalf("failed to backdate temp file: %v", err) + } + + err = walk(ctx) + t.Logf("walk of %s: error = %v", dir, err) + + if !errors.Is(err, context.Canceled) { + t.Errorf("walk of %s: error = %v, want context.Canceled", dir, err) + } + + _, err = os.Stat(tempFile) + if err != nil { + t.Errorf("walk of %s went on after cancellation: %v", dir, err) + } + } +} + +// TestReconciliationPassLogsNoWarningOnceCancelled checks that a +// reconciliation pass run with an already cancelled context logs no +// warning, so a periodic tick the loop takes after a stop adds no +// warning to the one from the pass the stop interrupted. +func TestReconciliationPassLogsNoWarningOnceCancelled(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + var logBuf bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil)) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + cache.runReconciliationPass(ctx) + t.Logf("log output: %s", logBuf.String()) + + if strings.Contains(logBuf.String(), `"level":"WARN"`) { + t.Errorf("runReconciliationPass logged a warning with a cancelled context") + } +} + // TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises // the exact TOCTOU window between evictSourceBlob's row-deletion // transaction commit and its content file unlink: a concurrent From 363774c058d2583c5d241e5a669de8e7143ca1b5 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 09:58:36 +0200 Subject: [PATCH 022/131] Document every route, encrypted URLs and the config file search (closes #75) README.md "Routes" lists every route pixa registers with its method, purpose, what it needs and the status codes it answers with, read from the handlers, says q and fit are part of what is cached, and says the login and generator forms need HTTPS unless debug is on. A new "Encrypted URLs" section covers logging in with the signing key, making a URL on the generator page, how long it lasts, and the 410 once it has expired. "Configuration" gives the order in which pixa looks for its config file. config.example.yml now lists db_url and env and gives every key's default. scripts/manual-test.sh is left to #97. Model: opus-5-5 --- README.md | 124 ++++++++++++++++++++++++++++++++++++++++++--- TODO.md | 10 +++- config.example.yml | 36 +++++++++---- 3 files changed, 152 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b6610e4..588e2db 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,69 @@ the metadata file stored beside it. ### Routes +pixa answers these routes; any other path answers 404. A path in this list asked +with a method the list does not give answers 405, except `/static/`, which +answers any method as it answers `GET`. A browser's CORS preflight request +(`OPTIONS` with `Origin` and `Access-Control-Request-Method` headers) to any +path under `/v1/` answers 200, in maintenance mode too. + +- `GET /` — the login page, or the URL generator page with a login session + (see Encrypted URLs). Needs: nothing. Answers: 200. +- `POST /` — log in with the signing key typed into the login page. Needs: the + login page's form (below). Answers: 303 to `/` with a login session cookie + that lasts 30 days for the right key; 200 with the login page and an error for + a wrong key; 429 over the login limit (below). +- `POST /generate` — make an encrypted URL from the generator page's form. + Needs: a login session and the generator page's form (below); without a login + session it answers 303 to `/`. Answers: 200 with the page showing the URL; 400 + with the page naming a field that is not valid; 500 when the URL cannot be + made. +- `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`. +- `GET` or `HEAD` `/v1/image///.` — an image, fetched, + resized and converted (below). Needs: a signature, unless the host is + allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches + the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a + missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when + the upstream host, or a host it redirects to, is `localhost`, ends in + `.localhost` or `.local`, or has an address in a blocked network (see + `blocked_networks`); 502 when the upstream answered with an error status, and + for 5 minutes after that for the same source URL; 503 when pixa is busy or in + maintenance mode; 500 for any other failure. +- `GET /v1/e//` — an image through an encrypted URL (see Encrypted + URLs). Needs: nothing but the URL. Answers: 200; 400 for a token that does not + decrypt, or that asks for a size or fit that is not valid; 410 once it has + expired; 504 when the upstream has not sent its response headers within + `upstream_fetch_timeout`, but 500 when that time runs out while the image + itself is still arriving; 403, 502, 503 and 500 as for `/v1/image/`. +- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: + nothing. Answers: 200. +- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, + `uptime_seconds`, `uptime_human`, `version`, `appname` and + `maintenance_mode`. Needs: nothing. Answers: 200, always. +- `GET /static/` — the script the login and generator pages load. Needs: + nothing. Answers: 200, or 404 for a file that does not exist. +- `GET /metrics` — Prometheus metrics (see Architecture). Needs: HTTP basic + authentication with `metrics.username` and `metrics.password`. Answers: 200; + 401 without them; 404 when they are not set, as the route then does not exist. + +Both `POST` routes accept only a form that pixa's own page served: the page puts +a token in the form and sets a cookie to match, and a request without both is +refused with 403, so another site cannot submit the form from a visitor's +browser. The login and generator pages are meant to be opened over HTTPS: while +`debug` is off, a form sent from a page opened over plain HTTP is refused with +403, and while it is on, so is one sent from a page opened over HTTPS. Plain +HTTP is for development on the browser's own machine: the login session cookie +is always marked `Secure`, and over plain HTTP a browser keeps such a cookie +only for its own machine (`localhost`), if at all. A form is also refused with +403 when the page's host is not the `Host` header pixa receives, so a reverse +proxy in front of pixa must pass that header on unchanged. A form body over +1 MiB is refused with 413. The image routes answer the errors listed for them +with JSON holding `error`, `status` and `timestamp`. + +An image URL has this form: + ``` -/v1/image///.?sig=&exp= +/v1/image///.?sig=&exp=&q=&fit= ``` Images are only fetched from origins using TLS with valid certificates, unless @@ -106,6 +167,11 @@ than once, is refused with 400. - ``: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, `avif`, `gif` - ``: `orig` or `x` (e.g. `800x600`) +- `sig` and `exp`: the signature and its expiry, needed unless the host is + allowlisted (see Signature Specification) +- `q` and `fit`: the output quality and how the image is fitted to ``, + both optional (values under Signature Specification). Both are part of what + is cached, so each value of either is a separate cached image. An image is served with `Cache-Control: public, max-age=, immutable`. When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), @@ -139,6 +205,39 @@ its own `X-Forwarded-For`, whether it connects directly or through the proxy, because its own address is trusted too. Setting `trusted_proxies` to only the address pixa sees for requests that come through the proxy closes this. +### Encrypted URLs + +An encrypted URL is an image URL made on pixa's own web page by someone who +knows the signing key. It works for any upstream host, allowlisted or not, +without a signature, and whoever gets it can neither read the source URL from it +nor change what it asks for. + +1. Open `/` in a browser over HTTPS (or over plain HTTP while `debug` is on, see + Routes) and log in with the signing key (`signing_key`). The login session + lasts 30 days, or until `/logout`. +2. On the generator page, give the source image's URL, the width and height, the + format, quality and fit, and how long the URL lasts, then submit the form + (`POST /generate`). Width and height both empty or `0` keep the original + size; if only one of them is empty or `0`, that side is scaled to keep the + image's proportions. +3. The page shows the URL, `https:///v1/e//img.`, and when + it expires. `` is the host the page was opened on, and the URL starts + with `http` instead while `debug` is on. The name after the token is ignored + and only gives the URL a file extension, `jpg` for `orig`. + +The token holds the source's host, path and query and the size, format, +quality, fit and expiry, encrypted with a key derived from `signing_key`. The +source URL's scheme is not kept: the image is fetched like any other (see +Routes), and the blocked networks still apply. + +How long the URL lasts is chosen on the page, from 1 minute to 1 year, or +never. The expiry is fixed in the token when the URL is made and cannot be +changed or revoked afterwards. Until then the image is served with a `max-age` +that ends at the expiry (see Routes); after it the URL answers 410 +`URL has expired`. A URL made to last forever stops working only when +`signing_key` changes: changing it makes every encrypted URL already handed out +answer 400, and ends every login session. + ### Image Metadata pixa decodes and re-encodes every image it serves, and removes all metadata from @@ -237,6 +336,17 @@ startup naming it, as an unknown config key does. The one other accepted name is `PIXA_CONFIG_PATH`, the config file's path (like `--config`). The variables set by the file's `env:` section are checked the same way. +pixa reads at most one config file: the one given with `--config` (or `-c`), +otherwise the one `PIXA_CONFIG_PATH` names, otherwise the first of these that +pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`, +`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml` +and `config.yaml` in the working directory. A named file that does not exist, +cannot be read or does not parse aborts startup. Of the files pixa looks for on +its own, one it finds but cannot read or parse aborts startup; one it cannot +find, for any reason, is passed over without a message, even when the file is +there in a directory pixa may not enter. With no file, pixa uses the environment +and the defaults. + | Variable | Config key | Meaning | | ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- | | `PIXA_SIGNING_KEY` | `signing_key` | Required: secret for signed and encrypted URLs and login, 32+ characters | @@ -322,12 +432,12 @@ Key settings in more detail: seconds for one to free up; if none does, and `downstream_timeout` has not ended first, it is answered 503 the same way - `maintenance_mode` — while `true`, the image routes (`/v1/image/` and - `/v1/e/`) answer every request with 503, a `Retry-After` header and a JSON - error body. The health check (`/.well-known/healthcheck.json`) still answers - 200 and reports `"maintenance_mode": true`. It stays 200 because the image's - Docker `HEALTHCHECK` requests it: a 503 there would make the container - unhealthy, and upaas marks a deploy failed when its container is unhealthy. - The login and URL generator pages and `/metrics` keep working + `/v1/e/`) answer every request for an image with 503, a `Retry-After` header + and a JSON error body. The health check (`/.well-known/healthcheck.json`) + still answers 200 and reports `"maintenance_mode": true`. It stays 200 + because the image's Docker `HEALTHCHECK` requests it: a 503 there would make + the container unhealthy, and upaas marks a deploy failed when its container + is unhealthy. The login and URL generator pages and `/metrics` keep working See `config.example.yml` for all options with defaults. diff --git a/TODO.md b/TODO.md index a15e955..0c4ac74 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,14 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 routes, encrypted URLs and config file documented (closes #75): + "Routes" in `README.md` lists every route with its method, purpose, what it + needs and the status codes it answers with, and says `q` and `fit` are part + of what is cached; "Encrypted URLs" covers logging in, making one on the + generator page, how long it lasts and the 410 once it has expired; + "Configuration" gives the order in which pixa looks for its config file; + `config.example.yml` lists `db_url` and `env` and gives every key's default; + `scripts/manual-test.sh` is left to #97. - 2026-10-04 shutdown stops cache eviction in progress (closes #102): `StartEviction` runs the eviction goroutine with its own context, which `StopEviction` cancels, so a pass in progress stops at its next database @@ -441,7 +449,5 @@ P2: security: referer blacklist - integration tests for the image proxy flow - load tests to verify the 1k to 5k req/s target - P2: documentation - - configuration options - - API endpoints - deployment guide - example nginx or caddy reverse proxy config diff --git a/config.example.yml b/config.example.yml index ac32c98..2133dd9 100644 --- a/config.example.yml +++ b/config.example.yml @@ -12,27 +12,38 @@ # Durations are Go duration strings such as 30s or 2m and must be # positive; a bare number has no unit and aborts startup. Sizes are a # whole number of bytes. +# +# A key left out takes the default its comment gives. -# Server settings +# Port to listen on (default: 8080) port: 8080 + +# Debug logging and plain-HTTP local development (default: false) debug: false # While true, the image routes (/v1/image/ and /v1/e/) answer every request -# with 503 and a Retry-After header. The health check keeps answering 200 and -# reports maintenance_mode as true. It stays 200 because the image's Docker -# HEALTHCHECK requests it: a 503 there would make the container unhealthy, and -# upaas marks a deploy failed when its container is unhealthy. +# for an image with 503 and a Retry-After header. The health check keeps +# answering 200 and reports maintenance_mode as true. It stays 200 because +# the image's Docker HEALTHCHECK requests it: a 503 there would make the +# container unhealthy, and upaas marks a deploy failed when its container is +# unhealthy. (default: false) maintenance_mode: false # Data directory for SQLite database and cache files +# (default: /var/lib/pixa) state_dir: ./data +# SQLite database URL (default: +# file:/state.sqlite3?_journal_mode=WAL). An empty value aborts +# startup; leave the key out to use the default. +# db_url: "file:./data/state.sqlite3?_journal_mode=WAL" + # Image proxy settings # HMAC signing key for URL signatures (required, at least 32 characters) # Generate with: openssl rand -base64 32 signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32" -# Hosts that don't require signatures +# Hosts that don't require signatures (default: none) # Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com") allowlist_hosts: - s3.sneak.cloud @@ -45,7 +56,7 @@ allowlist_hosts: # SSRF protection. These are added to the always-enforced built-in ranges # (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and # similar), never replacing them. Each entry must be a valid CIDR in IPv4 -# or IPv6 form; an invalid entry aborts startup. +# or IPv6 form; an invalid entry aborts startup. (default: none) # blocked_networks: # - 100.64.0.0/10 # - 2001:db8::/32 @@ -72,6 +83,7 @@ allowlist_hosts: # - 2001:db8::/32 # Allow HTTP upstream (only for testing, always use HTTPS in production) +# (default: false) allow_http: false # Maximum concurrent connections per upstream host (default: 20) @@ -121,10 +133,16 @@ access_control_allow_origin: "*" # with a minimum of 500 MiB. # cache_max_bytes: 10737418240 -# Sentry error reporting (optional) +# Sentry DSN for error reporting (default: empty, which turns it off) sentry_dsn: "" -# Metrics endpoint authentication (optional) +# Username and password for /metrics, set together (default: unset). Metrics +# are measured and /metrics is served only when both are set. # metrics: # username: "admin" # password: "secret" + +# Environment variables set while this file loads, as described at the top +# (default: none) +# env: +# PIXA_DEBUG: "true" From c7173c47d84ecb9b06ff463e75b8b237a3999b23 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 12:41:54 +0200 Subject: [PATCH 023/131] Return and pass on request IDs, and give /v1/e/ ETag, 304 and HEAD (closes #84) Every response carries X-Request-Id, the upstream fetch sends it, and the "upstream fetched", "image converted" and "image served" lines log it as request_id. pixa's own RequestID middleware keeps a request's own ID only when it is at most 64 letters, digits, '-', '_' or '.', and otherwise makes a random one with crypto/rand, so nothing a client chooses freely and nothing about the host reaches upstream. /v1/e/ now sets ETag, answers a matching If-None-Match with 304 and is routed for HEAD, through notModified, which both image handlers call. No Vary is added: go-chi/cors already sends Vary: Origin. Model: opus-5-5 --- README.md | 18 ++- TODO.md | 15 +- internal/handlers/image.go | 34 +++-- .../image_cache_control_internal_test.go | 13 +- internal/handlers/imageenc.go | 102 +++++++++----- internal/handlers/imageenc_internal_test.go | 67 +++++++++ internal/handlers/request_id_internal_test.go | 131 ++++++++++++++++++ internal/httpfetcher/httpfetcher.go | 9 ++ .../httpfetcher/request_id_internal_test.go | 50 +++++++ internal/imgcache/service.go | 3 + internal/middleware/middleware.go | 29 ++++ .../middleware/request_id_internal_test.go | 118 ++++++++++++++++ internal/server/request_id_internal_test.go | 58 ++++++++ internal/server/routes.go | 3 +- internal/server/routes_internal_test.go | 27 ++++ 15 files changed, 619 insertions(+), 58 deletions(-) create mode 100644 internal/handlers/request_id_internal_test.go create mode 100644 internal/httpfetcher/request_id_internal_test.go create mode 100644 internal/middleware/request_id_internal_test.go create mode 100644 internal/server/request_id_internal_test.go create mode 100644 internal/server/routes_internal_test.go diff --git a/README.md b/README.md index 588e2db..2ab577b 100644 --- a/README.md +++ b/README.md @@ -120,8 +120,9 @@ path under `/v1/` answers 200, in maintenance mode too. `blocked_networks`); 502 when the upstream answered with an error status, and for 5 minutes after that for the same source URL; 503 when pixa is busy or in maintenance mode; 500 for any other failure. -- `GET /v1/e//` — an image through an encrypted URL (see Encrypted - URLs). Needs: nothing but the URL. Answers: 200; 400 for a token that does not +- `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL + (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when + `If-None-Match` matches the image's `ETag`; 400 for a token that does not decrypt, or that asks for a size or fit that is not valid; 410 once it has expired; 504 when the upstream has not sent its response headers within `upstream_fetch_timeout`, but 500 when that time runs out while the image @@ -137,6 +138,15 @@ path under `/v1/` answers 200, in maintenance mode too. authentication with `metrics.username` and `metrics.password`. Answers: 200; 401 without them; 404 when they are not set, as the route then does not exist. +Every response carries an `X-Request-ID` header holding the request's ID, which +a client can quote when reporting a problem: the request's own `X-Request-ID`, +as a reverse proxy in front of pixa may send, when it is at most 64 letters, +digits, `-`, `_` or `.`; otherwise a random one pixa makes for the request, +which tells nothing about the machine or the other requests. pixa's log line for +the request carries the same ID as `request_id`, and so do the lines it logs +when it fetches, converts and serves an image; the fetch sends it to the +upstream host as `X-Request-ID`. + Both `POST` routes accept only a form that pixa's own page served: the page puts a token in the form and sets a cookie to match, and a request without both is refused with 403, so another site cannot submit the form from a visitor's @@ -186,7 +196,9 @@ source) and one transcode: the first request does the work, and the others wait for its image or its error, holding no upstream connection or processing slot of their own. A waiting request stops waiting when its own client goes away. The work goes on for the others even if the first request's client goes away, -until that request's `downstream_timeout` ends. +until that request's `downstream_timeout` ends. The shared fetch sends the first +request's ID upstream, and the lines logged for the fetch and the transcode +carry that ID. The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is diff --git a/TODO.md b/TODO.md index 0c4ac74..2138144 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,20 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 request IDs returned and passed on, and `/v1/e/` revalidates + (closes #84): pixa's own `RequestID` middleware, in place of chi's, gives each + request an ID, its own `X-Request-ID` when that is at most 64 letters, digits, + `-`, `_` or `.` and a random one otherwise, stores it where chi's did and + sends it back as `X-Request-ID` on every response; the upstream fetch sends + that ID, and the "upstream fetched", "image converted" and "image served" log + lines carry it as `request_id`, a fetch shared by several requests carrying + the first request's; `/v1/e/` sets `ETag`, answers a matching `If-None-Match` + with 304 and is routed for `HEAD`, the `ETag` and 304 code being + `notModified`, which `/v1/image/` calls too; its token checks moved unchanged + into `parseImageEncRequest` to keep `HandleImageEnc` within the line limit; no + `Vary` is added, as no response depends on a request header except the image + routes' CORS headers, for which `go-chi/cors` already sends `Vary: Origin`; + `Vary: Accept` is left to #88. - 2026-10-04 routes, encrypted URLs and config file documented (closes #75): "Routes" in `README.md` lists every route with its method, purpose, what it needs and the status codes it answers with, and says `q` and `fit` are part @@ -438,7 +452,6 @@ P2: security: referer blacklist - P2: HTTP response handling - Last-Modified headers - Vary header for content negotiation - - X-Request-ID propagation - P2: auto format selection (format=auto based on Accept header) - P2: configuration - YAML config file support diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 3c94bc8..1a46a58 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -10,6 +10,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" @@ -247,6 +248,25 @@ func cacheControl(expires time.Time) string { return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second)) } +// notModified sets the ETag header to etag and, when the request's +// If-None-Match is that ETag, answers 304 Not Modified. It reports whether it +// answered. An empty etag sets no header and never answers. +func notModified(w http.ResponseWriter, r *http.Request, etag string) bool { + if etag == "" { + return false + } + + w.Header().Set("ETag", etag) + + if r.Header.Get("If-None-Match") != etag { + return false + } + + w.WriteHeader(http.StatusNotModified) + + return true +} + // writeImageResponse writes headers and streams the image content, // handling conditional and HEAD requests. func (s *Handlers) writeImageResponse( @@ -265,17 +285,8 @@ func (s *Handlers) writeImageResponse( w.Header().Set("Cache-Control", cacheControl(req.Expires)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) - if resp.ETag != "" { - w.Header().Set("ETag", resp.ETag) - - // Check for conditional request (If-None-Match) - if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" { - if ifNoneMatch == resp.ETag { - w.WriteHeader(http.StatusNotModified) - - return - } - } + if notModified(w, r, resp.ETag) { + return } // Handle HEAD request - return headers only @@ -298,6 +309,7 @@ func (s *Handlers) writeImageResponse( // Log cache status and timing after serving duration := time.Since(startTime) s.log.Info("image served", + "request_id", middleware.GetReqID(r.Context()), "cache_key", cacheKey, "cache_status", resp.CacheStatus, "duration_ms", duration.Milliseconds(), diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index bb6a6b9..7845df5 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -23,8 +23,10 @@ const photoPath = "/images/photo.jpg" // newSignedHostServer returns a router for both image routes, and the Handlers // behind it, whose fetcher serves a JPEG at photoPath on signedHost. signedHost // is not on the allowlist, so a /v1/image/ URL for it is served only with a -// valid signature. -func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) { +// valid signature. The handlers and the image service log to log. +func newSignedHostServer( + t *testing.T, log *slog.Logger, +) (*Handlers, http.Handler) { t.Helper() cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ @@ -44,6 +46,7 @@ func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) { signedHost + photoPath: &fstest.MapFile{Data: jpegData}, }), SigningKey: testSigningKey, + Logger: log, }) if err != nil { t.Fatalf("imgcache.NewService() error = %v", err) @@ -55,7 +58,7 @@ func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) { } h := &Handlers{ - log: slog.New(slog.DiscardHandler), + log: log, imgSvc: svc, encGen: encGen, } @@ -103,7 +106,7 @@ func getMaxAge(t *testing.T, srv http.Handler, target string) int { func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) { t.Parallel() - h, srv := newSignedHostServer(t) + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ SourceHost: signedHost, @@ -179,7 +182,7 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h, srv := newSignedHostServer(t) + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) token, err := h.encGen.Generate(&encurl.Payload{ SourceHost: signedHost, diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 56af31b..dadb6bd 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -9,6 +9,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" @@ -24,43 +25,8 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { ctx := r.Context() start := time.Now() - // Extract token from URL - token := chi.URLParam(r, "token") - if token == "" { - s.respondError(w, "missing token", http.StatusBadRequest) - - return - } - - // Decrypt and validate the payload - payload, err := s.encGen.Parse(token) - if err != nil { - if errors.Is(err, encurl.ErrExpired) { - s.log.Debug("encrypted URL expired", "error", err) - s.respondError(w, "URL has expired", http.StatusGone) - - return - } - - s.log.Debug("failed to decrypt URL", "error", err) - s.respondError(w, "invalid encrypted URL", http.StatusBadRequest) - - return - } - - // Convert payload to ImageRequest - req := payload.ToImageRequest() - - // Apply the same dimension and fit-mode bounds as the plain image - // route: a sealed payload is trusted for its origin, not for staying - // within limits, so an over-limit size or unknown fit mode is a 400 - // here rather than an out-of-memory or a 500 from the processor. - err = imgcache.ValidateImageRequest(req) - if err != nil { - s.log.Debug("encrypted URL failed validation", "error", err) - s.respondError(w, "invalid encrypted URL: "+err.Error(), - http.StatusBadRequest) - + req, ok := s.parseImageEncRequest(w, r) + if !ok { return } @@ -94,6 +60,17 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { w.Header().Set("Cache-Control", cacheControl(req.Expires)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) + if notModified(w, r, resp.ETag) { + return + } + + // A HEAD request gets the headers only + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusOK) + + return + } + // Stream the response written, err := io.Copy(w, resp.Content) if err != nil { @@ -105,6 +82,7 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { // Log completion duration := time.Since(start) s.log.Info("image served", + "request_id", middleware.GetReqID(ctx), "cache_key", imgcache.CacheKey(req), "host", req.SourceHost, "path", req.SourcePath, @@ -116,6 +94,56 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { } } +// parseImageEncRequest decrypts the token of an encrypted image URL into an +// ImageRequest and checks it. On a token that is missing, does not decrypt, +// has expired or asks for something not valid, it writes an error response +// and returns false. +func (s *Handlers) parseImageEncRequest( + w http.ResponseWriter, r *http.Request, +) (*imgcache.ImageRequest, bool) { + // Extract token from URL + token := chi.URLParam(r, "token") + if token == "" { + s.respondError(w, "missing token", http.StatusBadRequest) + + return nil, false + } + + // Decrypt and validate the payload + payload, err := s.encGen.Parse(token) + if err != nil { + if errors.Is(err, encurl.ErrExpired) { + s.log.Debug("encrypted URL expired", "error", err) + s.respondError(w, "URL has expired", http.StatusGone) + + return nil, false + } + + s.log.Debug("failed to decrypt URL", "error", err) + s.respondError(w, "invalid encrypted URL", http.StatusBadRequest) + + return nil, false + } + + // Convert payload to ImageRequest + req := payload.ToImageRequest() + + // Apply the same dimension and fit-mode bounds as the plain image + // route: a sealed payload is trusted for its origin, not for staying + // within limits, so an over-limit size or unknown fit mode is a 400 + // here rather than an out-of-memory or a 500 from the processor. + err = imgcache.ValidateImageRequest(req) + if err != nil { + s.log.Debug("encrypted URL failed validation", "error", err) + s.respondError(w, "invalid encrypted URL: "+err.Error(), + http.StatusBadRequest) + + return nil, false + } + + return req, true +} + // handleImageError converts image service errors to HTTP responses. func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { switch { diff --git a/internal/handlers/imageenc_internal_test.go b/internal/handlers/imageenc_internal_test.go index be2db22..d4f8c6b 100644 --- a/internal/handlers/imageenc_internal_test.go +++ b/internal/handlers/imageenc_internal_test.go @@ -96,3 +96,70 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) { t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest) } } + +// TestHandleImageEnc_IfNoneMatch_Returns304 verifies that an image served +// through an encrypted URL carries an ETag, and that a request whose +// If-None-Match is that ETag is answered 304 Not Modified with no body. +func TestHandleImageEnc_IfNoneMatch_Returns304(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + target := encPhotoURL(t, h) + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + + etag := rec.Header().Get("ETag") + t.Logf("GET: %d, ETag %q", rec.Code, etag) + + if rec.Code != http.StatusOK || etag == "" { + t.Fatalf("GET: status = %d, ETag = %q, want %d and an ETag", + rec.Code, etag, http.StatusOK) + } + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil) + req.Header.Set("If-None-Match", etag) + + rec = httptest.NewRecorder() + srv.ServeHTTP(rec, req) + t.Logf("GET with If-None-Match: %d, %d body bytes", rec.Code, rec.Body.Len()) + + if rec.Code != http.StatusNotModified || rec.Body.Len() != 0 { + t.Errorf("status = %d with %d body bytes, want %d with none", + rec.Code, rec.Body.Len(), http.StatusNotModified) + } +} + +// TestHandleImageEnc_HEAD_ReturnsHeadersOnly verifies that HEAD on an +// encrypted URL is answered 200 with the headers GET sends and no body. +func TestHandleImageEnc_HEAD_ReturnsHeadersOnly(t *testing.T) { + t.Parallel() + + h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + r := chi.NewRouter() + r.Head("/v1/e/{token}/*", h.HandleImageEnc()) + + rec := httptest.NewRecorder() + r.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodHead, encPhotoURL(t, h), nil)) + t.Logf("HEAD: %d, headers %v, %d body bytes", + rec.Code, rec.Header(), rec.Body.Len()) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + + for _, name := range []string{ + "Content-Type", "Content-Length", "Cache-Control", "ETag", + } { + if rec.Header().Get(name) == "" { + t.Errorf("HEAD response has no %s", name) + } + } + + if rec.Body.Len() != 0 { + t.Errorf("HEAD response body has %d bytes, want none", rec.Body.Len()) + } +} diff --git a/internal/handlers/request_id_internal_test.go b/internal/handlers/request_id_internal_test.go new file mode 100644 index 0000000..d26a2f6 --- /dev/null +++ b/internal/handlers/request_id_internal_test.go @@ -0,0 +1,131 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-chi/chi/v5/middleware" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the +// JPEG at photoPath on signedHost at 50x50, made with h's image service. +func signedPhotoURL(t *testing.T, h *Handlers) string { + t.Helper() + + signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ + SourceHost: signedHost, + SourcePath: photoPath, + Size: imgcache.Size{Width: 50, Height: 50}, + Format: imgcache.FormatJPEG, + }, time.Minute) + if err != nil { + t.Fatalf("GenerateSignedURL() error = %v", err) + } + + return signedURL +} + +// encPhotoURL returns an encrypted /v1/e/ URL, which never expires, for the +// JPEG at photoPath on signedHost at 50x50, made with h's generator. +func encPhotoURL(t *testing.T, h *Handlers) string { + t.Helper() + + token, err := h.encGen.Generate(&encurl.Payload{ + SourceHost: signedHost, + SourcePath: photoPath, + Width: 50, + Height: 50, + Format: imgcache.FormatJPEG, + }) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + return "/v1/e/" + token + "/img.jpg" +} + +// requestIDByMessage reads the JSON log lines in logs and returns the +// request_id of each line, by its message. +func requestIDByMessage(t *testing.T, logs io.Reader) map[string]string { + t.Helper() + + logged := make(map[string]string) + + dec := json.NewDecoder(logs) + for dec.More() { + var line map[string]any + + err := dec.Decode(&line) + if err != nil { + t.Fatalf("decoding log line: %v", err) + } + + msg, _ := line["msg"].(string) + requestID, _ := line["request_id"].(string) + logged[msg] = requestID + } + + return logged +} + +// TestImageLogLinesCarryRequestID verifies that the lines logged when an image +// is fetched, converted and served through either image route carry the +// request's ID as request_id, as the request log line does, so they can be +// found from it. +func TestImageLogLinesCarryRequestID(t *testing.T) { + t.Parallel() + + const requestID = "test-request-id" + + imageURLs := map[string]func(*testing.T, *Handlers) string{ + "/v1/image/": signedPhotoURL, + "/v1/e/": encPhotoURL, + } + + for route, imageURL := range imageURLs { + t.Run(route, func(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + h, srv := newSignedHostServer(t, + slog.New(slog.NewJSONHandler(&logs, nil))) + + ctx := context.WithValue(t.Context(), + middleware.RequestIDKey, requestID) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequestWithContext( + ctx, http.MethodGet, imageURL(t, h), nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + + t.Logf("logged:\n%s", logs.String()) + + logged := requestIDByMessage(t, &logs) + + for _, msg := range []string{ + "upstream fetched", "image converted", "image served", + } { + got, ok := logged[msg] + if !ok { + t.Errorf("no %q line logged", msg) + } else if got != requestID { + t.Errorf("%q line has request_id %q, want %q", + msg, got, requestID) + } + } + }) + } +} diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 3cfee1c..f76e8b1 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -18,6 +18,8 @@ import ( "strings" "sync" "time" + + "github.com/go-chi/chi/v5/middleware" ) // Fetcher configuration constants. @@ -267,6 +269,13 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro req.Header.Set("User-Agent", f.config.UserAgent) req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", ")) + // The ID of the request this fetch serves, so the fetch can be found in + // the upstream host's logs + requestID := middleware.GetReqID(ctx) + if requestID != "" { + req.Header.Set(middleware.RequestIDHeader, requestID) + } + // Use httptrace to capture connection details var remoteAddr string diff --git a/internal/httpfetcher/request_id_internal_test.go b/internal/httpfetcher/request_id_internal_test.go new file mode 100644 index 0000000..a3c998e --- /dev/null +++ b/internal/httpfetcher/request_id_internal_test.go @@ -0,0 +1,50 @@ +package httpfetcher + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi/v5/middleware" +) + +// TestFetchSendsRequestID verifies that a fetch sends the ID of the request +// it serves, which the RequestID middleware stores in the request context, +// to the upstream host as X-Request-Id, so the fetch can be found in that +// host's logs. +func TestFetchSendsRequestID(t *testing.T) { + t.Parallel() + + const requestID = "test-request-id" + + received := make(chan string, 1) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + received <- r.Header.Get("X-Request-Id") + + w.Header().Set("Content-Type", contentTypeJPEG) + _, _ = io.WriteString(w, imagePayload) + })) + t.Cleanup(srv.Close) + + f, _ := newServerFetcher(t, srv, nil) + + ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID) + + res, err := f.Fetch(ctx, upstreamURL("/image")) + if err != nil { + t.Fatalf("Fetch() error = %v", err) + } + + _ = res.Content.Close() + + got := <-received + t.Logf("upstream received X-Request-Id %q", got) + + if got != requestID { + t.Errorf("upstream X-Request-Id = %q, want %q", got, requestID) + } +} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 968dd95..562595c 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -13,6 +13,7 @@ import ( "github.com/dustin/go-humanize" "github.com/getsentry/sentry-go" + "github.com/go-chi/chi/v5/middleware" "golang.org/x/sync/singleflight" "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" @@ -461,6 +462,7 @@ func (s *Service) fetchAndProcess( // Log upstream fetch details s.log.Info("upstream fetched", + "request_id", middleware.GetReqID(ctx), "host", req.SourceHost, "path", req.SourcePath, "bytes", fetchBytes, @@ -545,6 +547,7 @@ func (s *Service) processAndStore( } s.log.Info("image converted", + "request_id", middleware.GetReqID(ctx), "host", req.SourceHost, "path", req.SourcePath, "src_format", processResult.InputFormat, diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index 492ba87..358cbdc 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -2,9 +2,12 @@ package middleware import ( + "context" + "crypto/rand" "log/slog" "net/http" "net/netip" + "regexp" "time" basicauth "github.com/99designs/basicauth-go" @@ -115,6 +118,32 @@ func (s *Middleware) RateLimit( }) } +// requestIDPattern is what a request's own X-Request-Id must look like to be +// kept as its ID: 1 to 64 letters, digits, '-', '_' or '.'. +var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`) + +// RequestID returns a middleware that gives each request an ID and sends it as +// the X-Request-Id response header, so a client can quote it when reporting a +// problem. The ID is the request's own X-Request-Id when that matches +// requestIDPattern, and otherwise a random one, which tells nothing about the +// machine or the traffic. It is stored in the request context under chi's +// RequestIDKey, where the logging middleware, the handlers and the upstream +// fetch read it. +func (s *Middleware) RequestID() func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get(middleware.RequestIDHeader) + if !requestIDPattern.MatchString(id) { + id = rand.Text() + } + + w.Header().Set(middleware.RequestIDHeader, id) + ctx := context.WithValue(r.Context(), middleware.RequestIDKey, id) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} + type loggingResponseWriter struct { http.ResponseWriter diff --git a/internal/middleware/request_id_internal_test.go b/internal/middleware/request_id_internal_test.go new file mode 100644 index 0000000..5b381d9 --- /dev/null +++ b/internal/middleware/request_id_internal_test.go @@ -0,0 +1,118 @@ +package middleware + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/go-chi/chi/v5/middleware" + + "sneak.berlin/go/pixa/internal/config" +) + +// sendRequestID sends a request through the RequestID middleware, carrying +// incoming as its X-Request-Id unless that is empty. It returns the ID the +// next handler found in the request context, which the upstream fetch sends +// and the log lines carry, and the X-Request-Id of the response. +func sendRequestID(t *testing.T, incoming string) (string, string) { + t.Helper() + + mw := &Middleware{log: slog.Default(), config: &config.Config{}} + + var inContext string + + handler := mw.RequestID()(http.HandlerFunc( + func(_ http.ResponseWriter, r *http.Request) { + inContext = middleware.GetReqID(r.Context()) + })) + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + if incoming != "" { + req.Header.Set("X-Request-Id", incoming) + } + + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + return inContext, rec.Header().Get("X-Request-Id") +} + +// TestRequestIDKeepsShortPlainID verifies that a request's own X-Request-Id of +// at most 64 letters, digits, '-', '_' or '.' is kept as its ID. +func TestRequestIDKeepsShortPlainID(t *testing.T) { + t.Parallel() + + for _, incoming := range []string{ + "client-request-id", + "A1_b2.c3-d4", + strings.Repeat("a", 64), + } { + inContext, inResponse := sendRequestID(t, incoming) + + if inContext != incoming || inResponse != incoming { + t.Errorf("incoming %q: context has %q, response %q, want both %q", + incoming, inContext, inResponse, incoming) + } + } +} + +// TestRequestIDReplacesLongOrUnusualID verifies that a request's own +// X-Request-Id that is over 64 characters or holds anything but letters, +// digits, '-', '_' or '.' is neither sent back nor sent upstream: the request +// gets a fresh ID instead. +func TestRequestIDReplacesLongOrUnusualID(t *testing.T) { + t.Parallel() + + for _, incoming := range []string{ + strings.Repeat("a", 65), + strings.Repeat("a", 9000), + "has space", + "a/b", + "a,b", + " + - -
-
-

Pixa URL Generator

- + + + {{if .GeneratedURL}} -
-

Generated URL

-
+
+

Generated URL

+
-
-

+

Expires: {{.ExpiresAt}}

{{end}} {{if .Error}} -
+
{{.Error}}
{{end}} -
+ {{ .CSRFField }}
-
-
+
-
-
-
+
-
-
-
+
-
-
- -

+

Generated URLs are encrypted and cannot be modified. They will expire at the specified time.

+ diff --git a/internal/templates/login.html b/internal/templates/login.html index 71a918f..9fafa5a 100644 --- a/internal/templates/login.html +++ b/internal/templates/login.html @@ -4,22 +4,22 @@ Pixa - Login - + - -
-

Pixa Image Proxy

+ +
+

Pixa Image Proxy

{{if .Error}} -
+
{{.Error}}
{{end}} -
+ {{ .CSRFField }}
-
-
From 233a9c05adc09fe804d262277d3d3233cede07a1 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 19:24:39 +0200 Subject: [PATCH 032/131] Test logging in, logging out, the URL generator and /v1/e/ tokens (closes #77) New handler tests in internal/handlers, with no network: GET / shows the login form without a session; a wrong key shows it again with an error and sets no session cookie; the right key answers 303 with a session cookie marked Secure, HttpOnly and SameSite=Strict, with which GET / shows the generator page; GET /logout empties the cookie with Max-Age=0; POST /generate without a session answers 303 to /; /v1/e/ serves a valid token's image, answers 410 for an expired token and 400 for one changed, cut short or made with another signing key; a URL made on the generator page is served by /v1/e/. No code changes. Model: opus-5-5 --- TODO.md | 11 + .../handlers/auth_session_internal_test.go | 243 ++++++++++++++++++ .../handlers/imageenc_token_internal_test.go | 126 +++++++++ 3 files changed, 380 insertions(+) create mode 100644 internal/handlers/auth_session_internal_test.go create mode 100644 internal/handlers/imageenc_token_internal_test.go diff --git a/TODO.md b/TODO.md index d4be840..464cf22 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,17 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have + handler tests (closes #77): new tests in `internal/handlers`, with no + network, check that `GET /` without a login session shows the login form; a + wrong key shows it again with an error and sets no session cookie; the right + key answers 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and + `SameSite=Strict`, with which `GET /` shows the generator page; `GET /logout` + answers 303 to `/` with an empty session cookie sent with `Max-Age=0`; + `POST /generate` without a login session answers 303 to `/`; `/v1/e/` serves + the image for a valid token, answers 410 for an expired one and 400 for one + with a character changed, cut short or made with another signing key; and a + URL made on the generator page is served by `/v1/e/`. No code changes. - 2026-10-04 `TODO.md` merges with git's union merge (closes #190): a root `.gitattributes`, copied from `sneak/prompts`, marks it `merge=union`, so two branches that each add an entry at the top of Completed Steps merge without a diff --git a/internal/handlers/auth_session_internal_test.go b/internal/handlers/auth_session_internal_test.go new file mode 100644 index 0000000..3bc57f8 --- /dev/null +++ b/internal/handlers/auth_session_internal_test.go @@ -0,0 +1,243 @@ +package handlers + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "testing" + + "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/session" +) + +// formatField is the generator form's format field name. +const formatField = "format" + +// Markers telling the login page from the generator page. +const ( + loginForm = `action="/"` + loginKeyInput = `name="key"` + generatorForm = `action="/generate"` +) + +// generatedURLPattern extracts the path of the URL the generator page shows. +// The test router runs with debug on, so the URL starts with http, and its +// host is httptest's default request host. +var generatedURLPattern = regexp.MustCompile( + `value="http://example\.com(/v1/e/[^"]+)"`) + +// findSessionCookie returns the session cookie rec sets, or nil if it sets +// none. +func findSessionCookie(rec *httptest.ResponseRecorder) *http.Cookie { + for _, c := range rec.Result().Cookies() { + if c.Name == session.CookieName { + return c + } + } + + return nil +} + +// TestHandleRoot_NoSession_ShowsLoginForm verifies that GET / without a +// login session shows the login form. +func TestHandleRoot_NoSession_ShowsLoginForm(t *testing.T) { + t.Parallel() + + _, srv := newCSRFTestRouter(t) + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + + body := rec.Body.String() + if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) { + t.Errorf("page is not the login form: %s", body) + } +} + +// TestLoginPost_WrongKey_ShowsErrorWithoutSession verifies that a wrong key +// shows the login form again with an error, and sets no session cookie. +func TestLoginPost_WrongKey_ShowsErrorWithoutSession(t *testing.T) { + t.Parallel() + + _, srv := newCSRFTestRouter(t) + + cookies, token := csrfCredentials(t, srv, nil) + + rec := postForm(srv, "/", cookies, url.Values{ + loginKeyField: {"wrong-signing-key-fedcba9876543210"}, + csrfTokenField: {token}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + + body := rec.Body.String() + if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) { + t.Errorf("page is not the login form: %s", body) + } + + if !strings.Contains(body, "Invalid signing key") { + t.Error("login form does not show the error") + } + + if c := findSessionCookie(rec); c != nil { + t.Errorf("wrong key set a session cookie: %s", c) + } +} + +// TestLoginPost_RightKey_SetsSessionCookie verifies that the right key answers +// 303 to / with a session cookie marked Secure, HttpOnly and SameSite=Strict, +// and that GET / with that cookie shows the generator page. +func TestLoginPost_RightKey_SetsSessionCookie(t *testing.T) { + t.Parallel() + + _, srv := newCSRFTestRouter(t) + + cookies, token := csrfCredentials(t, srv, nil) + + rec := postForm(srv, "/", cookies, url.Values{ + loginKeyField: {testSigningKey}, + csrfTokenField: {token}, + }) + + if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" { + t.Fatalf("status = %d, Location = %q, want %d to /", + rec.Code, rec.Header().Get("Location"), http.StatusSeeOther) + } + + sessionCookie := findSessionCookie(rec) + if sessionCookie == nil { + t.Fatal("right key set no session cookie") + } + + t.Logf("Set-Cookie: %s", sessionCookie) + + if !sessionCookie.Secure { + t.Error("session cookie is not Secure") + } + + if !sessionCookie.HttpOnly { + t.Error("session cookie is not HttpOnly") + } + + if sessionCookie.SameSite != http.SameSiteStrictMode { + t.Errorf("session cookie SameSite = %v, want Strict", sessionCookie.SameSite) + } + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + req.AddCookie(sessionCookie) + + rec = httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK || + !strings.Contains(rec.Body.String(), generatorForm) { + t.Errorf("GET / with the session cookie: status = %d, "+ + "want %d and the generator page", rec.Code, http.StatusOK) + } +} + +// TestHandleLogout_ClearsSessionCookie verifies that GET /logout answers 303 +// to / and replaces the session cookie with an empty one sent with +// Max-Age=0, which makes the browser delete it. +func TestHandleLogout_ClearsSessionCookie(t *testing.T) { + t.Parallel() + + h, _ := newCSRFTestRouter(t) + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/logout", nil) + req.AddCookie(newSessionCookie(t, h)) + + rec := httptest.NewRecorder() + h.HandleLogout().ServeHTTP(rec, req) + + if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" { + t.Fatalf("status = %d, Location = %q, want %d to /", + rec.Code, rec.Header().Get("Location"), http.StatusSeeOther) + } + + t.Logf("Set-Cookie: %s", rec.Header().Get("Set-Cookie")) + + sessionCookie := findSessionCookie(rec) + if sessionCookie == nil { + t.Fatal("logout did not set the session cookie") + } + + if sessionCookie.Value != "" { + t.Errorf("session cookie value = %q, want empty", sessionCookie.Value) + } + + // net/http reads a Max-Age=0 attribute back as MaxAge -1. + if sessionCookie.MaxAge != -1 { + t.Errorf("session cookie MaxAge = %d, want -1 (Max-Age=0)", + sessionCookie.MaxAge) + } +} + +// TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate +// with a valid CSRF token but no login session answers 303 to / and makes no +// URL. +func TestGeneratePost_NoSession_RedirectsToLogin(t *testing.T) { + t.Parallel() + + _, srv := newCSRFTestRouter(t) + + cookies, token := csrfCredentials(t, srv, nil) + + rec := postForm(srv, "/generate", cookies, url.Values{ + sourceURLField: {testSourceURL}, + csrfTokenField: {token}, + }) + + if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" { + t.Fatalf("status = %d, Location = %q, want %d to /", + rec.Code, rec.Header().Get("Location"), http.StatusSeeOther) + } + + if strings.Contains(rec.Body.String(), "/v1/e/") { + t.Error("a URL was made without a login session") + } +} + +// TestGeneratePost_URLServesImage verifies that the URL the generator page +// makes is served by /v1/e/. The image route runs on handlers of its own, +// made with the same signing key. +func TestGeneratePost_URLServesImage(t *testing.T) { + t.Parallel() + + _, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + rec := generatePost(t, url.Values{ + sourceURLField: {"https://" + signedHost + photoPath}, + widthField: {"50"}, + heightField: {"50"}, + formatField: {string(imgcache.FormatJPEG)}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK) + } + + match := generatedURLPattern.FindStringSubmatch(rec.Body.String()) + if match == nil { + t.Fatalf("generator page shows no URL: %s", rec.Body.String()) + } + + t.Logf("generated URL path: %s", match[1]) + + imageRec := httptest.NewRecorder() + imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, match[1], nil)) + + requireServedPhoto(t, imageRec) +} diff --git a/internal/handlers/imageenc_token_internal_test.go b/internal/handlers/imageenc_token_internal_test.go new file mode 100644 index 0000000..f2fcda2 --- /dev/null +++ b/internal/handlers/imageenc_token_internal_test.go @@ -0,0 +1,126 @@ +package handlers + +import ( + "image/jpeg" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/encurl" +) + +// requireServedPhoto requires that rec answers 200 with the JPEG at photoPath +// on signedHost at the 50x50 that encPhotoURL and the generator tests ask for. +func requireServedPhoto(t *testing.T, rec *httptest.ResponseRecorder) { + t.Helper() + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body %q", + rec.Code, http.StatusOK, rec.Body.String()) + } + + contentType := rec.Header().Get("Content-Type") + if contentType != "image/jpeg" { + t.Errorf("Content-Type = %q, want image/jpeg", contentType) + } + + img, err := jpeg.DecodeConfig(rec.Body) + if err != nil { + t.Fatalf("body is not a JPEG: %v", err) + } + + if img.Width != 50 || img.Height != 50 { + t.Errorf("image is %dx%d, want 50x50", img.Width, img.Height) + } +} + +// TestHandleImageEnc_ValidToken_ServesImage verifies that a token made with +// the signing key serves the image it asks for. +func TestHandleImageEnc_ValidToken_ServesImage(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, encPhotoURL(t, h), nil)) + + requireServedPhoto(t, rec) +} + +// TestHandleImageEnc_RejectedToken verifies that a token that has expired +// answers 410, and that a token with one character changed, a token cut +// short, and a token made with another signing key answer 400. The server +// would serve the photo for a token it accepted. +func TestHandleImageEnc_RejectedToken(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + photo := encurl.Payload{ + SourceHost: signedHost, + SourcePath: photoPath, + Width: 50, + Height: 50, + } + + valid, err := h.encGen.Generate(&photo) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + expiredPhoto := photo + expiredPhoto.ExpiresAt = time.Now().Add(-time.Minute).Unix() + + expired, err := h.encGen.Generate(&expiredPhoto) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + otherGen, err := encurl.NewGenerator("another-signing-key-fedcba9876543210") + if err != nil { + t.Fatalf("encurl.NewGenerator() error = %v", err) + } + + otherKey, err := otherGen.Generate(&photo) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + // Changing a character in the middle always changes the decoded bytes; + // the last character of unpadded base64 can carry unused bits. + middle := len(valid) / 2 + + replacement := "A" + if valid[middle] == 'A' { + replacement = "B" + } + + changed := valid[:middle] + replacement + valid[middle+1:] + + tests := []struct { + name string + token string + wantStatus int + }{ + {"expired", expired, http.StatusGone}, + {"one character changed", changed, http.StatusBadRequest}, + {"cut short", valid[:middle], http.StatusBadRequest}, + {"another signing key", otherKey, http.StatusBadRequest}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := getEncToken(srv, tt.token) + t.Logf("GET /v1/e/%s/img.jpg: %d %s", tt.token, rec.Code, rec.Body) + + if rec.Code != tt.wantStatus { + t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus) + } + }) + } +} From 847ad5b42819c12130e1051dd2143f320f7948e9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 19:41:49 +0200 Subject: [PATCH 033/131] Count what the cache holds in the default cache_max_bytes (closes #184) The default cache_max_bytes was 75% of the space free at startup. The cache's own files are not free space, so a fuller cache got a smaller limit after a restart and eviction then deleted most of it. The default is now 75% of the sum of the free space and what the cache already holds by its own size accounting, at least 500 MiB. The cache works it out when it opens, after the database is open, so the computation and its tests moved from internal/config to internal/imgcache. The config only records whether cache_max_bytes was set; newCacheConfig in the handlers turns the disk cache off only for an explicit 0, and is tested for an omitted, a zero and a positive value. Model: opus-5-5 --- README.md | 17 +- TODO.md | 8 + config.example.yml | 5 +- .../config/cache_max_bytes_internal_test.go | 167 ++-------------- internal/config/cachesize.go | 116 ----------- internal/config/config.go | 41 ++-- internal/config/env_internal_test.go | 2 +- .../handlers/cache_max_bytes_internal_test.go | 147 ++++++++++++++ internal/handlers/handlers.go | 27 ++- internal/imgcache/cache.go | 33 ++- internal/imgcache/cachesize.go | 89 +++++++++ internal/imgcache/cachesize_internal_test.go | 188 ++++++++++++++++++ 12 files changed, 528 insertions(+), 312 deletions(-) delete mode 100644 internal/config/cachesize.go create mode 100644 internal/handlers/cache_max_bytes_internal_test.go create mode 100644 internal/imgcache/cachesize.go create mode 100644 internal/imgcache/cachesize_internal_test.go diff --git a/README.md b/README.md index fe05438..75c0d1d 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,10 @@ database and the disk cache: and files still being written come on top, and eviction runs in the background, so the cache can pass the limit for a while: leave room on the volume beyond it. -- Set `cache_max_bytes` for a lasting deployment. Its default is 75% of the - space free when pixa starts, which the cache's own files reduce, so a fuller - cache gives a smaller limit after a restart. +- Set `cache_max_bytes` for a lasting deployment. Its default, worked out each + time pixa starts, is 75% of the sum of the space free on the volume and the + space the cached images already take, so a restart keeps the limit the cache + had, but anything else that fills or frees space on the volume moves it. A load balancer's health check can request `/.well-known/healthcheck.json`, which answers 200 whenever pixa is running, in maintenance mode too (see @@ -108,7 +109,7 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: - `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature, comma-separated - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it; - default 75% of free space + default 75% of (free space + what the cache holds) - the rest are in the table under Configuration below - **Health check:** the image's `HEALTHCHECK` requests `/.well-known/healthcheck.json`. upaas reads the container's health 60 @@ -425,7 +426,7 @@ tell. With no file, pixa uses the environment and the defaults. | `PORT` | `port` | Port to listen on; default `8080` | | `PIXA_STATE_DIR` | `state_dir` | Directory for the database and the disk cache; default `/var/lib/pixa` | | `PIXA_DB_URL` | `db_url` | SQLite database URL; default `state.sqlite3` in the state directory | -| `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of free space | +| `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of (free + cached) | | `PIXA_ALLOWLIST_HOSTS` | `allowlist_hosts` | Upstream hosts served without a signature | | `PIXA_BLOCKED_NETWORKS` | `blocked_networks` | CIDR ranges never fetched from, on top of the built-in ones | | `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 | @@ -490,8 +491,10 @@ Key settings in more detail: each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds - `signing_key` — HMAC secret for URL signatures - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the - disk cache entirely; omitted defaults to 75% of the free space on - the filesystem containing `/cache/` (minimum 500 MiB) + disk cache entirely; omitted defaults to 75% of the sum of the free space on + the filesystem containing `/cache/` and the bytes of source and + transformed images the cache already holds, worked out at startup (minimum + 500 MiB) - `upstream_connections` — the most connections to upstream hosts at once, all hosts together, on top of `upstream_connections_per_host`; default `64`. A fetch holds its connection until its image has been processed. A fetch that diff --git a/TODO.md b/TODO.md index 464cf22..3c1d04b 100644 --- a/TODO.md +++ b/TODO.md @@ -50,6 +50,14 @@ P2: security: referer blacklist share an identical line can end up one inside the other, which a rebase can do to an entry already on `next`. The Workflow above says to read the merged entries after every merge or rebase. +- 2026-10-04 the default `cache_max_bytes` no longer shrinks as the cache fills + (closes #184): for an omitted key, the cache works out the limit when it + opens, after the database is open, as 75% of the sum of the free space on the + filesystem containing `/cache/` and what the cache already holds by + its own size accounting, at least 500 MiB, so a cache filled to its limit + keeps that limit across a restart. The computation and its tests moved from + `internal/config` to `internal/imgcache`; the config only records whether the + key was set. - 2026-10-04 `TestEvictionRunsOnPeriodicSchedule` no longer races the evictor (closes #183): it wrote each variant file and then inserted its accounting row by hand, and a reconciliation pass between the two adopted the file first, so diff --git a/config.example.yml b/config.example.yml index 2133dd9..be42aa4 100644 --- a/config.example.yml +++ b/config.example.yml @@ -128,8 +128,9 @@ access_control_allow_origin: "*" # Maximum disk cache size in bytes. Explicit values are used exactly as # given; 0 disables the disk cache entirely (every request fetches and -# processes uncached). When omitted, the default is 75% of the free -# space on the filesystem containing /cache/ at startup, +# processes uncached). When omitted, the default is 75% of the sum of +# the free space on the filesystem containing /cache/ and +# the bytes of images the cache already holds, worked out at startup, # with a minimum of 500 MiB. # cache_max_bytes: 10737418240 diff --git a/internal/config/cache_max_bytes_internal_test.go b/internal/config/cache_max_bytes_internal_test.go index 50933c2..6af1554 100644 --- a/internal/config/cache_max_bytes_internal_test.go +++ b/internal/config/cache_max_bytes_internal_test.go @@ -1,26 +1,10 @@ package config import ( - "errors" - "log/slog" - "os" - "path/filepath" "strings" "testing" ) -// Static errors returned by the stub free-space probes below. -var ( - errTestStatfsFailed = errors.New("statfs failed") - errTestProbeNotExpected = errors.New("probe must not be called") -) - -// discardLogger returns a logger that swallows all output, for tests -// that exercise code paths which log. -func discardLogger() *slog.Logger { - return slog.New(slog.DiscardHandler) -} - // TestCacheMaxBytesExplicitValueUsedWithoutFloor verifies that an // explicitly configured cache_max_bytes value is used exactly as // given: the 500 MiB floor applies only to the computed default, never @@ -151,155 +135,30 @@ func TestCacheMaxBytesInvalidValuesAbortStartup(t *testing.T) { } } -// TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace verifies the -// computed default is 75% of the probed free space when that exceeds -// the floor. -func TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace(t *testing.T) { +// TestCacheMaxBytesExplicitIsRecorded verifies that an omitted +// cache_max_bytes is recorded as not explicit, so the cache works out +// the default when it opens, and that an explicit zero is recorded as +// explicit, so it disables the disk cache instead. +func TestCacheMaxBytesExplicitIsRecorded(t *testing.T) { t.Parallel() - // 4 GiB free -> 3 GiB default. - probe := func(string) (uint64, error) { return 4294967296, nil } + signingKeyLine := "signing_key: " + validTestSigningKey + "\n" - got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe) - if err != nil { - t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err) - } - - if got != 3221225472 { - t.Errorf("ComputeDefaultCacheMaxBytes = %d, want 3221225472 (75%% of 4 GiB)", - got) - } -} - -// TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault -// verifies that when 75% of free space is below 500 MiB, the computed -// default is floored at DefaultCacheMaxBytesFloor. -func TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - freeBytes uint64 - }{ - {name: "100 MiB free", freeBytes: 104857600}, - {name: "zero free", freeBytes: 0}, - {name: "just below floor threshold", freeBytes: 699050665}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - probe := func(string) (uint64, error) { return tc.freeBytes, nil } - - got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe) - if err != nil { - t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err) - } - - if got != DefaultCacheMaxBytesFloor { - t.Errorf("ComputeDefaultCacheMaxBytes = %d, want floor %d", - got, DefaultCacheMaxBytesFloor) - } - }) - } -} - -// TestComputeDefaultCacheMaxBytesPropagatesProbeError verifies that a -// failing free-space probe produces an error naming the config key, -// instead of a silently wrong default. -func TestComputeDefaultCacheMaxBytesPropagatesProbeError(t *testing.T) { - t.Parallel() - - probe := func(string) (uint64, error) { return 0, errTestStatfsFailed } - - _, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe) - if err == nil { - t.Fatal("probe failure must produce an error, got nil") - } - - t.Logf("got expected error: %v", err) - - if !strings.Contains(err.Error(), keyCacheMaxBytes) { - t.Errorf("error %q does not name the config key cache_max_bytes", err.Error()) - } -} - -// TestResolveCacheMaxBytesComputesDefaultWhenOmitted verifies that an -// omitted cache_max_bytes key resolves to the computed default, that -// the probe is pointed at /cache/ (which must be created -// first so statfs measures the right filesystem), and that the result -// lands on the Config. -func TestResolveCacheMaxBytesComputesDefaultWhenOmitted(t *testing.T) { - t.Parallel() - - c, err := configFromYAML(t, "signing_key: "+validTestSigningKey+"\n") + omitted, err := configFromYAML(t, signingKeyLine) if err != nil { t.Fatalf("minimal config should be valid, got error: %v", err) } - c.StateDir = t.TempDir() - wantCacheDir := filepath.Join(c.StateDir, "cache") - - var probedPath string - - // 4 GiB free -> 3 GiB default. - probe := func(path string) (uint64, error) { - probedPath = path - - return 4294967296, nil + if omitted.CacheMaxBytesExplicit { + t.Error("omitted cache_max_bytes recorded as explicit") } - err = c.resolveCacheMaxBytes(discardLogger(), probe) + zero, err := configFromYAML(t, signingKeyLine+"cache_max_bytes: 0\n") if err != nil { - t.Fatalf("resolveCacheMaxBytes returned error: %v", err) + t.Fatalf("cache_max_bytes: 0 must be accepted, got error: %v", err) } - if c.CacheMaxBytes != 3221225472 { - t.Errorf("CacheMaxBytes = %d, want computed default 3221225472", - c.CacheMaxBytes) - } - - if probedPath != wantCacheDir { - t.Errorf("free space probed at %q, want cache directory %q", - probedPath, wantCacheDir) - } - - info, err := os.Stat(wantCacheDir) - if err != nil || !info.IsDir() { - t.Errorf("cache directory %q was not created before probing: info=%v err=%v", - wantCacheDir, info, err) - } -} - -// TestResolveCacheMaxBytesDoesNotOverrideExplicitValue verifies that -// an explicitly configured value survives resolution untouched and -// that the free-space probe is never consulted for it. -func TestResolveCacheMaxBytesDoesNotOverrideExplicitValue(t *testing.T) { - t.Parallel() - - yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 1024\n" - - c, err := configFromYAML(t, yamlContent) - if err != nil { - t.Fatalf("explicit cache_max_bytes must be accepted, got error: %v", err) - } - - c.StateDir = t.TempDir() - - probe := func(string) (uint64, error) { - t.Error("free-space probe must not be consulted for explicit values") - - return 0, errTestProbeNotExpected - } - - err = c.resolveCacheMaxBytes(discardLogger(), probe) - if err != nil { - t.Fatalf("resolveCacheMaxBytes returned error: %v", err) - } - - if c.CacheMaxBytes != 1024 { - t.Errorf("CacheMaxBytes = %d, want explicit 1024 (no floor, no recompute)", - c.CacheMaxBytes) + if !zero.CacheMaxBytesExplicit { + t.Error("cache_max_bytes: 0 not recorded as explicit") } } diff --git a/internal/config/cachesize.go b/internal/config/cachesize.go deleted file mode 100644 index 8f7a014..0000000 --- a/internal/config/cachesize.go +++ /dev/null @@ -1,116 +0,0 @@ -package config - -import ( - "fmt" - "log/slog" - "math" - "os" - "path/filepath" - "syscall" -) - -// DefaultCacheMaxBytesFloor is the minimum computed default for the -// cache_max_bytes setting: 500 MiB. The floor applies only to the -// computed default (when the key is omitted from the configuration), -// never to explicitly configured values. -const DefaultCacheMaxBytesFloor int64 = 524288000 - -// cacheDirPerms is the permission mode for the cache directory created -// before probing free space, matching the state directory permissions. -const cacheDirPerms = 0o750 - -// freeSpaceFractionNumerator and freeSpaceFractionDenominator express -// the 75% share of free space used for the computed default limit as -// integer arithmetic (dividing before multiplying avoids overflow). -const ( - freeSpaceFractionNumerator uint64 = 3 - freeSpaceFractionDenominator uint64 = 4 -) - -// FreeSpaceProbeFunc reports the number of free bytes available on the -// filesystem containing path. It is a function type so tests can -// inject a fake probe instead of depending on the host disk. -type FreeSpaceProbeFunc func(path string) (uint64, error) - -// defaultFreeSpaceProbe reports free filesystem bytes via statfs on -// the given path, as available to unprivileged processes. -func defaultFreeSpaceProbe(path string) (uint64, error) { - var stat syscall.Statfs_t - - err := syscall.Statfs(path, &stat) - if err != nil { - return 0, err - } - - if stat.Bsize < 0 { - return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path) - } - - blockSize := uint64(stat.Bsize) - - return stat.Bavail * blockSize, nil -} - -// ComputeDefaultCacheMaxBytes returns the default cache size limit for -// the filesystem containing cacheDir: 75% of the free bytes reported -// by probe, with a floor of DefaultCacheMaxBytesFloor. -func ComputeDefaultCacheMaxBytes( - cacheDir string, probe FreeSpaceProbeFunc, -) (int64, error) { - freeBytes, err := probe(cacheDir) - if err != nil { - return 0, fmt.Errorf("config key %q: cannot determine free space for %q: %w", - "cache_max_bytes", cacheDir, err) - } - - computed := freeBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator - computed = min(computed, math.MaxInt64) - - // gosec cannot see that min() above bounds computed, so it reads - // this conversion as potentially overflowing. It cannot: computed is - // at most math.MaxInt64 on every path here. - //nolint:gosec // G115: clamped to MaxInt64 by min above - limit := int64(computed) - limit = max(limit, DefaultCacheMaxBytesFloor) - - return limit, nil -} - -// resolveCacheMaxBytes finalizes CacheMaxBytes after state_dir -// validation: an explicitly configured value is kept as-is (no floor -// applies), while an omitted key receives the computed default based -// on free space in /cache/. The cache directory is created -// first so statfs measures the filesystem that will actually hold the -// cache. The effective limit is logged either way. -func (c *Config) resolveCacheMaxBytes( - log *slog.Logger, probe FreeSpaceProbeFunc, -) error { - if !c.cacheMaxBytesExplicit { - cacheDir := filepath.Join(c.StateDir, "cache") - - err := os.MkdirAll(cacheDir, cacheDirPerms) - if err != nil { - return fmt.Errorf("config key %q: cannot create cache directory %q: %w", - keyCacheMaxBytes, cacheDir, err) - } - - limit, err := ComputeDefaultCacheMaxBytes(cacheDir, probe) - if err != nil { - return err - } - - c.CacheMaxBytes = limit - - log.Info("computed default cache size limit from free space", - "cache_max_bytes", limit, - "cache_dir", cacheDir, - ) - } - - log.Info("effective cache size limit", - "cache_max_bytes", c.CacheMaxBytes, - "cache_disabled", c.CacheMaxBytes == 0, - ) - - return nil -} diff --git a/internal/config/config.go b/internal/config/config.go index a9e583f..a9f7b72 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -93,9 +93,7 @@ var ( errMustBeSetTogether = errors.New("must be set together") errMustNotBeNegative = errors.New("must not be negative") errOverflowsInt64 = errors.New("overflows a 64-bit integer") - errNegativeBlockSize = errors.New( - "statfs reported negative block size") - errValueNull = errors.New( + errValueNull = errors.New( "value is null; omit the key entirely to use the default") errValuesNull = errors.New( "value is null; omit a key entirely to use its default") @@ -171,18 +169,19 @@ type Config struct { // address, and an explicit list replaces the default. TrustedProxies []netip.Prefix - // CacheMaxBytes is the disk cache size limit in bytes. Zero - // disables the disk cache entirely. When cache_max_bytes is - // omitted from the configuration, this holds the computed default - // (75% of free space on the filesystem containing - // /cache/, floored at DefaultCacheMaxBytesFloor). + // CacheMaxBytes is the disk cache size limit in bytes. Only an + // explicit zero (CacheMaxBytesExplicit true) disables the disk + // cache. Zero with CacheMaxBytesExplicit false means + // cache_max_bytes was omitted, and the cache works out the default + // limit when it opens. CacheMaxBytes int64 - // cacheMaxBytesExplicit records whether cache_max_bytes was + // CacheMaxBytesExplicit records whether cache_max_bytes was // explicitly set, in the environment or the configuration file. - // Explicit values are used exactly as given; only an omitted key - // gets the computed default (and its floor) in resolveCacheMaxBytes. - cacheMaxBytesExplicit bool + // Explicit values are used exactly as given; for an omitted key the + // cache works out the default limit when it opens (see + // imgcache.CacheConfig.UseDefaultMaxBytes). + CacheMaxBytesExplicit bool } // New creates a new Config instance from the environment and the @@ -219,9 +218,13 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) { return nil, err } - err = c.resolveCacheMaxBytes(log, defaultFreeSpaceProbe) - if err != nil { - return nil, err + // An omitted cache_max_bytes is worked out and logged when the + // cache opens. + if c.CacheMaxBytesExplicit { + log.Info("effective cache size limit", + "cache_max_bytes", c.CacheMaxBytes, + "cache_disabled", c.CacheMaxBytes == 0, + ) } if c.Debug { @@ -300,11 +303,11 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { TrustedProxies: trustedProxies, } - // The computed default for cache_max_bytes needs a validated - // state_dir, so it is resolved later (resolveCacheMaxBytes); here - // we only record whether the operator set the key explicitly. + // The default for an omitted cache_max_bytes is worked out when + // the cache opens; here we only record whether the operator set + // the key explicitly. if _, present := lookupValue(sc, keyCacheMaxBytes); present { - c.cacheMaxBytesExplicit = true + c.CacheMaxBytesExplicit = true } // Build DBURL from StateDir if not explicitly set. The derived URL diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go index e61ffbf..dd90dfa 100644 --- a/internal/config/env_internal_test.go +++ b/internal/config/env_internal_test.go @@ -98,7 +98,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { UpstreamConnections: 10, MaxConcurrentProcessing: 3, CacheMaxBytes: 1024, - cacheMaxBytesExplicit: true, + CacheMaxBytesExplicit: true, BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, TrustedProxies: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}, AccessControlAllowOrigin: "https://app.example.com", diff --git a/internal/handlers/cache_max_bytes_internal_test.go b/internal/handlers/cache_max_bytes_internal_test.go new file mode 100644 index 0000000..af0a8ab --- /dev/null +++ b/internal/handlers/cache_max_bytes_internal_test.go @@ -0,0 +1,147 @@ +package handlers + +import ( + "os" + "path/filepath" + "testing" + + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/logger" +) + +// TestNewCacheConfigFromCacheMaxBytes checks the cache configuration +// built from cache_max_bytes: omitted, the cache works out the default +// limit; 0 turns the disk cache off; a positive value is the limit, +// unchanged. +func TestNewCacheConfigFromCacheMaxBytes(t *testing.T) { + t.Parallel() + + const oneGiB = 1 << 30 + + cases := []struct { + name string + cacheMaxBytes int64 + cacheMaxBytesExplicit bool + wantMaxBytes int64 + wantUseDefaultMaxBytes bool + wantDisableDiskCache bool + }{ + { + name: "cache_max_bytes omitted", + cacheMaxBytes: 0, + cacheMaxBytesExplicit: false, + wantMaxBytes: 0, + wantUseDefaultMaxBytes: true, + wantDisableDiskCache: false, + }, + { + name: "cache_max_bytes: 0", + cacheMaxBytes: 0, + cacheMaxBytesExplicit: true, + wantMaxBytes: 0, + wantUseDefaultMaxBytes: false, + wantDisableDiskCache: true, + }, + { + name: "cache_max_bytes: 1 GiB", + cacheMaxBytes: oneGiB, + cacheMaxBytesExplicit: true, + wantMaxBytes: oneGiB, + wantUseDefaultMaxBytes: false, + wantDisableDiskCache: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + CacheMaxBytes: tc.cacheMaxBytes, + CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit, + } + + got := newCacheConfig(cfg, nil) + t.Logf("MaxBytes = %d, UseDefaultMaxBytes = %v, DisableDiskCache = %v", + got.MaxBytes, got.UseDefaultMaxBytes, got.DisableDiskCache) + + if got.MaxBytes != tc.wantMaxBytes { + t.Errorf("MaxBytes = %d, want %d", got.MaxBytes, tc.wantMaxBytes) + } + + if got.UseDefaultMaxBytes != tc.wantUseDefaultMaxBytes { + t.Errorf("UseDefaultMaxBytes = %v, want %v", + got.UseDefaultMaxBytes, tc.wantUseDefaultMaxBytes) + } + + if got.DisableDiskCache != tc.wantDisableDiskCache { + t.Errorf("DisableDiskCache = %v, want %v", + got.DisableDiskCache, tc.wantDisableDiskCache) + } + }) + } +} + +// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes starts the handlers +// once with cache_max_bytes omitted and once with cache_max_bytes: 0, +// and checks by whether the cache directories were created that the +// disk cache is on in the first case and off in the second. +func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + cacheMaxBytesExplicit bool + wantDiskCache bool + }{ + {name: "cache_max_bytes omitted", cacheMaxBytesExplicit: false, wantDiskCache: true}, + {name: "cache_max_bytes: 0", cacheMaxBytesExplicit: true, wantDiskCache: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + CacheMaxBytes: 0, + CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit, + } + + lc := fxtest.NewLifecycle(t) + + log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}}) + if err != nil { + t.Fatalf("logger.New() error = %v", err) + } + + db, err := database.New(lc, database.Params{Logger: log, Config: cfg}) + if err != nil { + t.Fatalf("database.New() error = %v", err) + } + + _, err = New(lc, Params{Logger: log, Database: db, Config: cfg}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + lc.RequireStart() + t.Cleanup(lc.RequireStop) + + _, err = os.Stat(filepath.Join(stateDir, "cache", "variants")) + gotDiskCache := err == nil + + if gotDiskCache != tc.wantDiskCache { + t.Errorf("cache directories created = %v, want %v", + gotDiskCache, tc.wantDiskCache) + } + }) + } +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index cecb5d7..489e3c3 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -80,18 +80,25 @@ func (s *Handlers) WaitForProcessing(ctx context.Context) int { return s.imgSvc.WaitForProcessing(ctx) } +// newCacheConfig builds the image cache's configuration from cfg. +// cache_max_bytes: 0 disables the disk cache entirely; any other value +// is the eviction limit in bytes; when it is omitted, the cache works +// out the default limit itself. +func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig { + return imgcache.CacheConfig{ + StateDir: cfg.StateDir, + CacheTTL: imgcache.DefaultCacheTTL, + NegativeTTL: imgcache.DefaultNegativeTTL, + MaxBytes: cfg.CacheMaxBytes, + UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit, + DisableDiskCache: cfg.CacheMaxBytesExplicit && cfg.CacheMaxBytes == 0, + Logger: log, + } +} + // initImageService initializes the image cache and service. func (s *Handlers) initImageService() error { - // Create the cache. cache_max_bytes: 0 disables the disk cache - // entirely; any other value is the eviction limit in bytes. - cache, err := imgcache.NewCache(s.db.DB(), imgcache.CacheConfig{ - StateDir: s.config.StateDir, - CacheTTL: imgcache.DefaultCacheTTL, - NegativeTTL: imgcache.DefaultNegativeTTL, - MaxBytes: s.config.CacheMaxBytes, - DisableDiskCache: s.config.CacheMaxBytes == 0, - Logger: s.log, - }) + cache, err := imgcache.NewCache(s.db.DB(), newCacheConfig(s.config, s.log)) if err != nil { return err } diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 5cd47ae..3c31956 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -37,11 +37,16 @@ type CacheConfig struct { NegativeTTL time.Duration // MaxBytes is the disk cache size limit in bytes that eviction - // enforces. Zero means no limit is enforced (no eviction). The - // config layer supplies the computed default when the operator - // omits cache_max_bytes. + // enforces. Zero means no limit is enforced (no eviction). MaxBytes int64 + // UseDefaultMaxBytes makes NewCache replace MaxBytes with the + // default limit: 75% of the sum of the space free on the filesystem + // holding the cache and the bytes the cache already holds, at least + // DefaultCacheMaxBytesFloor. The config layer sets this when the + // operator omits cache_max_bytes. + UseDefaultMaxBytes bool + // DisableDiskCache turns the disk cache off entirely: no cache // directories are created, lookups always miss, stores are // no-ops, and no eviction machinery runs. The config layer sets @@ -95,6 +100,14 @@ type Cache struct { // NewCache creates a new cache instance. func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { + return newCache(db, config, defaultFreeSpaceProbe) +} + +// newCache is NewCache with the free-space probe passed in, so tests +// can fake the free space the default limit is worked out from. +func newCache( + db *sql.DB, config CacheConfig, probe FreeSpaceProbeFunc, +) (*Cache, error) { log := config.Logger if log == nil { log = slog.Default() @@ -145,6 +158,20 @@ func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { c.variants = variants c.srcMetadata = srcMetadata + if config.UseDefaultMaxBytes { + limit, err := c.computeDefaultMaxBytes(context.Background(), probe) + if err != nil { + return nil, err + } + + c.config.MaxBytes = limit + + log.Info("computed default cache size limit from free space and cache contents", + "cache_max_bytes", limit, + "cache_dir", filepath.Join(config.StateDir, "cache"), + ) + } + return c, nil } diff --git a/internal/imgcache/cachesize.go b/internal/imgcache/cachesize.go new file mode 100644 index 0000000..437687e --- /dev/null +++ b/internal/imgcache/cachesize.go @@ -0,0 +1,89 @@ +package imgcache + +import ( + "context" + "errors" + "fmt" + "math" + "path/filepath" + "syscall" +) + +// DefaultCacheMaxBytesFloor is the minimum computed default for the +// cache_max_bytes setting: 500 MiB. The floor applies only to the +// computed default (when the key is omitted from the configuration), +// never to explicitly configured values. +const DefaultCacheMaxBytesFloor int64 = 524288000 + +// freeSpaceFractionNumerator and freeSpaceFractionDenominator express +// the 75% share used for the computed default limit as integer +// arithmetic (dividing before multiplying avoids overflow). +const ( + freeSpaceFractionNumerator uint64 = 3 + freeSpaceFractionDenominator uint64 = 4 +) + +var errNegativeBlockSize = errors.New("statfs reported negative block size") + +// FreeSpaceProbeFunc reports the number of free bytes available on the +// filesystem containing path. It is a function type so tests can +// inject a fake probe instead of depending on the host disk. +type FreeSpaceProbeFunc func(path string) (uint64, error) + +// defaultFreeSpaceProbe reports free filesystem bytes via statfs on +// the given path, as available to unprivileged processes. +func defaultFreeSpaceProbe(path string) (uint64, error) { + var stat syscall.Statfs_t + + err := syscall.Statfs(path, &stat) + if err != nil { + return 0, err + } + + if stat.Bsize < 0 { + return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path) + } + + blockSize := uint64(stat.Bsize) + + return stat.Bavail * blockSize, nil +} + +// computeDefaultMaxBytes returns the default cache size limit: 75% of +// the sum of the free bytes probe reports for /cache/ and +// the bytes the cache already holds, with a floor of +// DefaultCacheMaxBytesFloor. Counting what the cache holds keeps the +// limit from shrinking as the cache fills. +func (c *Cache) computeDefaultMaxBytes( + ctx context.Context, probe FreeSpaceProbeFunc, +) (int64, error) { + cacheDir := filepath.Join(c.config.StateDir, "cache") + + freeBytes, err := probe(cacheDir) + if err != nil { + return 0, fmt.Errorf( + "default cache_max_bytes: cannot determine free space for %q: %w", + cacheDir, err) + } + + usedBytes, err := c.UsageBytes(ctx) + if err != nil { + return 0, err + } + + // Both terms are at most math.MaxInt64, so the sum cannot overflow. + //nolint:gosec // G115: UsageBytes sums file sizes, never negative + spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes) + + computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator + computed = min(computed, math.MaxInt64) + + // gosec cannot see that min() above bounds computed, so it reads + // this conversion as potentially overflowing. It cannot: computed is + // at most math.MaxInt64 on every path here. + //nolint:gosec // G115: clamped to MaxInt64 by min above + limit := int64(computed) + limit = max(limit, DefaultCacheMaxBytesFloor) + + return limit, nil +} diff --git a/internal/imgcache/cachesize_internal_test.go b/internal/imgcache/cachesize_internal_test.go new file mode 100644 index 0000000..2416366 --- /dev/null +++ b/internal/imgcache/cachesize_internal_test.go @@ -0,0 +1,188 @@ +package imgcache + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// Static errors returned by the stub free-space probes below. +var ( + errTestStatfsFailed = errors.New("statfs failed") + errTestProbeNotExpected = errors.New("probe must not be called") +) + +// TestComputeDefaultMaxBytesCountsWhatTheCacheHolds verifies that the +// default limit is 75% of the free space plus what the cache already +// holds, so a cache filled to its limit keeps that limit across a +// restart instead of shrinking to 75% of the space left free. +func TestComputeDefaultMaxBytesCountsWhatTheCacheHolds(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + // Empty cache, 4 GiB free -> 3 GiB default. + got, err := cache.computeDefaultMaxBytes(t.Context(), + func(string) (uint64, error) { return 4294967296, nil }) + if err != nil { + t.Fatalf("computeDefaultMaxBytes returned error: %v", err) + } + + t.Logf("default for an empty cache with 4 GiB free: %d", got) + + if got != 3221225472 { + t.Errorf("default for an empty cache = %d, want 3221225472 (75%% of 4 GiB)", + got) + } + + // The cache now holds those 3 GiB, which leaves 1 GiB free. + _, err = cache.db.ExecContext(t.Context(), + `INSERT INTO variant_content (cache_key, size_bytes, content_type) + VALUES (?, ?, ?)`, + string(testVariantKeyOne), 3221225472, testContentTypeWebP, + ) + if err != nil { + t.Fatalf("failed to insert variant accounting row: %v", err) + } + + got, err = cache.computeDefaultMaxBytes(t.Context(), + func(string) (uint64, error) { return 1073741824, nil }) + if err != nil { + t.Fatalf("computeDefaultMaxBytes returned error: %v", err) + } + + t.Logf("default for a cache holding 3 GiB with 1 GiB free: %d", got) + + if got != 3221225472 { + t.Errorf("default for a cache holding 3 GiB with 1 GiB free = %d, "+ + "want 3221225472 (75%% of 1 GiB + 3 GiB)", got) + } +} + +// TestComputeDefaultMaxBytesAppliesFloor verifies that when 75% of the +// free space plus what the cache holds is below 500 MiB, the default +// is floored at DefaultCacheMaxBytesFloor. +func TestComputeDefaultMaxBytesAppliesFloor(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + freeBytes uint64 + }{ + {name: "100 MiB free", freeBytes: 104857600}, + {name: "zero free", freeBytes: 0}, + {name: "just below floor threshold", freeBytes: 699050665}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + got, err := cache.computeDefaultMaxBytes(t.Context(), + func(string) (uint64, error) { return tc.freeBytes, nil }) + if err != nil { + t.Fatalf("computeDefaultMaxBytes returned error: %v", err) + } + + if got != DefaultCacheMaxBytesFloor { + t.Errorf("computeDefaultMaxBytes = %d, want floor %d", + got, DefaultCacheMaxBytesFloor) + } + }) + } +} + +// TestComputeDefaultMaxBytesPropagatesProbeError verifies that a +// failing free-space probe produces an error naming cache_max_bytes, +// instead of a silently wrong default. +func TestComputeDefaultMaxBytesPropagatesProbeError(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + _, err := cache.computeDefaultMaxBytes(t.Context(), + func(string) (uint64, error) { return 0, errTestStatfsFailed }) + if err == nil { + t.Fatal("probe failure must produce an error, got nil") + } + + t.Logf("got expected error: %v", err) + + if !strings.Contains(err.Error(), "cache_max_bytes") { + t.Errorf("error %q does not name cache_max_bytes", err.Error()) + } +} + +// TestNewCacheComputesDefaultMaxBytesWhenAsked verifies that with +// UseDefaultMaxBytes set, the cache's limit becomes the computed +// default, and that the probe is pointed at /cache/, which +// must be created first so statfs measures the right filesystem. +func TestNewCacheComputesDefaultMaxBytesWhenAsked(t *testing.T) { + t.Parallel() + + stateDir := t.TempDir() + wantCacheDir := filepath.Join(stateDir, "cache") + + var probedPath string + + // 4 GiB free -> 3 GiB default. + probe := func(path string) (uint64, error) { + probedPath = path + + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + t.Errorf("cache directory %q was not created before probing: info=%v err=%v", + path, info, err) + } + + return 4294967296, nil + } + + cache, err := newCache(evictionTestDB(t), CacheConfig{ + StateDir: stateDir, + UseDefaultMaxBytes: true, + }, probe) + if err != nil { + t.Fatalf("newCache returned error: %v", err) + } + + if cache.config.MaxBytes != 3221225472 { + t.Errorf("MaxBytes = %d, want computed default 3221225472", + cache.config.MaxBytes) + } + + if probedPath != wantCacheDir { + t.Errorf("free space probed at %q, want cache directory %q", + probedPath, wantCacheDir) + } +} + +// TestNewCacheKeepsExplicitMaxBytes verifies that without +// UseDefaultMaxBytes the cache keeps MaxBytes exactly as given and +// never consults the free-space probe. +func TestNewCacheKeepsExplicitMaxBytes(t *testing.T) { + t.Parallel() + + probe := func(string) (uint64, error) { + t.Error("free-space probe must not be consulted for explicit values") + + return 0, errTestProbeNotExpected + } + + cache, err := newCache(evictionTestDB(t), CacheConfig{ + StateDir: t.TempDir(), + MaxBytes: 1024, + }, probe) + if err != nil { + t.Fatalf("newCache returned error: %v", err) + } + + if cache.config.MaxBytes != 1024 { + t.Errorf("MaxBytes = %d, want explicit 1024 (no floor, no recompute)", + cache.config.MaxBytes) + } +} From 66e71b42072139c39c6c78dbf38a606fad3a227c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 19:59:36 +0200 Subject: [PATCH 034/131] Make the periodic reconciliation test wait for the startup pass (closes #189) TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup slept for three eviction intervals before writing its file, so on a slow start the startup pass could still be running and adopt the file itself, and the test passed without a periodic pass. It now holds the test database's only connection until the startup pass waits for it, writes the file and lets the connection go, as TestEvictionRunsOnPeriodicSchedule does, so only a periodic reconciliation pass can adopt the file. Test only. Model: opus-5-5 --- TODO.md | 8 ++++++ internal/imgcache/eviction_internal_test.go | 31 +++++++++++++++------ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/TODO.md b/TODO.md index 3c1d04b..8c3c6df 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,14 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup` + only passes through a periodic pass (closes #189): it slept for three + eviction intervals before writing its file, and a startup pass still running + then could adopt the file itself. It now holds the test database's only + connection until the startup pass waits for it after walking the empty + variant directory, writes the file and lets the connection go, as + `TestEvictionRunsOnPeriodicSchedule` does, so only a periodic reconciliation + pass can adopt the file. Test only. - 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have handler tests (closes #77): new tests in `internal/handlers`, with no network, check that `GET /` without a login session shows the login form; a diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 4b0bbf7..af91ac5 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -847,15 +847,28 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { cache, _ := newEvictionTestCache(t, 1<<30) - const interval = 100 * time.Millisecond + // Hold the test database's only connection, so the startup pass + // waits for it after walking the still empty variant directory: the + // file written while it waits is first seen by a periodic pass. + conn, err := cache.db.Conn(t.Context()) + if err != nil { + t.Fatalf("failed to take the database connection: %v", err) + } - cache.StartEviction(interval) + defer func() { _ = conn.Close() }() + + cache.StartEviction(100 * time.Millisecond) defer func() { _ = cache.StopEviction(t.Context()) }() - // Let startup reconciliation run and settle on an empty cache - // before introducing the untracked file, so the adoption we assert - // below can only be the work of a later, periodic pass. - time.Sleep(3 * interval) + deadline := time.Now().Add(5 * time.Second) + + for cache.db.Stats().WaitCount == 0 { + if time.Now().After(deadline) { + t.Fatal("the startup pass never waited for the database") + } + + time.Sleep(10 * time.Millisecond) + } // Simulate a variant whose accounting insert failed after the // process was already running and serving requests: the content @@ -864,14 +877,16 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { // insert had failed and only the file write had succeeded. untracked := bytes.Repeat([]byte{0x41}, 900) - _, err := cache.variants.Store( + _, err = cache.variants.Store( "aabbccdd0099", bytes.NewReader(untracked), "image/webp", ) if err != nil { t.Fatalf("failed to store untracked variant file: %v", err) } - deadline := time.Now().Add(5 * time.Second) + _ = conn.Close() + + deadline = time.Now().Add(5 * time.Second) var usage int64 From 625fd42ace36691d918910c22871cd2fe4eaab0d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 20:58:37 +0200 Subject: [PATCH 035/131] Wait on a busy SQLite database and turn on WAL mode (closes #198) Requests and the eviction pass write on separate connections, and with no busy timeout a write that met another one failed at once with "database is locked" and was lost. internal/database now adds _pragma=busy_timeout(5000) to every db_url, the default or one the operator sets, so such a write waits up to five seconds. The default db_url's _journal_mode=WAL is not a parameter the driver reads, so it is now _pragma=journal_mode(WAL). README.md and config.example.yml say what pixa adds to db_url. Model: opus-5-5 --- README.md | 6 + TODO.md | 6 + config.example.yml | 7 +- internal/config/config.go | 3 +- .../config/config_validation_internal_test.go | 36 ++++- .../concurrent_writes_internal_test.go | 153 ++++++++++++++++++ internal/database/database.go | 12 +- 7 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 internal/database/concurrent_writes_internal_test.go diff --git a/README.md b/README.md index 75c0d1d..f2ee4e2 100644 --- a/README.md +++ b/README.md @@ -490,6 +490,12 @@ Key settings in more detail: waits for an upstream connection and for a processing slot (up to 10 seconds each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds - `signing_key` — HMAC secret for URL signatures +- `db_url` — the SQLite database to open; omitted, it is + `file:/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the + database in WAL mode. pixa adds `_pragma=busy_timeout(5000)` to any `db_url`, + so a write that finds another in progress waits up to five seconds for it + instead of failing. WAL mode comes only from the URL: keep + `_pragma=journal_mode(WAL)` in one you set - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk cache entirely; omitted defaults to 75% of the sum of the free space on the filesystem containing `/cache/` and the bytes of source and diff --git a/TODO.md b/TODO.md index 8c3c6df..89a78f1 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,12 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 SQLite writes no longer fail with "database is locked" (closes + #198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write + that finds another in progress on another connection waits up to five seconds + for it, and the default `db_url` turns on WAL mode with + `_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a + parameter the driver reads, so the database was never in WAL mode. - 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup` only passes through a periodic pass (closes #189): it slept for three eviction intervals before writing its file, and a startup pass still running diff --git a/config.example.yml b/config.example.yml index be42aa4..098d936 100644 --- a/config.example.yml +++ b/config.example.yml @@ -34,9 +34,10 @@ maintenance_mode: false state_dir: ./data # SQLite database URL (default: -# file:/state.sqlite3?_journal_mode=WAL). An empty value aborts -# startup; leave the key out to use the default. -# db_url: "file:./data/state.sqlite3?_journal_mode=WAL" +# file:/state.sqlite3?_pragma=journal_mode(WAL)). pixa adds +# _pragma=busy_timeout(5000) to it. An empty value aborts startup; leave the +# key out to use the default. +# db_url: "file:./data/state.sqlite3?_pragma=journal_mode(WAL)" # Image proxy settings # HMAC signing key for URL signatures (required, at least 32 characters) diff --git a/internal/config/config.go b/internal/config/config.go index a9f7b72..7160ca7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -320,7 +320,8 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { settingName(keyDBURL), errValueEmpty) } - c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_journal_mode=WAL", c.StateDir) + // The driver sets the journal mode only through a _pragma parameter. + c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_pragma=journal_mode(WAL)", c.StateDir) } if loader.err != nil { diff --git a/internal/config/config_validation_internal_test.go b/internal/config/config_validation_internal_test.go index 1800ab3..d3a3736 100644 --- a/internal/config/config_validation_internal_test.go +++ b/internal/config/config_validation_internal_test.go @@ -1,6 +1,7 @@ package config import ( + "database/sql" "log/slog" "os" "path/filepath" @@ -9,6 +10,8 @@ import ( "time" "git.eeqj.de/sneak/smartconfig" + + _ "modernc.org/sqlite" // SQLite driver registration ) // validTestSigningKey is a 32-character signing key that satisfies the @@ -94,12 +97,43 @@ func TestOmittedValuesUseDefaults(t *testing.T) { t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts) } - wantDBURL := "file:" + DefaultStateDir + "/state.sqlite3?_journal_mode=WAL" + wantDBURL := "file:" + DefaultStateDir + + "/state.sqlite3?_pragma=journal_mode(WAL)" if c.DBURL != wantDBURL { t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL) } } +// TestDefaultDBURLOpensTheDatabaseInWALMode opens the db_url derived from +// state_dir with the SQLite driver pixad uses and checks that the database +// is in WAL mode: the driver ignores any parameter it does not know. +func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+"state_dir: "+t.TempDir()+"\n") + if err != nil { + t.Fatalf("config with only state_dir set should be valid, got: %v", err) + } + + db, err := sql.Open("sqlite", c.DBURL) + if err != nil { + t.Fatalf("failed to open %q: %v", c.DBURL, err) + } + + t.Cleanup(func() { _ = db.Close() }) + + var journalMode string + + err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode) + if err != nil { + t.Fatalf("failed to read the journal mode of %q: %v", c.DBURL, err) + } + + if journalMode != "wal" { + t.Errorf("journal mode of %q = %q, want wal", c.DBURL, journalMode) + } +} + func TestExplicitValidValuesAreUsed(t *testing.T) { t.Parallel() diff --git a/internal/database/concurrent_writes_internal_test.go b/internal/database/concurrent_writes_internal_test.go new file mode 100644 index 0000000..3db4c4e --- /dev/null +++ b/internal/database/concurrent_writes_internal_test.go @@ -0,0 +1,153 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "path/filepath" + "sync" + "testing" + + "sneak.berlin/go/pixa/internal/config" +) + +// TestConcurrentWritesAllSucceed opens a database the way pixad does and +// writes to it from several goroutines at once, so the writes run on +// separate connections, as one request's writes and the background eviction +// pass do. Every write must succeed, none failing with "database is locked", +// whether or not db_url already has parameters, and the parameters it has +// must still apply. +func TestConcurrentWritesAllSucceed(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + wantJournalMode string + }{ + { + name: "db_url without parameters", + query: "", + wantJournalMode: "delete", + }, + { + name: "db_url with the WAL parameter", + query: "?_pragma=journal_mode(WAL)", + wantJournalMode: "wal", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dbURL := "file:" + filepath.Join(t.TempDir(), "state.sqlite3") + tt.query + + d := &Database{ + log: slog.New(slog.DiscardHandler), + config: &config.Config{DBURL: dbURL}, + } + + err := d.connect(t.Context()) + if err != nil { + t.Fatalf("failed to connect to %q: %v", dbURL, err) + } + + t.Cleanup(func() { _ = d.db.Close() }) + + writeConcurrently(t, d.db) + + var journalMode string + + err = d.db.QueryRowContext(t.Context(), "PRAGMA journal_mode"). + Scan(&journalMode) + if err != nil { + t.Fatalf("failed to read the journal mode: %v", err) + } + + if journalMode != tt.wantJournalMode { + t.Errorf("journal mode = %q, want %q", journalMode, tt.wantJournalMode) + } + }) + } +} + +// writeConcurrently runs writeLikeOneRequest from several goroutines at once +// and checks that every write was made. +func writeConcurrently(t *testing.T, db *sql.DB) { + t.Helper() + + const ( + writers = 4 + requestsEach = 20 + totalRequests = writers * requestsEach + ) + + ctx := t.Context() + + var wg sync.WaitGroup + + for writer := range writers { + wg.Go(func() { + for request := range requestsEach { + key := fmt.Sprintf("%d-%d", writer, request) + + err := writeLikeOneRequest(ctx, db, key) + if err != nil { + t.Errorf("writer %d: %v", writer, err) + + return + } + } + }) + } + + wg.Wait() + + var hits, sources int + + err := db.QueryRowContext(ctx, ` + SELECT hit_count, (SELECT COUNT(*) FROM source_content) + FROM cache_stats WHERE id = 1 + `).Scan(&hits, &sources) + if err != nil { + t.Fatalf("failed to count the writes: %v", err) + } + + if hits != totalRequests || sources != totalRequests { + t.Errorf("hit_count = %d and %d source_content rows, want %d of each", + hits, sources, totalRequests) + } +} + +// writeLikeOneRequest makes the writes one request and the eviction pass +// make: it counts a cache hit, stores a source, records a transformed image +// and deletes that record again. +func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error { + _, err := db.ExecContext(ctx, + `UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`) + if err != nil { + return fmt.Errorf("counting a cache hit: %w", err) + } + + _, err = db.ExecContext(ctx, `INSERT INTO source_content + (content_hash, content_type, size_bytes) VALUES (?, 'image/png', 1)`, key) + if err != nil { + return fmt.Errorf("storing a source: %w", err) + } + + _, err = db.ExecContext(ctx, `INSERT INTO variant_content + (cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key) + if err != nil { + return fmt.Errorf("recording a transformed image: %w", err) + } + + _, err = db.ExecContext(ctx, + `DELETE FROM variant_content WHERE cache_key = ?`, key) + if err != nil { + return fmt.Errorf("evicting a transformed image: %w", err) + } + + return nil +} diff --git a/internal/database/database.go b/internal/database/database.go index ab59f75..88d9840 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -243,7 +243,17 @@ func (s *Database) DB() *sql.DB { } func (s *Database) connect(ctx context.Context) error { - dbURL := s.config.DBURL + // Requests and the eviction pass write on separate connections. With + // a busy timeout, a write that finds another one in progress waits up + // to five seconds for it instead of failing at once with "database is + // locked". The driver runs each _pragma parameter on every connection + // it opens. + separator := "?" + if strings.Contains(s.config.DBURL, "?") { + separator = "&" + } + + dbURL := s.config.DBURL + separator + "_pragma=busy_timeout(5000)" s.log.Info("connecting to database", "url", dbURL) From 8568c17d1b40b08f7f9a62b4fb92f9900436321c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 21:07:52 +0200 Subject: [PATCH 036/131] Move the example config to configs/, delete scripts/ and CONVENTIONS.md (closes #97) config.example.yml moves unchanged to configs/config.example.yml, the directory REPO_POLICIES.md names for configuration examples. README.md, the comments in internal/config/config.go and the startup error for the placeholder signing key name the new path. scripts/manual-test.sh and its directory are deleted. The handler tests in internal/handlers cover every check it made except two: fetching a real image from the internet, and a URL made on the generator page with a ttl answering 410 once the ttl has passed (https://git.eeqj.de/sneak/pixa/issues/199). CONVENTIONS.md, a reformatted copy of the Go HTTP server conventions, is deleted, as REPO_POLICIES.md links the canonical document. Model: opus-5-5 --- CONVENTIONS.md | 1259 ----------------- README.md | 4 +- TODO.md | 10 + .../config.example.yml | 0 internal/config/config.go | 8 +- scripts/manual-test.sh | 147 -- 6 files changed, 16 insertions(+), 1412 deletions(-) delete mode 100644 CONVENTIONS.md rename config.example.yml => configs/config.example.yml (100%) delete mode 100755 scripts/manual-test.sh diff --git a/CONVENTIONS.md b/CONVENTIONS.md deleted file mode 100644 index 9f8c1d3..0000000 --- a/CONVENTIONS.md +++ /dev/null @@ -1,1259 +0,0 @@ -# Go HTTP Server Conventions - -This document defines the architectural patterns, design decisions, and conventions for building Go HTTP servers. All new projects must follow these standards. - -## Table of Contents - -1. [Required Libraries](#1-required-libraries) -2. [Project Structure](#2-project-structure) -3. [Dependency Injection (Uber fx)](#3-dependency-injection-uber-fx) -4. [Server Architecture](#4-server-architecture) -5. [Routing (go-chi)](#5-routing-go-chi) -6. [Handler Conventions](#6-handler-conventions) -7. [Middleware Conventions](#7-middleware-conventions) -8. [Configuration (Viper)](#8-configuration-viper) -9. [Logging (slog)](#9-logging-slog) -10. [Database Wrapper](#10-database-wrapper) -11. [Globals Package](#11-globals-package) -12. [Static Assets & Templates](#12-static-assets--templates) -13. [Health Check](#13-health-check) -14. [External Integrations](#14-external-integrations) - ---- - -## 1. Required Libraries - -These libraries are **mandatory** for all new projects: - -| Purpose | Library | Import Path | -| -------------------- | --------------- | ------------------------------------- | -| Dependency Injection | Uber fx | `go.uber.org/fx` | -| HTTP Router | go-chi | `github.com/go-chi/chi` | -| Logging | slog (stdlib) | `log/slog` | -| Configuration | Viper | `github.com/spf13/viper` | -| Environment Loading | godotenv | `github.com/joho/godotenv/autoload` | -| CORS | go-chi/cors | `github.com/go-chi/cors` | -| Error Reporting | Sentry | `github.com/getsentry/sentry-go` | -| Metrics | Prometheus | `github.com/prometheus/client_golang` | -| Metrics Middleware | go-http-metrics | `github.com/slok/go-http-metrics` | -| Basic Auth | basicauth-go | `github.com/99designs/basicauth-go` | - ---- - -## 2. Project Structure - -``` -project-root/ -├── cmd/ -│ └── {appname}/ -│ └── main.go # Entry point -├── internal/ -│ ├── config/ -│ │ └── config.go # Configuration loading -│ ├── database/ -│ │ └── database.go # Database wrapper -│ ├── globals/ -│ │ └── globals.go # Build-time variables -│ ├── handlers/ -│ │ ├── handlers.go # Base handler struct and helpers -│ │ ├── index.go # Individual handlers... -│ │ ├── healthcheck.go -│ │ └── {feature}.go -│ ├── healthcheck/ -│ │ └── healthcheck.go # Health check service -│ ├── logger/ -│ │ └── logger.go # Logger setup -│ ├── middleware/ -│ │ └── middleware.go # All middleware definitions -│ └── server/ -│ ├── server.go # Server struct and lifecycle -│ ├── http.go # HTTP server setup -│ └── routes.go # Route definitions -├── static/ -│ ├── static.go # Embed directive -│ ├── css/ -│ └── js/ -├── templates/ -│ ├── templates.go # Embed and parse -│ └── *.html -├── go.mod -├── go.sum -├── Makefile -└── Dockerfile -``` - -### Key Principles - -- **`cmd/{appname}/`**: Only the entry point. Minimal logic, just bootstrapping. -- **`internal/`**: All application packages. Not importable by external projects. -- **One package per concern**: config, database, handlers, middleware, etc. -- **Flat handler files**: One file per handler or logical group of handlers. - ---- - -## 3. Dependency Injection (Uber fx) - -### Entry Point Pattern - -```go -// cmd/httpd/main.go -package main - -import ( - "yourproject/internal/config" - "yourproject/internal/database" - "yourproject/internal/globals" - "yourproject/internal/handlers" - "yourproject/internal/healthcheck" - "yourproject/internal/logger" - "yourproject/internal/middleware" - "yourproject/internal/server" - "go.uber.org/fx" -) - -var ( - Appname string = "CHANGEME" - Version string -) - -func main() { - globals.Appname = Appname - globals.Version = Version - - fx.New( - fx.Provide( - config.New, - database.New, - globals.New, - handlers.New, - logger.New, - server.New, - middleware.New, - healthcheck.New, - ), - fx.Invoke(func(*server.Server) {}), - ).Run() -} -``` - -### Params Struct Pattern - -Every component that receives dependencies uses a params struct with `fx.In`: - -```go -type HandlersParams struct { - fx.In - Logger *logger.Logger - Globals *globals.Globals - Database *database.Database - Healthcheck *healthcheck.Healthcheck -} - -type Handlers struct { - params *HandlersParams - log *slog.Logger - hc *healthcheck.Healthcheck -} -``` - -### Factory Function Pattern - -All components expose a `New` function with this signature: - -```go -func New(lc fx.Lifecycle, params SomeParams) (*Something, error) { - s := &Something{ - log: params.Logger.Get(), - config: params.Config, - db: params.Database, - } - - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - // Initialize resources - return nil - }, - OnStop: func(ctx context.Context) error { - // Cleanup resources - return nil - }, - }) - return s, nil -} -``` - -### Params vs Struct Fields - -The `Params` struct is **only for dependency injection in the constructor**. Dependencies should be copied to struct fields for use at runtime: - -```go -// WRONG - accessing dependencies via params -type Server struct { - params ServerParams // Don't store params -} - -func (s *Server) SomeMethod() { - port := s.params.Config.Port // Bad: indirect access -} - -// CORRECT - store dependencies directly on struct -type Server struct { - log *slog.Logger - config *config.Config - db *database.Database -} - -func (s *Server) SomeMethod() { - port := s.config.Port // Good: direct access -} -``` - -This pattern: -- Makes dependencies explicit in the struct definition -- Avoids unnecessary indirection at runtime -- Clearly separates DI concerns (params) from runtime state (struct fields) - -### Dependency Order - -Providers are resolved automatically by fx, but conceptually follow this order: - -1. `globals.New` - Build-time variables (no dependencies) -2. `logger.New` - Logger (depends on Globals) -3. `config.New` - Configuration (depends on Globals, Logger) -4. `database.New` - Database (depends on Logger, Config) -5. `healthcheck.New` - Health check (depends on Globals, Config, Logger, Database) -6. `middleware.New` - Middleware (depends on Logger, Globals, Config) -7. `handlers.New` - Handlers (depends on Logger, Globals, Database, Healthcheck) -8. `server.New` - Server (depends on all above) - ---- - -## 4. Server Architecture - -### Server Struct - -The Server struct is the central orchestrator: - -```go -// internal/server/server.go -type ServerParams struct { - fx.In - Logger *logger.Logger - Globals *globals.Globals - Config *config.Config - Middleware *middleware.Middleware - Handlers *handlers.Handlers -} - -type Server struct { - startupTime time.Time - port int - exitCode int - sentryEnabled bool - log *slog.Logger - ctx context.Context - cancelFunc context.CancelFunc - httpServer *http.Server - router *chi.Mux - params ServerParams - mw *middleware.Middleware - h *handlers.Handlers -} -``` - -### Server Factory - -```go -func New(lc fx.Lifecycle, params ServerParams) (*Server, error) { - s := new(Server) - s.params = params - s.mw = params.Middleware - s.h = params.Handlers - s.log = params.Logger.Get() - - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - s.startupTime = time.Now() - go s.Run() - return nil - }, - OnStop: func(ctx context.Context) error { - // Server shutdown logic - return nil - }, - }) - return s, nil -} -``` - -### HTTP Server Setup - -```go -// internal/server/http.go -func (s *Server) serveUntilShutdown() { - listenAddr := fmt.Sprintf(":%d", s.params.Config.Port) - s.httpServer = &http.Server{ - Addr: listenAddr, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - Handler: s, - } - - s.SetupRoutes() - - s.log.Info("http begin listen", "listenaddr", listenAddr) - if err := s.httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - s.log.Error("listen error", "error", err) - if s.cancelFunc != nil { - s.cancelFunc() - } - } -} - -func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.router.ServeHTTP(w, r) -} -``` - -### Signal Handling and Graceful Shutdown - -```go -func (s *Server) serve() int { - s.ctx, s.cancelFunc = context.WithCancel(context.Background()) - - // Signal watcher - go func() { - c := make(chan os.Signal, 1) - signal.Ignore(syscall.SIGPIPE) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - sig := <-c - s.log.Info("signal received", "signal", sig) - if s.cancelFunc != nil { - s.cancelFunc() - } - }() - - go s.serveUntilShutdown() - - for range s.ctx.Done() { - } - s.cleanShutdown() - return s.exitCode -} - -func (s *Server) cleanShutdown() { - s.exitCode = 0 - ctxShutdown, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := s.httpServer.Shutdown(ctxShutdown); err != nil { - s.log.Error("server clean shutdown failed", "error", err) - } - if shutdownCancel != nil { - shutdownCancel() - } - s.cleanupForExit() - if s.sentryEnabled { - sentry.Flush(2 * time.Second) - } -} -``` - ---- - -## 5. Routing (go-chi) - -### Route Setup Pattern - -```go -// internal/server/routes.go -func (s *Server) SetupRoutes() { - s.router = chi.NewRouter() - - // Global middleware (applied to all routes) - s.router.Use(middleware.Recoverer) - s.router.Use(middleware.RequestID) - s.router.Use(s.mw.Logging()) - - // Conditional middleware - if viper.GetString("METRICS_USERNAME") != "" { - s.router.Use(s.mw.Metrics()) - } - - s.router.Use(s.mw.CORS()) - s.router.Use(middleware.Timeout(60 * time.Second)) - - if s.sentryEnabled { - sentryHandler := sentryhttp.New(sentryhttp.Options{ - Repanic: true, - }) - s.router.Use(sentryHandler.Handle) - } - - // Routes - s.router.Get("/", s.h.HandleIndex()) - - // Static files - s.router.Mount("/s", http.StripPrefix("/s", http.FileServer(http.FS(static.Static)))) - - // API versioning - s.router.Route("/api/v1", func(r chi.Router) { - r.Get("/now", s.h.HandleNow()) - }) - - // Routes with specific middleware - auth := s.mw.Auth() - s.router.Get("/login", auth(s.h.HandleLoginGET()).ServeHTTP) - - // Health check (standard path) - s.router.Get("/.well-known/healthcheck.json", s.h.HandleHealthCheck()) - - // Protected route groups - if viper.GetString("METRICS_USERNAME") != "" { - s.router.Group(func(r chi.Router) { - r.Use(s.mw.MetricsAuth()) - r.Get("/metrics", http.HandlerFunc(promhttp.Handler().ServeHTTP)) - }) - } -} -``` - -### Middleware Ordering (Critical) - -1. `middleware.Recoverer` - Panic recovery (must be first) -2. `middleware.RequestID` - Generate request IDs -3. `s.mw.Logging()` - Request logging -4. `s.mw.Metrics()` - Prometheus metrics (if enabled) -5. `s.mw.CORS()` - CORS headers -6. `middleware.Timeout(60s)` - Request timeout -7. `sentryhttp.Handler` - Sentry error reporting (if enabled) - -### API Versioning - -Use route groups for API versioning: - -```go -s.router.Route("/api/v1", func(r chi.Router) { - r.Get("/resource", s.h.HandleResource()) -}) -``` - -### Static File Serving - -Static files are served at `/s/` prefix: - -```go -s.router.Mount("/s", http.StripPrefix("/s", http.FileServer(http.FS(static.Static)))) -``` - ---- - -## 6. Handler Conventions - -### Handler Base Struct - -```go -// internal/handlers/handlers.go -type HandlersParams struct { - fx.In - Logger *logger.Logger - Globals *globals.Globals - Database *database.Database - Healthcheck *healthcheck.Healthcheck -} - -type Handlers struct { - params *HandlersParams - log *slog.Logger - hc *healthcheck.Healthcheck -} - -func New(lc fx.Lifecycle, params HandlersParams) (*Handlers, error) { - s := new(Handlers) - s.params = ¶ms - s.log = params.Logger.Get() - s.hc = params.Healthcheck - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - // Compile templates or other initialization - return nil - }, - }) - return s, nil -} -``` - -### Closure-Based Handler Pattern - -For JSON route handlers, both the request and the response structures are -defined in the scope of the method that returns the HandlerFunc. They can -be called simply `Request` and `Response` or slightly more descriptive -names. - -All handlers return `http.HandlerFunc` using the closure pattern. This allows initialization logic to run once when the handler is created: - -```go -// internal/handlers/index.go -func (s *Handlers) HandleIndex() http.HandlerFunc { - // Initialization runs once - t := templates.GetParsed() - - // Handler runs per-request - return func(w http.ResponseWriter, r *http.Request) { - err := t.ExecuteTemplate(w, "index.html", nil) - if err != nil { - s.log.Error("template execution failed", "error", err) - http.Error(w, http.StatusText(500), 500) - } - } -} -``` - -### JSON Handler Pattern - -```go -// internal/handlers/now.go -func (s *Handlers) HandleNow() http.HandlerFunc { - - type request struct { - // request format - } - - // Response struct defined in closure scope - type response struct { - Now time.Time `json:"now"` - } - return func(w http.ResponseWriter, r *http.Request) { - s.respondJSON(w, r, &response{Now: time.Now()}, 200) - } -} -``` - -### Response Helpers - -```go -// internal/handlers/handlers.go -func (s *Handlers) respondJSON(w http.ResponseWriter, r *http.Request, data interface{}, status int) { - w.WriteHeader(status) - w.Header().Set("Content-Type", "application/json") - if data != nil { - err := json.NewEncoder(w).Encode(data) - if err != nil { - s.log.Error("json encode error", "error", err) - } - } -} - -func (s *Handlers) decodeJSON(w http.ResponseWriter, r *http.Request, v interface{}) error { - return json.NewDecoder(r.Body).Decode(v) -} -``` - -### Handler Naming Convention - -- `HandleIndex()` - Main page -- `HandleLoginGET()` / `HandleLoginPOST()` - Form handlers with HTTP method suffix -- `HandleNow()` - API endpoints -- `HandleHealthCheck()` - System endpoints - ---- - -## 7. Middleware Conventions - -### Middleware Struct - -```go -// internal/middleware/middleware.go -type MiddlewareParams struct { - fx.In - Logger *logger.Logger - Globals *globals.Globals - Config *config.Config -} - -type Middleware struct { - log *slog.Logger - params *MiddlewareParams -} - -func New(lc fx.Lifecycle, params MiddlewareParams) (*Middleware, error) { - s := new(Middleware) - s.params = ¶ms - s.log = params.Logger.Get() - return s, nil -} -``` - -### Middleware Signature - -All custom middleware methods return `func(http.Handler) http.Handler`: - -```go -func (s *Middleware) Auth() func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Before request - s.log.Info("AUTH: before request") - - next.ServeHTTP(w, r) - - // After request (optional) - }) - } -} -``` - -### Logging Middleware with Status Capture - -```go -type loggingResponseWriter struct { - http.ResponseWriter - statusCode int -} - -func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter { - return &loggingResponseWriter{w, http.StatusOK} -} - -func (lrw *loggingResponseWriter) WriteHeader(code int) { - lrw.statusCode = code - lrw.ResponseWriter.WriteHeader(code) -} - -func (s *Middleware) Logging() func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - lrw := NewLoggingResponseWriter(w) - ctx := r.Context() - defer func() { - latency := time.Since(start) - s.log.InfoContext(ctx, "request", - "request_start", start, - "method", r.Method, - "url", r.URL.String(), - "useragent", r.UserAgent(), - "request_id", ctx.Value(middleware.RequestIDKey).(string), - "referer", r.Referer(), - "proto", r.Proto, - "remoteIP", ipFromHostPort(r.RemoteAddr), - "status", lrw.statusCode, - "latency_ms", latency.Milliseconds(), - ) - }() - next.ServeHTTP(lrw, r) - }) - } -} -``` - -### CORS Middleware - -```go -func (s *Middleware) CORS() func(http.Handler) http.Handler { - return cors.Handler(cors.Options{ - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"}, - ExposedHeaders: []string{"Link"}, - AllowCredentials: false, - MaxAge: 300, - }) -} -``` - -### Metrics Middleware - -```go -func (s *Middleware) Metrics() func(http.Handler) http.Handler { - mdlw := ghmm.New(ghmm.Config{ - Recorder: metrics.NewRecorder(metrics.Config{}), - }) - return func(next http.Handler) http.Handler { - return std.Handler("", mdlw, next) - } -} - -func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler { - return basicauth.New( - "metrics", - map[string][]string{ - viper.GetString("METRICS_USERNAME"): { - viper.GetString("METRICS_PASSWORD"), - }, - }, - ) -} -``` - ---- - -## 8. Configuration (Viper) - -### Config Struct - -```go -// internal/config/config.go -type ConfigParams struct { - fx.In - Globals *globals.Globals - Logger *logger.Logger -} - -type Config struct { - DBURL string - Debug bool - MaintenanceMode bool - MetricsPassword string - MetricsUsername string - Port int - SentryDSN string - params *ConfigParams - log *slog.Logger -} -``` - -### Configuration Loading - -```go -func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) { - log := params.Logger.Get() - name := params.Globals.Appname - - // Config file settings - viper.SetConfigName(name) - viper.SetConfigType("yaml") - viper.AddConfigPath(fmt.Sprintf("/etc/%s", name)) - viper.AddConfigPath(fmt.Sprintf("$HOME/.config/%s", name)) - - // Environment variables override everything - viper.AutomaticEnv() - - // Defaults - viper.SetDefault("DEBUG", "false") - viper.SetDefault("MAINTENANCE_MODE", "false") - viper.SetDefault("PORT", "8080") - viper.SetDefault("DBURL", "") - viper.SetDefault("SENTRY_DSN", "") - viper.SetDefault("METRICS_USERNAME", "") - viper.SetDefault("METRICS_PASSWORD", "") - - // Read config file (optional) - if err := viper.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - // Config file not found is OK - } else { - log.Error("config file malformed", "error", err) - panic(err) - } - } - - // Build config struct - s := &Config{ - DBURL: viper.GetString("DBURL"), - Debug: viper.GetBool("debug"), - Port: viper.GetInt("PORT"), - SentryDSN: viper.GetString("SENTRY_DSN"), - MaintenanceMode: viper.GetBool("MAINTENANCE_MODE"), - MetricsUsername: viper.GetString("METRICS_USERNAME"), - MetricsPassword: viper.GetString("METRICS_PASSWORD"), - log: log, - params: ¶ms, - } - - // Enable debug logging if configured - if s.Debug { - params.Logger.EnableDebugLogging() - s.log = params.Logger.Get() - } - - return s, nil -} -``` - -### Configuration Precedence - -1. **Environment variables** (highest priority via `AutomaticEnv()`) -2. **`.env` file** (loaded via `godotenv/autoload` import) -3. **Config files**: `/etc/{appname}/{appname}.yaml`, `~/.config/{appname}/{appname}.yaml` -4. **Defaults** (lowest priority) - -### Environment Loading - -Import godotenv with autoload to automatically load `.env` files: - -```go -import ( - _ "github.com/joho/godotenv/autoload" -) -``` - ---- - -## 9. Logging (slog) - -### Logger Struct - -```go -// internal/logger/logger.go -type LoggerParams struct { - fx.In - Globals *globals.Globals -} - -type Logger struct { - log *slog.Logger - level *slog.LevelVar - params LoggerParams -} -``` - -### Logger Setup with TTY Detection - -```go -func New(lc fx.Lifecycle, params LoggerParams) (*Logger, error) { - l := new(Logger) - l.level = new(slog.LevelVar) - l.level.Set(slog.LevelInfo) - - // TTY detection for dev vs prod output - tty := false - if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) != 0 { - tty = true - } - - var handler slog.Handler - if tty { - // Text output for development - handler = slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ - Level: l.level, - AddSource: true, - }) - } else { - // JSON output for production - handler = slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ - Level: l.level, - AddSource: true, - }) - } - - l.log = slog.New(handler) - return l, nil -} -``` - -### Logger Methods - -```go -func (l *Logger) EnableDebugLogging() { - l.level.Set(slog.LevelDebug) - l.log.Debug("debug logging enabled", "debug", true) -} - -func (l *Logger) Get() *slog.Logger { - return l.log -} - -func (l *Logger) Identify() { - l.log.Info("starting", - "appname", l.params.Globals.Appname, - "version", l.params.Globals.Version, - "arch", runtime.GOARCH, - ) -} -``` - -### Logging Patterns - -```go -// Info with fields -s.log.Info("message", "key", "value") - -// Error with error object -s.log.Error("operation failed", "error", err) - -// With context -s.log.InfoContext(ctx, "processing request", "request_id", reqID) - -// Structured request logging -s.log.Info("request completed", - "request_start", start, - "method", r.Method, - "url", r.URL.String(), - "status", statusCode, - "latency_ms", latency.Milliseconds(), -) - -// Using slog.Group for nested attributes -s.log.Info("request", - slog.Group("http", - "method", r.Method, - "url", r.URL.String(), - ), - slog.Group("timing", - "start", start, - "latency_ms", latency.Milliseconds(), - ), -) -``` - ---- - -## 10. Database Wrapper - -### Database Struct - -```go -// internal/database/database.go -type DatabaseParams struct { - fx.In - Logger *logger.Logger - Config *config.Config -} - -type Database struct { - URL string - log *slog.Logger - params *DatabaseParams -} -``` - -### Database Factory with Lifecycle - -```go -func New(lc fx.Lifecycle, params DatabaseParams) (*Database, error) { - s := new(Database) - s.params = ¶ms - s.log = params.Logger.Get() - - s.log.Info("Database instantiated") - - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - s.log.Info("Database OnStart Hook") - // Connect to database here - // Example: s.db, err = sql.Open("postgres", s.params.Config.DBURL) - return nil - }, - OnStop: func(ctx context.Context) error { - // Disconnect from database here - // Example: s.db.Close() - return nil - }, - }) - return s, nil -} -``` - -### Usage Pattern - -The Database struct is injected into handlers and other services: - -```go -type HandlersParams struct { - fx.In - Database *database.Database - // ... -} - -// Access in handler -func (s *Handlers) HandleSomething() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Use s.params.Database - } -} -``` - ---- - -## 11. Globals Package - -### Package Variables and Struct - -```go -// internal/globals/globals.go -package globals - -import "go.uber.org/fx" - -// Package-level variables (set from main) -var ( - Appname string - Version string -) - -// Struct for DI -type Globals struct { - Appname string - Version string -} - -func New(lc fx.Lifecycle) (*Globals, error) { - n := &Globals{ - Appname: Appname, - Version: Version, - } - return n, nil -} -``` - -### Setting Globals in Main - -```go -// cmd/httpd/main.go -var ( - Appname string = "CHANGEME" // Default, overridden by build - Version string // Set at build time -) - -func main() { - globals.Appname = Appname - globals.Version = Version - // ... -} -``` - -### Build-Time Variable Injection - -Use ldflags to inject version information at build time: - -```makefile -VERSION := $(shell git describe --tags --always) - -build: - go build -ldflags "-X main.Version=$(VERSION)" ./cmd/httpd -``` - ---- - -## 12. Static Assets & Templates - -### Static File Embedding - -```go -// static/static.go -package static - -import "embed" - -//go:embed css js -var Static embed.FS -``` - -Directory structure: - -``` -static/ -├── static.go -├── css/ -│ ├── bootstrap-4.5.3.min.css -│ └── style.css -└── js/ - ├── bootstrap-4.5.3.bundle.min.js - └── jquery-3.5.1.slim.min.js -``` - -### Template Embedding and Lazy Parsing - -```go -// templates/templates.go -package templates - -import ( - "embed" - "text/template" -) - -//go:embed *.html -var TemplatesRaw embed.FS -var TemplatesParsed *template.Template - -func GetParsed() *template.Template { - if TemplatesParsed == nil { - TemplatesParsed = template.Must(template.ParseFS(TemplatesRaw, "*")) - } - return TemplatesParsed -} -``` - -### Template Composition - -Templates use Go's template composition: - -```html - -{{ template "htmlheader.html" . }} {{ template "navbar.html" . }} - -
- -
- -{{ template "pagefooter.html" . }} {{ template "htmlfooter.html" . }} -``` - -### Static Asset References - -Reference static files with `/s/` prefix: - -```html - - - -``` - ---- - -## 13. Health Check - -### Health Check Service - -```go -// internal/healthcheck/healthcheck.go -type HealthcheckParams struct { - fx.In - Globals *globals.Globals - Config *config.Config - Logger *logger.Logger - Database *database.Database -} - -type Healthcheck struct { - StartupTime time.Time - log *slog.Logger - params *HealthcheckParams -} - -func New(lc fx.Lifecycle, params HealthcheckParams) (*Healthcheck, error) { - s := new(Healthcheck) - s.params = ¶ms - s.log = params.Logger.Get() - - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - s.StartupTime = time.Now() - return nil - }, - OnStop: func(ctx context.Context) error { - return nil - }, - }) - return s, nil -} -``` - -### Health Check Response - -```go -type HealthcheckResponse struct { - Status string `json:"status"` - Now string `json:"now"` - UptimeSeconds int64 `json:"uptime_seconds"` - UptimeHuman string `json:"uptime_human"` - Version string `json:"version"` - Appname string `json:"appname"` - Maintenance bool `json:"maintenance_mode"` -} - -func (s *Healthcheck) uptime() time.Duration { - return time.Since(s.StartupTime) -} - -func (s *Healthcheck) Healthcheck() *HealthcheckResponse { - resp := &HealthcheckResponse{ - Status: "ok", - Now: time.Now().UTC().Format(time.RFC3339Nano), - UptimeSeconds: int64(s.uptime().Seconds()), - UptimeHuman: s.uptime().String(), - Appname: s.params.Globals.Appname, - Version: s.params.Globals.Version, - } - return resp -} -``` - -### Standard Endpoint - -Health check is served at the standard `.well-known` path: - -```go -s.router.Get("/.well-known/healthcheck.json", s.h.HandleHealthCheck()) -``` - ---- - -## 14. External Integrations - -### Sentry Error Reporting - -Sentry is conditionally enabled based on `SENTRY_DSN` environment variable: - -```go -func (s *Server) enableSentry() { - s.sentryEnabled = false - - if s.params.Config.SentryDSN == "" { - return - } - - err := sentry.Init(sentry.ClientOptions{ - Dsn: s.params.Config.SentryDSN, - Release: fmt.Sprintf("%s-%s", s.params.Globals.Appname, s.params.Globals.Version), - }) - if err != nil { - s.log.Error("sentry init failure", "error", err) - os.Exit(1) - return - } - s.log.Info("sentry error reporting activated") - s.sentryEnabled = true -} -``` - -Sentry middleware with repanic (bubbles panics to chi's Recoverer): - -```go -if s.sentryEnabled { - sentryHandler := sentryhttp.New(sentryhttp.Options{ - Repanic: true, - }) - s.router.Use(sentryHandler.Handle) -} -``` - -Flush Sentry on shutdown: - -```go -if s.sentryEnabled { - sentry.Flush(2 * time.Second) -} -``` - -### Prometheus Metrics - -Metrics are conditionally enabled and protected by basic auth: - -```go -// Only enable if credentials are configured -if viper.GetString("METRICS_USERNAME") != "" { - s.router.Use(s.mw.Metrics()) -} - -// Protected /metrics endpoint -if viper.GetString("METRICS_USERNAME") != "" { - s.router.Group(func(r chi.Router) { - r.Use(s.mw.MetricsAuth()) - r.Get("/metrics", http.HandlerFunc(promhttp.Handler().ServeHTTP)) - }) -} -``` - -### Environment Variables Summary - -| Variable | Description | Default | -| ------------------ | -------------------------------- | ------- | -| `PORT` | HTTP listen port | 8080 | -| `DEBUG` | Enable debug logging | false | -| `DBURL` | Database connection URL | "" | -| `SENTRY_DSN` | Sentry DSN for error reporting | "" | -| `MAINTENANCE_MODE` | Enable maintenance mode | false | -| `METRICS_USERNAME` | Basic auth username for /metrics | "" | -| `METRICS_PASSWORD` | Basic auth password for /metrics | "" | diff --git a/README.md b/README.md index f2ee4e2..0906560 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ make build # run with a config file: copy the example and set a real signing key # (the example placeholder is refused at startup), e.g. with # openssl rand -base64 32 -cp config.example.yml config.yml +cp configs/config.example.yml config.yml $EDITOR config.yml # replace the signing_key placeholder ./bin/pixad --config config.yml @@ -520,7 +520,7 @@ Key settings in more detail: the container unhealthy, and upaas marks a deploy failed when its container is unhealthy. The login and URL generator pages and `/metrics` keep working -See `config.example.yml` for all options with defaults. +See `configs/config.example.yml` for all options with defaults. ### Architecture diff --git a/TODO.md b/TODO.md index 89a78f1..18d33b4 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,16 @@ P2: security: referer blacklist # Completed Steps +- 2026-10-04 fewer files in the repository root (closes #97): + `config.example.yml` moved unchanged to `configs/config.example.yml`, and + `README.md`, the comments in `internal/config/config.go` and the startup error + for the placeholder signing key name the new path; `scripts/manual-test.sh` + and its directory are deleted, as the handler tests in `internal/handlers` + cover every check it made except two: fetching a real image from the + internet, and a URL made on the generator page with a `ttl` answering 410 once + the `ttl` has passed (https://git.eeqj.de/sneak/pixa/issues/199); + `CONVENTIONS.md` is deleted, as `REPO_POLICIES.md` links the canonical Go HTTP + server conventions. - 2026-10-04 SQLite writes no longer fail with "database is locked" (closes #198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write that finds another in progress on another connection waits up to five seconds diff --git a/config.example.yml b/configs/config.example.yml similarity index 100% rename from config.example.yml rename to configs/config.example.yml diff --git a/internal/config/config.go b/internal/config/config.go index 7160ca7..98addb0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,7 +62,7 @@ const ( ) // placeholderSigningKey is the dummy signing_key shipped in -// config.example.yml. It is 45 characters, so it passes the length +// configs/config.example.yml. It is 45 characters, so it passes the length // check, but it is public in this repository and must be rejected at // startup so no deployment ever signs URLs with it. const placeholderSigningKey = "CHANGE_ME_generate_with_openssl_rand_base64_32" @@ -88,7 +88,7 @@ var ( errMustBeAtLeastOne = errors.New("must be at least 1") errValueTooShort = errors.New("value too short") errPlaceholderKey = errors.New( - "is the placeholder from config.example.yml; " + + "is the placeholder from configs/config.example.yml; " + "generate a real key with: openssl rand -base64 32") errMustBeSetTogether = errors.New("must be set together") errMustNotBeNegative = errors.New("must not be negative") @@ -554,8 +554,8 @@ func (c *Config) ensureStateDirWritable() error { } // validateSigningKey checks that the signing key is present, long -// enough, and not the public placeholder from config.example.yml. The -// key value itself is never echoed in error messages. +// enough, and not the public placeholder from configs/config.example.yml. +// The key value itself is never echoed in error messages. func (c *Config) validateSigningKey() error { if c.SigningKey == "" { return fmt.Errorf("%s: %w", settingName(keySigningKey), errValueRequired) diff --git a/scripts/manual-test.sh b/scripts/manual-test.sh deleted file mode 100755 index b0118a3..0000000 --- a/scripts/manual-test.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash -# -# Manual test script for pixa server -# Requires: server running on localhost:8080 -# -set -e - -BASE_URL="${BASE_URL:-http://localhost:8080}" -SIGNING_KEY="${SIGNING_KEY:-test-signing-key-for-development-only}" -TEST_IMAGE_URL="https://s3.sneak.cloud/sneak-public/2021/2021-04-18.untitled.a7r4.07723.jpg" -COOKIE_JAR=$(mktemp) - -cleanup() { - rm -f "$COOKIE_JAR" -} -trap cleanup EXIT - -pass() { - echo "✓ PASS: $1" -} - -fail() { - echo "✗ FAIL: $1" - exit 1 -} - -echo "=== Pixa Manual Test Suite ===" -echo "Base URL: $BASE_URL" -echo "" - -# Test 1: Healthcheck -echo "--- Test 1: Healthcheck endpoint ---" -HEALTH=$(curl -sf "$BASE_URL/.well-known/healthcheck.json") -if echo "$HEALTH" | grep -q '"status"'; then - pass "Healthcheck returns status" -else - fail "Healthcheck did not return expected response" -fi - -# Test 2: Login page displays -echo "--- Test 2: Login page (GET /) ---" -LOGIN_PAGE=$(curl -sf "$BASE_URL/") -if echo "$LOGIN_PAGE" | grep -qi "password\|login\|sign"; then - pass "Login page displays password form" -else - fail "Login page did not display expected content" -fi - -# Test 3: Wrong password shows error -echo "--- Test 3: Login with wrong password ---" -WRONG_LOGIN=$(curl -sf -X POST "$BASE_URL/" -d "key=wrong-key" -c "$COOKIE_JAR") -if echo "$WRONG_LOGIN" | grep -qi "invalid\|error\|incorrect\|wrong"; then - pass "Wrong password shows error message" -else - fail "Wrong password did not show error" -fi - -# Test 4: Correct password redirects to generator -echo "--- Test 4: Login with correct signing key ---" -curl -sf -X POST "$BASE_URL/" -d "key=$SIGNING_KEY" -c "$COOKIE_JAR" -b "$COOKIE_JAR" -L -o /dev/null -GENERATOR_PAGE=$(curl -sf "$BASE_URL/" -b "$COOKIE_JAR") -if echo "$GENERATOR_PAGE" | grep -qi "generate\|url\|source\|logout"; then - pass "Correct password shows generator page" -else - fail "Generator page not displayed after login" -fi - -# Test 5: Generate encrypted URL -echo "--- Test 5: Generate encrypted URL ---" -GEN_RESULT=$(curl -sf -X POST "$BASE_URL/generate" -b "$COOKIE_JAR" \ - -d "url=$TEST_IMAGE_URL" \ - -d "width=800" \ - -d "height=600" \ - -d "format=jpeg" \ - -d "quality=85" \ - -d "fit=cover" \ - -d "ttl=3600") -if echo "$GEN_RESULT" | grep -q "/v1/e/"; then - pass "Encrypted URL generated" - # Extract the encrypted URL - ENC_URL=$(echo "$GEN_RESULT" | grep -o '/v1/e/[^"<]*' | head -1) - echo " Generated URL: $ENC_URL" -else - fail "Failed to generate encrypted URL" -fi - -# Test 6: Fetch image via encrypted URL -echo "--- Test 6: Fetch image via encrypted URL ---" -if [ -n "$ENC_URL" ]; then - HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$BASE_URL$ENC_URL") - if [ "$HTTP_CODE" = "200" ]; then - pass "Encrypted URL returns image (HTTP 200)" - else - fail "Encrypted URL returned HTTP $HTTP_CODE" - fi -else - fail "No encrypted URL to test" -fi - -# Test 7: Fetch image via allowlisted host (direct proxy) -echo "--- Test 7: Fetch image via direct proxy (allowlisted host) ---" -# URL format: /v1/image///. -PROXY_PATH="/v1/image/s3.sneak.cloud/sneak-public/2021/2021-04-18.untitled.a7r4.07723.jpg/400x300.jpeg" -HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$BASE_URL$PROXY_PATH") -if [ "$HTTP_CODE" = "200" ]; then - pass "Direct proxy returns image (HTTP 200)" -else - fail "Direct proxy returned HTTP $HTTP_CODE" -fi - -# Test 8: Logout -echo "--- Test 8: Logout ---" -curl -sf "$BASE_URL/logout" -b "$COOKIE_JAR" -c "$COOKIE_JAR" -L -o /dev/null -AFTER_LOGOUT=$(curl -sf "$BASE_URL/" -b "$COOKIE_JAR") -if echo "$AFTER_LOGOUT" | grep -qi "password\|login"; then - pass "Logout redirects to login page" -else - fail "Logout did not redirect to login" -fi - -# Test 9: Generate short-TTL URL and verify expiration -echo "--- Test 9: Expired URL returns 410 ---" -# Login again -curl -sf -X POST "$BASE_URL/" -d "key=$SIGNING_KEY" -c "$COOKIE_JAR" -b "$COOKIE_JAR" -L -o /dev/null -# Generate URL with 1 second TTL -GEN_RESULT=$(curl -sf -X POST "$BASE_URL/generate" -b "$COOKIE_JAR" \ - -d "url=$TEST_IMAGE_URL" \ - -d "width=100" \ - -d "height=100" \ - -d "format=jpeg" \ - -d "ttl=1") -SHORT_URL=$(echo "$GEN_RESULT" | grep -o '/v1/e/[^"<]*' | head -1) -if [ -n "$SHORT_URL" ]; then - echo " Waiting 2 seconds for URL to expire..." - sleep 2 - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$BASE_URL$SHORT_URL") - if [ "$HTTP_CODE" = "410" ]; then - pass "Expired URL returns 410 Gone" - else - fail "Expired URL returned HTTP $HTTP_CODE (expected 410)" - fi -else - fail "Could not generate short-TTL URL" -fi - -echo "" -echo "=== All tests passed! ===" From 8314099abd4651496cfa9f1800ae263177acad97 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 22:24:50 +0200 Subject: [PATCH 037/131] Refuse image requests whose Referer is on referer_blocklist (closes #90) A new setting, referer_blocklist (PIXA_REFERER_BLOCKLIST), lists hosts whose pages may not show pixa's images. Entries are written and matched as allowlist_hosts are, with the same matcher. Both image routes check the Referer before the signature, the cache and the upstream fetch, and answer 403 with the JSON error, so a blocked request costs nothing and is refused whether or not the image is cached. No Referer, or one that does not parse, is served; README.md says this makes the list easy to get around. An entry of either host list that is not a host name (letters, digits, hyphens, underscores, dots, at most one leading dot) or an IP address now aborts startup naming the setting and the entry. Model: opus-5-5 --- README.md | 33 +++- TODO.md | 15 +- configs/config.example.yml | 12 ++ internal/config/config.go | 119 +++++++---- .../config/config_validation_internal_test.go | 34 ++++ internal/config/env_internal_test.go | 2 + .../config/referer_blocklist_internal_test.go | 166 ++++++++++++++++ internal/handlers/handlers.go | 16 +- internal/handlers/image.go | 21 ++ internal/handlers/imageenc.go | 4 + .../referer_blocklist_internal_test.go | 185 ++++++++++++++++++ 11 files changed, 559 insertions(+), 48 deletions(-) create mode 100644 internal/config/referer_blocklist_internal_test.go create mode 100644 internal/handlers/referer_blocklist_internal_test.go diff --git a/README.md b/README.md index 0906560..8597c53 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ everything in this list without further settings. The reverse proxy must: Routes); - pass the `Host`, `Origin` and `Referer` headers on unchanged, as pixa refuses a form from those pages unless `Origin` or `Referer` names the host in `Host`, - and builds encrypted URLs from `Host`; + builds encrypted URLs from `Host`, and checks `Referer` against + `referer_blocklist`; - set `X-Forwarded-For` to the client's address, with `trusted_proxies` set to the address pixa sees the proxy's requests come from, so the login limit counts each client by its own address (see `trusted_proxies` under @@ -175,11 +176,13 @@ path under `/v1/` answers 200, in maintenance mode too. allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when - the upstream host, or a host it redirects to, is `localhost`, ends in - `.localhost` or `.local`, or has an address in a blocked network (see - `blocked_networks`); 502 when the upstream answered with an error status, and - for 5 minutes after that for the same source URL; 503 when pixa is busy or in - maintenance mode; 500 for any other failure. + the request's `Referer` names a host in `referer_blocklist`, checked before + the signature, the cache and the upstream fetch; 403 when the upstream host, + or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`, + or has an address in a blocked network (see `blocked_networks`); 502 when the + upstream answered with an error status, and for 5 minutes after that for the + same source URL; 503 when pixa is busy or in maintenance mode; 500 for any + other failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when `If-None-Match` matches the image's `ETag`; 400 for a token that does not @@ -392,6 +395,11 @@ and the URL is - **Suffix match**: `.example.com` — matches `cdn.example.com`, `images.example.com`, and `example.com` +An IP address is matched exactly; write an IPv6 address without brackets. An +entry that is neither a host name (letters, digits, hyphens, underscores and +dots, with at most one leading dot) nor an IP address, such as one with a port +or a `*.` wildcard, aborts startup. + ### Configuration Every setting can be given as an environment variable, in a YAML config @@ -428,6 +436,7 @@ tell. With no file, pixa uses the environment and the defaults. | `PIXA_DB_URL` | `db_url` | SQLite database URL; default `state.sqlite3` in the state directory | | `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of (free + cached) | | `PIXA_ALLOWLIST_HOSTS` | `allowlist_hosts` | Upstream hosts served without a signature | +| `PIXA_REFERER_BLOCKLIST` | `referer_blocklist` | Hosts whose pages the image routes refuse with 403, by `Referer` | | `PIXA_BLOCKED_NETWORKS` | `blocked_networks` | CIDR ranges never fetched from, on top of the built-in ones | | `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 | | `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` | @@ -456,6 +465,18 @@ Key settings in more detail: that has no leading zero and is not the scheme's default. Any other value, including another scheme such as a browser extension's, aborts startup - `allowlist_hosts` — list of allowed upstream hosts +- `referer_blocklist` — list of hosts whose pages may not show pixa's images, to + stop other sites hotlinking them. Entries are written and matched as for + `allowlist_hosts` (see Allowlist patterns), and an entry that is neither a + host name nor an IP address aborts startup. A request to `/v1/image/` or + `/v1/e/` whose `Referer` header names a listed host is refused with 403 before + its signature or token is checked and before the cache or the upstream host is + used, so it fetches nothing, and it is refused even when the image is cached. + A request with no `Referer`, or one that does not parse as a URL + with a host, is served, as many clients send none. So this is easily got + around: a site whose pages send no `Referer` (for example with + `Referrer-Policy: no-referrer`) is not stopped. It does not apply to the login + and generator pages. Default: empty - `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added to the always-enforced built-in ranges (loopback, private, link-local, CGNAT, benchmark, NAT64, and the like); an invalid CIDR diff --git a/TODO.md b/TODO.md index 18d33b4..9307f32 100644 --- a/TODO.md +++ b/TODO.md @@ -27,10 +27,22 @@ The disk cache is now size-bounded with LRU eviction # Next Step -P2: security: referer blacklist +P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 referer blocklist (closes #90): `referer_blocklist` + (`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for + `allowlist_hosts` with the same matcher; an entry of either list that is + neither a host name (letters, digits, hyphens, underscores and dots, with at + most one leading dot) nor an IP address, such as one with a port or a `*.` + wildcard, aborts startup naming the setting and the entry. Both image routes + refuse a request whose `Referer` names a listed host with 403 and a JSON error + before the signature, the cache and the upstream fetch, so it fetches nothing + and is refused whether or not the image is cached. A request with no + `Referer`, or one that does not parse as a URL with a host, is served, so the + list is easily got around; `README.md` and `configs/config.example.yml` say + so. It does not apply to the login and generator pages. - 2026-10-04 fewer files in the repository root (closes #97): `config.example.yml` moved unchanged to `configs/config.example.yml`, and `README.md`, the comments in `internal/config/config.go` and the startup error @@ -561,7 +573,6 @@ P2: security: referer blacklist # Future Steps - P2: security - - per-IP rate limiting on the image routes - per-origin rate limiting - P2: HTTP response handling - Last-Modified headers diff --git a/configs/config.example.yml b/configs/config.example.yml index 098d936..cede448 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -46,6 +46,8 @@ signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32" # Hosts that don't require signatures (default: none) # Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com") +# An entry that is neither a host name nor an IP address (IPv6 without +# brackets), such as one with a port or a "*." wildcard, aborts startup. allowlist_hosts: - s3.sneak.cloud - static.sneak.cloud @@ -53,6 +55,16 @@ allowlist_hosts: - github.com - user-images.githubusercontent.com +# Hosts whose pages may not show pixa's images, written as for +# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header +# names one of them is answered 403 before anything is fetched, even when +# the image is cached. A request with no Referer, or one that does not +# parse, is served, so a site whose pages send no Referer is not stopped. +# The login and generator pages are not covered. (default: none) +# referer_blocklist: +# - leech.example +# - .hotlinker.example + # Additional CIDR ranges to refuse when fetching upstream, extending the # SSRF protection. These are added to the always-enforced built-in ranges # (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and diff --git a/internal/config/config.go b/internal/config/config.go index 98addb0..f4e0fc4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "runtime" "sort" "strconv" @@ -48,6 +49,7 @@ const ( keyMetricsPassword = "metrics.password" keySigningKey = "signing_key" keyAllowlistHosts = "allowlist_hosts" + keyRefererBlocklist = "referer_blocklist" keyAllowHTTP = "allow_http" keyUpstreamConnectionsPerHost = "upstream_connections_per_host" keyUpstreamConnections = "upstream_connections" @@ -97,12 +99,11 @@ var ( "value is null; omit the key entirely to use the default") errValuesNull = errors.New( "value is null; omit a key entirely to use its default") - errNotBareHostname = errors.New( - "must be a bare hostname without scheme, path, or whitespace") - errNoHostnameLabels = errors.New("contains no hostname labels") - errNotADuration = errors.New("not a duration such as 30s or 2m") - errMustBePositive = errors.New("must be positive") - errNotAnOrigin = errors.New( + errNotAHost = errors.New("must be a host name such as " + + "cdn.example.com or .example.com, or an IP address") + errNotADuration = errors.New("not a duration such as 30s or 2m") + errMustBePositive = errors.New("must be positive") + errNotAnOrigin = errors.New( `not "*" or an origin such as https://example.com`) ) @@ -130,6 +131,10 @@ type Config struct { AllowHTTP bool // Allow non-TLS upstream (testing only) UpstreamConnectionsPerHost int // Max concurrent connections per upstream host + // RefererBlocklist holds host patterns, matched as AllowlistHosts is: the + // image routes refuse a request whose Referer names a matching host. + RefererBlocklist []string + // UpstreamConnections is the most concurrent connections to all // upstream hosts together, on top of the per-host limit. // MaxConcurrentProcessing is the most images processed at once. @@ -270,7 +275,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { } loader := &strictLoader{sc: sc} - c := &Config{ Debug: loader.boolVal(keyDebug, false), MaintenanceMode: loader.boolVal(keyMaintenanceMode, false), @@ -298,9 +302,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin), DownstreamTimeout: loader.durationVal( keyDownstreamTimeout, DefaultDownstreamTimeout), - CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0), - BlockedNetworks: blockedNetworks, - TrustedProxies: trustedProxies, + CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0), + BlockedNetworks: blockedNetworks, + TrustedProxies: trustedProxies, + RefererBlocklist: loader.hostListVal(keyRefererBlocklist), } // The default for an omitted cache_max_bytes is worked out when @@ -420,7 +425,8 @@ func isKnownConfigKey(key string) bool { keyUpstreamConnectionsPerHost, keyUpstreamConnections, keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks, keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout, - keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env": + keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist, + "env": return true } @@ -443,6 +449,7 @@ func envVarNames() map[string]string { keyMetricsPassword: "PIXA_METRICS_PASSWORD", keySigningKey: "PIXA_SIGNING_KEY", keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS", + keyRefererBlocklist: "PIXA_REFERER_BLOCKLIST", keyAllowHTTP: "PIXA_ALLOW_HTTP", keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST", keyUpstreamConnections: "PIXA_UPSTREAM_CONNECTIONS", @@ -612,7 +619,7 @@ func (c *Config) validate() error { } for _, host := range c.AllowlistHosts { - err := validateAllowlistHost(host) + err := validateHostPattern(keyAllowlistHosts, host) if err != nil { return err } @@ -735,25 +742,24 @@ func (c *Config) validateConcurrencyLimits() error { return nil } -// validateAllowlistHost checks that an allowlist_hosts entry is a bare -// hostname, optionally with a leading dot for suffix matching. URLs, -// paths, and whitespace indicate a misconfigured entry. An entry with -// no hostname labels (such as ".") is rejected: the allowlist matcher -// treats a leading dot as a suffix pattern, so a bare "." would match -// any upstream host written in FQDN trailing-dot form and effectively -// disable URL signing. -func validateAllowlistHost(host string) error { - if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") { - return fmt.Errorf("%s: entry %q %w", - settingName(keyAllowlistHosts), host, errNotBareHostname) +// hostNamePattern matches a host name: letters, digits, hyphens, underscores +// and dots, optionally after one leading dot. +var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`) + +// validateHostPattern checks that an entry of the named key, allowlist_hosts +// or referer_blocklist, is an IP address or a host name, the host name +// optionally with one leading dot for suffix matching. Anything else, such as +// a URL, a port or a "*." wildcard, can never match a host name that resolves, +// so it is refused. +// So is "." alone: the allowlist matcher would match it against any host +// written with a trailing dot, which in allowlist_hosts disables URL signing. +func validateHostPattern(key, host string) error { + _, err := netip.ParseAddr(host) + if err == nil || hostNamePattern.MatchString(host) { + return nil } - if strings.Trim(host, ".") == "" { - return fmt.Errorf("%s: entry %q %w", - settingName(keyAllowlistHosts), host, errNoHostnameLabels) - } - - return nil + return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost) } // loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var @@ -883,6 +889,19 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool { return val } +func (l *strictLoader) hostListVal(key string) []string { + if l.err != nil { + return nil + } + + val, err := parseHostList(l.sc, key) + if err != nil { + l.err = err + } + + return val +} + // getString returns the string value for key, or defaultVal if the key // is omitted. A present value that is not a string, or is explicitly // null, is an error. @@ -1180,7 +1199,7 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { return nil, errNullConfigValue(key) } - entries, err := cidrListEntries(raw, key) + entries, err := listEntries(raw, key) if err != nil { return nil, err } @@ -1200,11 +1219,41 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { return prefixes, nil } -// cidrListEntries extracts the raw entries of the named CIDR-list key as -// trimmed, non-empty strings, from either a YAML list of strings or a -// comma-separated string; an empty string is an empty list, as for -// allowlist_hosts. Any other shape is a configuration error. -func cidrListEntries(raw any, key string) ([]string, error) { +// parseHostList parses the value of the named config key into host patterns, +// or returns nil if the key is omitted. It accepts a YAML list of strings or a +// comma-separated string. An explicitly null value, a wrong type, an empty +// entry, a non-string entry, or an entry validateHostPattern rejects aborts +// startup naming the key and the offending value. +func parseHostList(sc *smartconfig.Config, key string) ([]string, error) { + raw, ok := lookupValue(sc, key) + if !ok { + return nil, nil + } + + if raw == nil { + return nil, errNullConfigValue(key) + } + + entries, err := listEntries(raw, key) + if err != nil { + return nil, err + } + + for _, entry := range entries { + err := validateHostPattern(key, entry) + if err != nil { + return nil, err + } + } + + return entries, nil +} + +// listEntries extracts the raw entries of the named list key as trimmed, +// non-empty strings, from either a YAML list of strings or a comma-separated +// string; an empty string is an empty list, as for allowlist_hosts. Any other +// shape is a configuration error. +func listEntries(raw any, key string) ([]string, error) { switch val := raw.(type) { case []any: entries := make([]string, 0, len(val)) diff --git a/internal/config/config_validation_internal_test.go b/internal/config/config_validation_internal_test.go index d3a3736..abfdee7 100644 --- a/internal/config/config_validation_internal_test.go +++ b/internal/config/config_validation_internal_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -216,6 +217,25 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) { } } +// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with +// an underscore, which pixa can fetch from, is accepted as an entry. +func TestAllowlistHostsAcceptsUnderscore(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts: + - my_bucket.example.com + - .my_bucket.example.org +`) + if err != nil { + t.Fatalf("host names with an underscore should load, got error: %v", err) + } + + want := []string{"my_bucket.example.com", ".my_bucket.example.org"} + if !slices.Equal(c.AllowlistHosts, want) { + t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want) + } +} + // runAbortCases asserts that each case's config aborts startup with an // error message mentioning every expected substring. func runAbortCases(t *testing.T, cases []abortCase) { @@ -318,6 +338,20 @@ func invalidHostAndCredentialCases() []abortCase { keyAllowlistHosts, "example.com/images", }, }, + { + name: "allowlist host with wildcard", + yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n", + wantErrSubstrings: []string{ + keyAllowlistHosts, "*.example.com", + }, + }, + { + name: "allowlist host with port", + yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n", + wantErrSubstrings: []string{ + keyAllowlistHosts, "example.com:8443", + }, + }, { name: "allowlist host with whitespace", yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n", diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go index dd90dfa..6eb79d9 100644 --- a/internal/config/env_internal_test.go +++ b/internal/config/env_internal_test.go @@ -65,6 +65,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { t.Setenv("PIXA_METRICS_PASSWORD", "metricspass") t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey) t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com") + t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example") t.Setenv("PIXA_ALLOW_HTTP", "true") t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5") t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10") @@ -93,6 +94,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { MetricsPassword: "metricspass", SigningKey: validTestSigningKey, AllowlistHosts: []string{testHostS3, ".example.com"}, + RefererBlocklist: []string{"hotlinker.example", ".leech.example"}, AllowHTTP: true, UpstreamConnectionsPerHost: 5, UpstreamConnections: 10, diff --git a/internal/config/referer_blocklist_internal_test.go b/internal/config/referer_blocklist_internal_test.go new file mode 100644 index 0000000..43d5753 --- /dev/null +++ b/internal/config/referer_blocklist_internal_test.go @@ -0,0 +1,166 @@ +package config + +import ( + "slices" + "testing" +) + +// TestRefererBlocklistParsed loads a referer_blocklist with a host and a +// pattern starting with "." and checks both are kept in order. +func TestRefererBlocklistParsed(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+`referer_blocklist: + - leech.example + - .hotlinker.example +`) + if err != nil { + t.Fatalf("valid referer_blocklist should load, got error: %v", err) + } + + want := []string{"leech.example", ".hotlinker.example"} + if !slices.Equal(c.RefererBlocklist, want) { + t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want) + } +} + +// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses, +// the IPv6 one written without brackets, are accepted as entries. +func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+`referer_blocklist: + - 192.0.2.7 + - "2001:db8::7" +`) + if err != nil { + t.Fatalf("IP address entries should load, got error: %v", err) + } + + want := []string{"192.0.2.7", "2001:db8::7"} + if !slices.Equal(c.RefererBlocklist, want) { + t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want) + } +} + +// TestRefererBlocklistAcceptsUnderscore checks that a host name with an +// underscore, which a page can be served from, is accepted as an entry. +func TestRefererBlocklistAcceptsUnderscore(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+`referer_blocklist: + - my_site.leech.example + - .my_site.hotlinker.example +`) + if err != nil { + t.Fatalf("host names with an underscore should load, got error: %v", err) + } + + want := []string{"my_site.leech.example", ".my_site.hotlinker.example"} + if !slices.Equal(c.RefererBlocklist, want) { + t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want) + } +} + +// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no +// referer. +func TestRefererBlocklistOmittedIsEmpty(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine) + if err != nil { + t.Fatalf("minimal config should be valid, got error: %v", err) + } + + if len(c.RefererBlocklist) != 0 { + t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist) + } +} + +// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a +// host, or a value that is not a list of them, aborts startup with an error +// naming the key and the entry. +func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) { + t.Parallel() + + runAbortCases(t, []abortCase{ + { + name: "entry with a scheme", + yaml: signingKeyLine + "referer_blocklist:\n - https://leech.example\n", + wantErrSubstrings: []string{ + keyRefererBlocklist, "https://leech.example", + }, + }, + { + name: "entry with a path", + yaml: signingKeyLine + "referer_blocklist:\n - leech.example/page\n", + wantErrSubstrings: []string{ + keyRefererBlocklist, "leech.example/page", + }, + }, + { + name: "wildcard entry", + yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n", + wantErrSubstrings: []string{ + keyRefererBlocklist, "*.leech.example", + }, + }, + { + name: "entry with a port", + yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n", + wantErrSubstrings: []string{ + keyRefererBlocklist, "leech.example:8080", + }, + }, + { + name: "two leading dots", + yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n", + wantErrSubstrings: []string{ + keyRefererBlocklist, "..leech.example", + }, + }, + { + name: "dot only", + yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n", + wantErrSubstrings: []string{keyRefererBlocklist, `"."`}, + }, + { + name: "empty entry", + yaml: signingKeyLine + "referer_blocklist:\n - \"\"\n", + wantErrSubstrings: []string{keyRefererBlocklist}, + }, + { + name: "entry not a string", + yaml: signingKeyLine + "referer_blocklist:\n - 42\n", + wantErrSubstrings: []string{keyRefererBlocklist, "42"}, + }, + { + name: "null value", + yaml: signingKeyLine + "referer_blocklist:\n", + wantErrSubstrings: []string{keyRefererBlocklist, nullValueText}, + }, + }) +} + +// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST +// takes comma-separated entries, and that an entry in it that is not a host +// aborts startup naming the variable and the entry. +func TestRefererBlocklistFromEnvironment(t *testing.T) { + t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey) + t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ") + + c, err := newFromSmartConfig(nil) + if err != nil { + t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err) + } + + want := []string{"leech.example", ".hotlinker.example"} + if !slices.Equal(c.RefererBlocklist, want) { + t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want) + } + + t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example") + + _, err = newFromSmartConfig(nil) + wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example") +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 489e3c3..a65f3b5 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -9,6 +9,7 @@ import ( "time" "go.uber.org/fx" + "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/encurl" @@ -40,6 +41,10 @@ type Handlers struct { sessMgr *session.Manager encGen *encurl.Generator csrfProtect func(http.Handler) http.Handler + + // refererBlocklist matches the hosts of referer_blocklist; its IsAllowed + // reports whether a URL's host is on that list. + refererBlocklist *allowlist.HostAllowList } // New creates a new Handlers instance. @@ -50,11 +55,12 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { } s := &Handlers{ - log: params.Logger.Get(), - hc: params.Healthcheck, - db: params.Database, - config: params.Config, - csrfProtect: csrfProtect, + log: params.Logger.Get(), + hc: params.Healthcheck, + db: params.Database, + config: params.Config, + csrfProtect: csrfProtect, + refererBlocklist: allowlist.New(params.Config.RefererBlocklist), } lc.Append(fx.Hook{ diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 1a46a58..e099799 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -21,6 +21,10 @@ import ( // /v1/image///x. func (s *Handlers) HandleImage() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if s.refuseBlockedReferer(w, r) { + return + } + req, ok := s.parseImageRequest(w, r) if !ok { return @@ -248,6 +252,23 @@ func cacheControl(expires time.Time) string { return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second)) } +// refuseBlockedReferer answers 403 with a JSON error when the request's Referer +// names a host on referer_blocklist, and reports whether it answered. A request +// with no Referer, or one that does not parse as a URL with a host, is not +// refused. +func (s *Handlers) refuseBlockedReferer( + w http.ResponseWriter, r *http.Request, +) bool { + referer, err := url.Parse(r.Referer()) + if err != nil || !s.refererBlocklist.IsAllowed(referer) { + return false + } + + s.respondError(w, "referer blocked", http.StatusForbidden) + + return true +} + // notModified sets the ETag header to etag and, when the request's // If-None-Match is that ETag, answers 304 Not Modified. It reports whether it // answered. An empty etag sets no header and never answers. diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index dadb6bd..a9c672f 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -22,6 +22,10 @@ import ( // browsers identify the content type. func (s *Handlers) HandleImageEnc() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if s.refuseBlockedReferer(w, r) { + return + } + ctx := r.Context() start := time.Now() diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go new file mode 100644 index 0000000..9235eb1 --- /dev/null +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -0,0 +1,185 @@ +package handlers + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "sneak.berlin/go/pixa/internal/allowlist" + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// blockedReferer is a page on leech.example, which newRefererRoutes puts on +// referer_blocklist. +const blockedReferer = "https://leech.example/page.html" + +// countingFetcher passes each fetch on to the fetcher it holds and counts it. +type countingFetcher struct { + httpfetcher.Fetcher + + fetches atomic.Int32 +} + +// Fetch counts the fetch and passes it on. +func (f *countingFetcher) Fetch( + ctx context.Context, url string, +) (*httpfetcher.FetchResult, error) { + f.fetches.Add(1) + + return f.Fetcher.Fetch(ctx, url) +} + +// newRefererRoutes returns both image routes of a Handlers whose +// referer_blocklist is "leech.example" and ".hotlinker.example", the +// Handlers, and the fetcher the routes fetch through. The JPEG at photoPath +// exists on allowlistedHost and on signedHost. +func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) { + t.Helper() + + fetcher := &countingFetcher{ + Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost), + } + + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + StateDir: t.TempDir(), + CacheTTL: time.Hour, + NegativeTTL: 5 * time.Minute, + }) + if err != nil { + t.Fatalf("imgcache.NewCache() error = %v", err) + } + + svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + Cache: cache, + Fetcher: fetcher, + SigningKey: testSigningKey, + Allowlist: []string{allowlistedHost}, + }) + if err != nil { + t.Fatalf("imgcache.NewService() error = %v", err) + } + + encGen, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator() error = %v", err) + } + + h := &Handlers{ + log: slog.New(slog.DiscardHandler), + imgSvc: svc, + encGen: encGen, + refererBlocklist: allowlist.New( + []string{"leech.example", ".hotlinker.example"}), + } + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + r.Get("/v1/e/{token}/*", h.HandleImageEnc()) + + return r, h, fetcher +} + +// getWithReferer sends a GET for target to routes with referer as its +// Referer header, or with none when referer is empty, and returns the +// response. +func getWithReferer( + t *testing.T, routes http.Handler, target, referer string, +) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil) + if referer != "" { + req.Header.Set("Referer", referer) + } + + rec := httptest.NewRecorder() + + routes.ServeHTTP(rec, req) + t.Logf("GET %s with Referer %q: %d", target, referer, rec.Code) + + return rec +} + +// TestRefererBlocklist verifies that both image routes refuse a request whose +// Referer names a host on referer_blocklist with 403 and the JSON error, +// without fetching from the upstream host, and serve a request with no +// Referer, one that does not parse, or one naming any other host. Hosts are +// matched as allowlist_hosts matches them. +func TestRefererBlocklist(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + referer string + want int + }{ + {"no referer", "", http.StatusOK}, + {"unlisted host", "https://unlisted.example/page.html", http.StatusOK}, + {"unparseable", "%zz", http.StatusOK}, + {"listed host", blockedReferer, http.StatusForbidden}, + {"subdomain of listed host", "https://www.leech.example/", http.StatusOK}, + {"subdomain of dot pattern", "https://www.hotlinker.example/a.html", + http.StatusForbidden}, + {"dot pattern without its dot", "https://hotlinker.example/", + http.StatusForbidden}, + {"host continuing past dot pattern", + "https://hotlinker.example.evil.example/", http.StatusOK}, + } + + // The photo's URL on each image route. + photoURLs := map[string]func(t *testing.T, h *Handlers) string{ + "plain URL": func(t *testing.T, _ *Handlers) string { + t.Helper() + + return photoURL(allowlistedHost) + }, + "encrypted URL": encPhotoURL, + } + + for urlName, photoURLFor := range photoURLs { + for _, tc := range cases { + t.Run(urlName+", "+tc.name, func(t *testing.T) { + t.Parallel() + + routes, h, fetcher := newRefererRoutes(t) + + rec := getWithReferer(t, routes, photoURLFor(t, h), tc.referer) + + if tc.want == http.StatusOK { + requireServedPhoto(t, rec) + + return + } + + checkErrorBody(t, rec, http.StatusForbidden, "referer blocked") + + if n := fetcher.fetches.Load(); n != 0 { + t.Errorf("upstream fetched %d times, want 0", n) + } + }) + } + } +} + +// TestBlockedRefererRefusedWhenImageIsCached verifies that a request whose +// Referer is on referer_blocklist is refused even when the image it asks for +// is already cached, so the answer does not depend on the cache. +func TestBlockedRefererRefusedWhenImageIsCached(t *testing.T) { + t.Parallel() + + routes, h, _ := newRefererRoutes(t) + + for _, target := range []string{photoURL(allowlistedHost), encPhotoURL(t, h)} { + requireServedPhoto(t, getWithReferer(t, routes, target, "")) + + rec := getWithReferer(t, routes, target, blockedReferer) + checkErrorBody(t, rec, http.StatusForbidden, "referer blocked") + } +} From 708a9bec208c6c6fc696f7f7238c24463e0ab8e3 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 22:42:03 +0200 Subject: [PATCH 038/131] Test that a URL made on the generator page with a ttl expires (closes #199) A new handler test makes a URL on the generator page with a ttl of one second, checks that /v1/e/ serves it at once, waits two seconds and checks that it then answers 410. The expiry is kept in whole seconds, so two seconds is the longest a one-second ttl can take to pass. Test only. Model: opus-5-5 --- TODO.md | 7 +++ .../handlers/auth_session_internal_test.go | 49 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/TODO.md b/TODO.md index 9307f32..da0ced6 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,13 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 a URL made on the generator page with a `ttl` is tested to + expire (closes #199): a new test in `internal/handlers` makes a URL on the + generator page with a `ttl` of one second, checks that `/v1/e/` serves it at + once, waits two seconds and checks that it then answers 410. The test waits + for real, as pixa reads the clock directly when it makes and checks a URL; it + waits two seconds because the time a URL expires is kept in whole seconds. + Test only. - 2026-10-04 referer blocklist (closes #90): `referer_blocklist` (`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for `allowlist_hosts` with the same matcher; an entry of either list that is diff --git a/internal/handlers/auth_session_internal_test.go b/internal/handlers/auth_session_internal_test.go index 3bc57f8..883e434 100644 --- a/internal/handlers/auth_session_internal_test.go +++ b/internal/handlers/auth_session_internal_test.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" "testing" + "time" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/session" @@ -241,3 +242,51 @@ func TestGeneratePost_URLServesImage(t *testing.T) { requireServedPhoto(t, imageRec) } + +// TestGeneratePost_URLWithTTLExpires verifies that a URL the generator page +// makes with a ttl of one second is served by /v1/e/ at once and answers 410 +// once the ttl has passed. +func TestGeneratePost_URLWithTTLExpires(t *testing.T) { + t.Parallel() + + _, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + rec := generatePost(t, url.Values{ + sourceURLField: {"https://" + signedHost + photoPath}, + widthField: {"50"}, + heightField: {"50"}, + formatField: {string(imgcache.FormatJPEG)}, + ttlField: {"1"}, + }) + + if rec.Code != http.StatusOK { + t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK) + } + + match := generatedURLPattern.FindStringSubmatch(rec.Body.String()) + if match == nil { + t.Fatalf("generator page shows no URL: %s", rec.Body.String()) + } + + imageRec := httptest.NewRecorder() + imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, match[1], nil)) + + requireServedPhoto(t, imageRec) + + // The URL keeps the time it expires in whole seconds and is served + // through the whole of that second, so a ttl of one second has passed + // for certain two seconds after the URL was made. + time.Sleep(2 * time.Second) + + imageRec = httptest.NewRecorder() + imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, match[1], nil)) + + t.Logf("GET %s after the ttl: %d %q", match[1], imageRec.Code, imageRec.Body) + + if imageRec.Code != http.StatusGone { + t.Errorf("status after the ttl = %d, want %d", + imageRec.Code, http.StatusGone) + } +} From cca2e3f926d380baeb11244991c3fd220ef231ff Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 23:24:46 +0200 Subject: [PATCH 039/131] Test the image proxy flow end to end (closes #80) TestImageProxyFlow in internal/server starts the database, handlers and middleware from the constructors pixad uses, with a fresh state directory, and replaces only the upstream origin with an httptest server. For a resize with a format change and for orig it checks a 200 MISS with the right type and size, then a HIT after one upstream request, and the files and rows the cache keeps. Two optional test seams make that possible: httpfetcher.Config.DialContext and handlers.Params.Fetcher. pixad sets neither and the config file and environment cannot, and tests show production still uses the checked dialer and builds its own fetcher. Model: opus-5-5 --- TODO.md | 16 +- internal/handlers/fetcher_internal_test.go | 61 ++++ internal/handlers/handlers.go | 11 +- .../httpfetcher/dial_context_internal_test.go | 66 ++++ internal/httpfetcher/httpfetcher.go | 23 +- internal/imgcache/service.go | 3 +- internal/server/image_proxy_internal_test.go | 302 ++++++++++++++++++ 7 files changed, 473 insertions(+), 9 deletions(-) create mode 100644 internal/handlers/fetcher_internal_test.go create mode 100644 internal/httpfetcher/dial_context_internal_test.go create mode 100644 internal/server/image_proxy_internal_test.go diff --git a/TODO.md b/TODO.md index da0ced6..27f0147 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,21 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 an integration test of the image proxy flow (closes #80): + `TestImageProxyFlow` in `internal/server` starts the database, handlers and + middleware from the constructors `pixad` uses, with a fresh state directory, + and replaces only the upstream origin with a local test server. For a resize + with a change to JPEG and for `orig`, the first request goes through the + router, the real fetcher, libvips, the disk cache and SQLite and answers 200 + with the right content type and size and `X-Pixa-Cache: MISS`; the second + answers `HIT` with the same image and the upstream has had one request; the + source and the converted image are then in `cache/sources` and + `cache/variants`, with their rows in `source_content`, `source_metadata` and + `variant_content`. Two optional fields make this possible, which `pixad` does + not set and the config file and environment cannot: + `httpfetcher.Config.DialContext` connects in place of the dialer that refuses + internal addresses, the URL and redirect checks still running, and + `handlers.Params.Fetcher` replaces the fetcher the handlers build. - 2026-10-04 a URL made on the generator page with a `ttl` is tested to expire (closes #199): a new test in `internal/handlers` makes a URL on the generator page with a `ttl` of one second, checks that `/v1/e/` serves it at @@ -591,5 +606,4 @@ P2: security: per-IP rate limiting on the image routes - optional Sentry error reporting - comprehensive request logging - Prometheus performance metrics - - integration tests for the image proxy flow - load tests to verify the 1k to 5k req/s target diff --git a/internal/handlers/fetcher_internal_test.go b/internal/handlers/fetcher_internal_test.go new file mode 100644 index 0000000..d78091a --- /dev/null +++ b/internal/handlers/fetcher_internal_test.go @@ -0,0 +1,61 @@ +package handlers + +import ( + "net/http" + "net/netip" + "path/filepath" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/healthcheck" + "sneak.berlin/go/pixa/internal/logger" +) + +// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as +// pixad does, in an fx app that provides no fetcher, and requests an image +// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL +// check accepts that address; only the dialer that refuses internal +// addresses checks blocked_networks, so the answer is 403 only if the +// fetcher the handlers build from the config connects with that dialer. Any +// other dialer would try to connect until the upstream fetch timeout, which +// is short so that the test then fails quickly. +func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) { + t.Parallel() + + const host = "192.0.2.10" + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + AllowlistHosts: []string{host}, + BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}, + UpstreamFetchTimeout: 2 * time.Second, + // With no connection slots, the fetch would fail before dialing. + UpstreamConnections: config.DefaultUpstreamConnections, + } + + var h *Handlers + + app := fxtest.New(t, + fx.Supply(cfg), + fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New), + fx.Populate(&h), + ) + app.RequireStart() + t.Cleanup(app.RequireStop) + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + + rec := sendGet(t, r, photoURL(host)) + checkErrorBody(t, rec, http.StatusForbidden, "forbidden") +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index a65f3b5..0414026 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -28,6 +28,11 @@ type Params struct { Healthcheck *healthcheck.Healthcheck Database *database.Database Config *config.Config + + // Fetcher, when provided, fetches upstream images in place of the + // fetcher the handlers build from the config. Only tests provide one; + // pixad does not. + Fetcher httpfetcher.Fetcher `optional:"true"` } // Handlers provides HTTP request handlers. @@ -36,6 +41,7 @@ type Handlers struct { hc *healthcheck.Healthcheck db *database.Database config *config.Config + fetcher httpfetcher.Fetcher imgSvc *imgcache.Service imgCache *imgcache.Cache sessMgr *session.Manager @@ -59,6 +65,7 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { hc: params.Healthcheck, db: params.Database, config: params.Config, + fetcher: params.Fetcher, csrfProtect: csrfProtect, refererBlocklist: allowlist.New(params.Config.RefererBlocklist), } @@ -128,10 +135,12 @@ func (s *Handlers) initImageService() error { fetcherCfg.MaxConnections = s.config.UpstreamConnections fetcherCfg.BlockedNetworks = s.config.BlockedNetworks - // Create the service + // Create the service. With no fetcher provided, it builds its own from + // fetcherCfg. svc, err := imgcache.NewService(&imgcache.ServiceConfig{ Cache: cache, FetcherConfig: fetcherCfg, + Fetcher: s.fetcher, SigningKey: s.config.SigningKey, Allowlist: s.config.AllowlistHosts, MaxConcurrentProcessing: s.config.MaxConcurrentProcessing, diff --git a/internal/httpfetcher/dial_context_internal_test.go b/internal/httpfetcher/dial_context_internal_test.go new file mode 100644 index 0000000..4553799 --- /dev/null +++ b/internal/httpfetcher/dial_context_internal_test.go @@ -0,0 +1,66 @@ +package httpfetcher + +import ( + "errors" + "net" + "testing" +) + +// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built +// without DialContext, as pixa builds it, refuses to connect to a local +// server. +func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + transport := transportOf(t, New(DefaultConfig())) + + addr := srv.Listener.Addr().String() + + _, err := transport.DialContext(testContext(t), "tcp", addr) + if !errors.Is(err, ErrSSRFBlocked) { + t.Fatalf("DialContext(%s) error = %v, want ErrSSRFBlocked", addr, err) + } +} + +// TestDialContextReplacesOnlyTheDialer checks that a fetcher built with +// DialContext connects through it, while the URL check still refuses a +// loopback URL and the redirect check a redirect to a link-local address. +func TestDialContextReplacesOnlyTheDialer(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + dialer := &recordingDialer{target: srv.Listener.Addr().String()} + + cfg := DefaultConfig() + cfg.AllowHTTP = true + cfg.DialContext = dialer.dialContext + f := New(cfg) + + if body := fetchBody(t, f, "/image"); body != imagePayload { + t.Errorf("body = %q, want %q", body, imagePayload) + } + + _, err := f.Fetch(testContext(t), "http://127.0.0.1/image") + if !errors.Is(err, ErrSSRFBlocked) { + t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err) + } + + _, err = f.Fetch(testContext(t), upstreamURL("/redirect/private")) + if !errors.Is(err, ErrSSRFBlocked) { + t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err) + } + + // The upstream server is reached through DialContext, and nothing else + // is asked of it. + dialed := dialer.dialedAddrs() + if len(dialed) == 0 { + t.Error("DialContext was never called") + } + + for _, addr := range dialed { + if addr != net.JoinHostPort(testPublicHost, "80") { + t.Errorf("DialContext was asked to connect to %s", addr) + } + } +} diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index f76e8b1..5640cf0 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -137,6 +137,11 @@ type Config struct { // BlockedNetworks are operator-supplied CIDR ranges refused by the // dialer, in addition to the always-enforced built-in ranges. BlockedNetworks []netip.Prefix + // DialContext, when set, makes the fetcher's connections in place of + // the dialer that refuses internal addresses; the URL and redirect + // checks still run. Only tests set it, to reach a local server; the + // config file and the environment cannot. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) } // DefaultConfig returns a Config with sensible defaults. @@ -190,13 +195,19 @@ func New(config *Config) *HTTPFetcher { config = DefaultConfig() } - // Create transport with SSRF-safe dialer. The dialer re-resolves and - // re-checks at connect time (closing the DNS-rebinding window) against - // both the built-in ranges and the operator-supplied blocklist. - transport := &http.Transport{ - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + // Unless config.DialContext replaces it, the transport connects with + // the SSRF-safe dialer, which re-resolves and re-checks at connect time + // (closing the DNS-rebinding window) against both the built-in ranges + // and the operator-supplied blocklist. + dialContext := config.DialContext + if dialContext == nil { + dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { return dialSSRFSafe(ctx, network, addr, config.BlockedNetworks) - }, + } + } + + transport := &http.Transport{ + DialContext: dialContext, TLSHandshakeTimeout: DefaultTLSTimeout, MaxIdleConns: DefaultMaxIdleConns, IdleConnTimeout: DefaultIdleConnTimeout, diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 562595c..2a40c8c 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -42,7 +42,8 @@ type Service struct { type ServiceConfig struct { // Cache is the cache instance Cache *Cache - // FetcherConfig configures the upstream fetcher (ignored if Fetcher is set) + // FetcherConfig configures the upstream fetcher built when Fetcher is + // not set. Its AllowHTTP and MaxResponseSize are used either way. FetcherConfig *httpfetcher.Config // Fetcher is an optional custom fetcher (for testing) Fetcher httpfetcher.Fetcher diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go new file mode 100644 index 0000000..24b7961 --- /dev/null +++ b/internal/server/image_proxy_internal_test.go @@ -0,0 +1,302 @@ +package server + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/handlers" + "sneak.berlin/go/pixa/internal/healthcheck" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/middleware" +) + +// upstreamHost is the upstream host of the image URLs below. It is a +// documentation address (RFC 5737), which the fetcher's URL check accepts as +// public; the fetcher's dial function connects it to the test upstream server. +const upstreamHost = "192.0.2.10" + +// TestImageProxyFlow requests images through pixa's router, handlers, +// upstream fetcher, image processor, disk cache and database, with only the +// upstream origin replaced by a local test server. The first request for a URL +// is fetched and converted; the second is served from the cache without +// another upstream request. The source and the converted image are then on +// disk, with their rows in the database. +func TestImageProxyFlow(t *testing.T) { + t.Parallel() + + source := encodeTestPNG(t, 64, 48) + + tests := []struct { + name string + sizeFormat string // the . part of the image URL + contentType string + decodeConfig func(io.Reader) (image.Config, error) + width, height int + }{ + {"resize and convert to JPEG", "32x24.jpeg", "image/jpeg", + jpeg.DecodeConfig, 32, 24}, + {"orig", "orig.orig", "image/png", png.DecodeConfig, 64, 48}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var upstreamRequests atomic.Int32 + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests.Add(1) + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(source) + })) + t.Cleanup(upstream.Close) + + s, db, stateDir := startImageProxy(t, upstream) + target := "/v1/image/" + upstreamHost + "/photo.png/" + tt.sizeFormat + + first := getImage(t, s, target) + + if got := first.Header().Get("X-Pixa-Cache"); got != "MISS" { + t.Errorf("first X-Pixa-Cache = %q, want MISS", got) + } + + if got := first.Header().Get("Content-Type"); got != tt.contentType { + t.Errorf("Content-Type = %q, want %q", got, tt.contentType) + } + + decoded, err := tt.decodeConfig(bytes.NewReader(first.Body.Bytes())) + if err != nil { + t.Fatalf("decoding the image: %v", err) + } + + if decoded.Width != tt.width || decoded.Height != tt.height { + t.Errorf("image is %dx%d, want %dx%d", + decoded.Width, decoded.Height, tt.width, tt.height) + } + + second := getImage(t, s, target) + + if got := second.Header().Get("X-Pixa-Cache"); got != "HIT" { + t.Errorf("second X-Pixa-Cache = %q, want HIT", got) + } + + if !bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) { + t.Error("the second response is not the image the first served") + } + + if got := upstreamRequests.Load(); got != 1 { + t.Errorf("upstream received %d requests, want 1", got) + } + + checkSourceCached(t, db, stateDir, source) + checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType) + }) + } +} + +// startImageProxy starts the components pixad's fx app builds, from a config +// with a fresh state directory and upstreamHost on the allowlist, and with an +// upstream fetcher that connects every upstream address to upstream. It +// returns the server with its routes, the database and the state directory. +func startImageProxy( + t *testing.T, upstream *httptest.Server, +) (*Server, *sql.DB, string) { + t.Helper() + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + AllowlistHosts: []string{upstreamHost}, + // The test upstream server has no TLS. + AllowHTTP: true, + // A limit of its own, so the cache does not size itself from the + // host's free disk space. + CacheMaxBytes: 64 << 20, + CacheMaxBytesExplicit: true, + UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize, + DownstreamTimeout: config.DefaultDownstreamTimeout, + } + + fetcherCfg := httpfetcher.DefaultConfig() + fetcherCfg.AllowHTTP = true + fetcherCfg.DialContext = func( + ctx context.Context, network, _ string, + ) (net.Conn, error) { + var dialer net.Dialer + + return dialer.DialContext(ctx, network, upstream.Listener.Addr().String()) + } + fetcher := httpfetcher.New(fetcherCfg) + + var ( + h *handlers.Handlers + mw *middleware.Middleware + db *database.Database + ) + + app := fxtest.New(t, + fx.Supply(cfg), + fx.Provide( + globals.New, + logger.New, + database.New, + healthcheck.New, + handlers.New, + middleware.New, + func() httpfetcher.Fetcher { return fetcher }, + ), + fx.Populate(&h, &mw, &db), + ) + app.RequireStart() + t.Cleanup(app.RequireStop) + + // Requests go straight to the router, as in newTestServer; the server's + // own start hook, which listens on a port, is left out. + s := &Server{config: cfg, mw: mw, h: h} + s.SetupRoutes() + + return s, db.DB(), stateDir +} + +// getImage sends a GET for target to s and fails unless it answers 200. +func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder { + t.Helper() + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d, X-Pixa-Cache %s", + target, rec.Code, rec.Header().Get("X-Pixa-Cache")) + + if rec.Code != http.StatusOK { + t.Fatalf("GET %s status = %d, want %d; body %s", + target, rec.Code, http.StatusOK, rec.Body.String()) + } + + return rec +} + +// checkSourceCached checks that source is stored under its SHA-256 in +// cache/sources, recorded in source_content, and that the source URL's row in +// source_metadata points at it. +func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) { + t.Helper() + + sum := sha256.Sum256(source) + hash := hex.EncodeToString(sum[:]) + + checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash), + source) + + var rows int + + err := db.QueryRowContext(t.Context(), + "SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash, + ).Scan(&rows) + if err != nil || rows != 1 { + t.Errorf("source_content rows for the source = %d (error %v), want 1", + rows, err) + } + + var metadataHash string + + err = db.QueryRowContext(t.Context(), + `SELECT content_hash FROM source_metadata + WHERE source_host = ? AND source_path = ?`, + upstreamHost, "/photo.png", + ).Scan(&metadataHash) + if err != nil || metadataHash != hash { + t.Errorf("source_metadata content_hash = %q (error %v), want %q", + metadataHash, err, hash) + } +} + +// checkVariantCached checks that the converted image served is recorded in +// variant_content with contentType, and stored under its cache key in +// cache/variants. +func checkVariantCached( + t *testing.T, db *sql.DB, stateDir string, served []byte, contentType string, +) { + t.Helper() + + var cacheKey, storedType string + + err := db.QueryRowContext(t.Context(), + "SELECT cache_key, content_type FROM variant_content", + ).Scan(&cacheKey, &storedType) + if err != nil { + t.Fatalf("variant_content row: %v", err) + } + + if storedType != contentType { + t.Errorf("variant_content content_type = %q, want %q", + storedType, contentType) + } + + checkFile(t, filepath.Join(stateDir, "cache", "variants", + cacheKey[0:2], cacheKey[2:4], cacheKey), served) +} + +// checkFile checks that the file at path holds want. +func checkFile(t *testing.T, path string, want []byte) { + t.Helper() + + //nolint:gosec // G304: a path under the test's state directory + got, err := os.ReadFile(path) + if err != nil { + t.Errorf("reading %s: %v", path, err) + + return + } + + if !bytes.Equal(got, want) { + t.Errorf("%s holds %d bytes that are not the %d expected", + path, len(got), len(want)) + } +} + +// encodeTestPNG returns an opaque width x height PNG of one color. +func encodeTestPNG(t *testing.T, width, height int) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + img.Set(x, y, color.RGBA{R: 200, G: 40, B: 40, A: 255}) + } + } + + var buf bytes.Buffer + + err := png.Encode(&buf, img) + if err != nil { + t.Fatalf("encoding the test PNG: %v", err) + } + + return buf.Bytes() +} From f3231a3c5ab6950024c68e684ddd600aa6db7351 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 23:59:29 +0200 Subject: [PATCH 040/131] Replace REPO_POLICIES.md with the canonical copy from sneak/prompts (closes #196) REPO_POLICIES.md is fetched unchanged from prompts/REPO_POLICIES.md on sneak/prompts main. The new rules pixa's tree breaks are filed as https://git.eeqj.de/sneak/pixa/issues/202 through https://git.eeqj.de/sneak/pixa/issues/206 and https://git.eeqj.de/sneak/pixa/issues/208 and not fixed here. Its rule that no build stage runs git describe is not followed, per https://git.eeqj.de/sneak/pixa/issues/166. Model: opus-5-5 --- REPO_POLICIES.md | 345 ++++++++++++++++++++++++++++++++++++----------- TODO.md | 15 +++ 2 files changed, 285 insertions(+), 75 deletions(-) diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..2256291 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-09-08 --- This document covers repository structure, tooling, and workflow standards. Code @@ -60,17 +60,28 @@ style conventions are in separate documents: prerequisite since nvm requires bash. yarn is then pinned via `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific - pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in - `script/precommit`, not in the hook itself. Model scripts are at + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -89,87 +100,140 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + Dockerfiles install development prerequisites by running `script/bootstrap` + rather than duplicating installs inline; COPY `script/` and the dependency + manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before + running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make fmt-check - RUN make lint + RUN golangci-lint run --config .golangci.yml ./... - # Build stage + # Test phase # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS test + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } + + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make test ARG VERSION=dev RUN CGO_ENABLED=0 go build -trimpath \ -ldflags="-s -w -X main.Version=${VERSION}" \ -o /app ./cmd/app/ - # Runtime stage + # Runtime stage, and the last one FROM alpine@sha256:... COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + `vips-dev`), install them in the lint phase with `apk add`. + - `ARG VERSION=dev` is declared in the stage that compiles and supplied by + `script/docker` and `script/cibuild`; no stage may call `git describe`. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -189,14 +253,21 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -209,11 +280,24 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so the target cannot report a pass it did not earn, and the rerun + reproduces a failure instead of replaying it. It leaves the build cache + alone, so it costs the runtime of the suite and no recompilation. + + Note that this is a second, independent cache, stacked below the Docker + layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26) + addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes; + it does not guarantee `go test` inside that step does any work, because the + `GOCACHE` baked into earlier image layers survives into the re-executed + step. They are two separate defects requiring two separate fixes, and a fix + for one must not be recorded as covering the other. + Python example: ```makefile @@ -239,10 +323,83 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.git`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **Excluding `.git` means `git describe` cannot run inside any build stage, and + it fails quietly there.** In a build stage there is no repository, so + `git describe` writes nothing to stdout, `-X main.Version=` comes out empty, + the binary reports no version at all, and the build still exits 0. Compute the + version on the host and thread it in as a build arg. `script/docker` and + `script/cibuild` do this, byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -258,9 +415,45 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`, + which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the + only pin, since no repo installs golangci-lint on the host: bumping the + version means changing it and nothing else. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -379,7 +572,9 @@ style conventions are in separate documents: language-specific config). Everything else goes in a subdirectory. Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) diff --git a/TODO.md b/TODO.md index 27f0147..7fecf2b 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,21 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 `REPO_POLICIES.md` matches the canonical copy again (closes #196): + it is replaced, unchanged, by `prompts/REPO_POLICIES.md` from `sneak/prompts` + `main`. The rules it adds that pixa's tree breaks are filed: + https://git.eeqj.de/sneak/pixa/issues/202 (lint and tests as `Dockerfile` + phases built with `--no-cache`), https://git.eeqj.de/sneak/pixa/issues/203 + (the workflow's `script/docker-smoke` step), + https://git.eeqj.de/sneak/pixa/issues/204 (`.claude/` in `.gitignore`), + https://git.eeqj.de/sneak/pixa/issues/205 (`.dockerignore` patterns at every + depth), https://git.eeqj.de/sneak/pixa/issues/206 (a thin + `cmd/pixad/main.go`) and https://git.eeqj.de/sneak/pixa/issues/208 + (`fetch-depth: 0` on the CI checkout, so the build sees the tags). Its rule + that no build stage runs `git describe` is not followed: pixa takes the + version from the `.git` in the build context, per + https://git.eeqj.de/sneak/pixa/issues/166, as the copy on `sneak/prompts` + `next` already says. - 2026-10-04 an integration test of the image proxy flow (closes #80): `TestImageProxyFlow` in `internal/server` starts the database, handlers and middleware from the constructors `pixad` uses, with a fresh state directory, From ef828f71a55d804f83843aa2994b941d7e10a999 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 00:07:37 +0200 Subject: [PATCH 041/131] Keep secrets out of the Docker build context at every depth (closes #205) .dockerignore patterns without a leading **/ match only at the root of the build context, so a nested .env or private key still reached it and, through COPY . ., a build-stage layer. The file is now the standard one from sneak/prompts: every pattern that should match anywhere has **/, and private keys and environment files are matched in any letter case. pixa keeps its own differences: .git is still sent without .git/config in place of the standard .git line, which the version stamp needs, and .gitignore, /bin and /data stay out. Model: opus-5-5 --- .dockerignore | 74 ++++++++++++++++++++++++++++++++++++++++++++------- TODO.md | 6 +++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index 549cdb4..8790b32 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,68 @@ -# .git is sent without its config. Without a VERSION build argument the -# stage that compiles runs `git describe --tags --always` on .git, which -# does not need .git/config; that file can hold a credential, such as a +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. + +# Unlike the standard file, which leaves out all of .git, pixa sends +# .git without its config. Without a VERSION build argument the stage +# that compiles runs `git describe --tags --always` on .git, which does +# not need .git/config; that file can hold a credential, such as a # password in a remote URL or the token the CI checkout step stores there. .git/config -.gitignore -.DS_Store -.env* + +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. .claude -node_modules -bin/ -data/ + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][dD]25519 + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* + +# pixa's own entries. Nothing in the build reads .gitignore. On the +# host, `make build` writes bin/pixad, and the example config keeps its +# state directory in data/. +.gitignore +/bin +/data diff --git a/TODO.md b/TODO.md index 7fecf2b..2a34c53 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,12 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 `.dockerignore` keeps secrets out at every depth (closes #205): the + file is now the standard one from `sneak/prompts`, whose patterns match in + every directory and, for environment files and private keys, in any letter + case, so a nested `.env` or `server.key` no longer reaches the build context. + pixa still sends `.git` without `.git/config` in place of the standard file's + `.git` line, and still leaves out `.gitignore`, `/bin` and `/data`. - 2026-10-04 `REPO_POLICIES.md` matches the canonical copy again (closes #196): it is replaced, unchanged, by `prompts/REPO_POLICIES.md` from `sneak/prompts` `main`. The rules it adds that pixa's tree breaks are filed: From 23ec4026f65e4f689688769085f8cc1ec3e20765 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 00:41:42 +0200 Subject: [PATCH 042/131] Ignore .claude/ in .gitignore (closes #204) REPO_POLICIES.md says in-repo agent scratch belongs in both .gitignore and .dockerignore. .dockerignore already has .claude; .gitignore now has the .claude/ entry and its comment exactly as the canonical .gitignore in sneak/prompts has them, unanchored so it matches at every depth. Model: opus-5-5 --- .gitignore | 6 ++++++ TODO.md | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 50fad0e..24690f7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,12 @@ Thumbs.db .vscode/ *.sublime-* +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + # Environment / secrets .env .env.* diff --git a/TODO.md b/TODO.md index 2a34c53..b13608a 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,10 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 `.gitignore` ignores `.claude/` (closes #204): the entry and its + comment are copied from the canonical `.gitignore` in `sneak/prompts`, + unanchored so it matches at every depth. `.dockerignore` already has + `.claude`. - 2026-10-04 `.dockerignore` keeps secrets out at every depth (closes #205): the file is now the standard one from `sneak/prompts`, whose patterns match in every directory and, for environment files and private keys, in any letter From 2beba15ae7fc5e5198325ed74d500d9f78f42cae Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 01:07:47 +0200 Subject: [PATCH 043/131] Move pixad's startup from cmd/pixad into internal/app (closes #206) cmd/pixad/main.go built the command line and its --config flag, set PIXA_CONFIG_PATH from that flag, ignored SIGPIPE and started the fx app. REPO_POLICIES.md now requires cmd/ to hold only one call into internal/ or pkg/, so that code moves unchanged to Run in the new internal/app package, and main calls app.Run(Version). Version stays in main, so the -X main.Version build flags in the Dockerfile and the Makefile do not change. Model: opus-5-5 --- TODO.md | 5 +++ cmd/pixad/main.go | 63 ++------------------------------- internal/app/app.go | 70 +++++++++++++++++++++++++++++++++++++ internal/globals/globals.go | 4 +-- 4 files changed, 79 insertions(+), 63 deletions(-) create mode 100644 internal/app/app.go diff --git a/TODO.md b/TODO.md index b13608a..f725367 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,11 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 `cmd/pixad/main.go` is one call into `internal/` (closes #206): + what it did (the command line and its `--config` flag, setting + `PIXA_CONFIG_PATH`, ignoring `SIGPIPE`, starting the fx app) is now `Run` in + `internal/app`, unchanged, and `main` calls it with `Version`, which the build + still sets through `-X main.Version`. That code had no tests to move. - 2026-10-04 `.gitignore` ignores `.claude/` (closes #204): the entry and its comment are copied from the canonical `.gitignore` in `sneak/prompts`, unanchored so it matches at every depth. `.dockerignore` already has diff --git a/cmd/pixad/main.go b/cmd/pixad/main.go index be56d5f..7806619 100644 --- a/cmd/pixad/main.go +++ b/cmd/pixad/main.go @@ -1,69 +1,10 @@ // Package main is the entry point for the pixad image proxy server. package main -import ( - "fmt" - "os" - "os/signal" - "syscall" - - "github.com/spf13/cobra" - "go.uber.org/fx" - "sneak.berlin/go/pixa/internal/config" - "sneak.berlin/go/pixa/internal/database" - "sneak.berlin/go/pixa/internal/globals" - "sneak.berlin/go/pixa/internal/handlers" - "sneak.berlin/go/pixa/internal/healthcheck" - "sneak.berlin/go/pixa/internal/logger" - "sneak.berlin/go/pixa/internal/middleware" - "sneak.berlin/go/pixa/internal/server" -) +import "sneak.berlin/go/pixa/internal/app" var Version string //nolint:gochecknoglobals // set by ldflags -var configPath string //nolint:gochecknoglobals // cobra flag - func main() { - rootCmd := &cobra.Command{ - Use: "pixad", - Short: "Pixa image caching proxy server", - Run: run, - } - - rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file") - - err := rootCmd.Execute() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func run(_ *cobra.Command, _ []string) { - globals.Version = Version - - // Set config path in environment if specified via flag - if configPath != "" { - _ = os.Setenv("PIXA_CONFIG_PATH", configPath) - } - - // A write to a closed stdout or stderr must not end the process. - signal.Ignore(syscall.SIGPIPE) - - fx.New( - fx.Provide( - config.New, - database.New, - globals.New, - handlers.New, - logger.New, - server.New, - middleware.New, - healthcheck.New, - ), - fx.Invoke( - func(log *logger.Logger) { log.Identify() }, - func(*server.Server) {}, - ), - ).Run() + app.Run(Version) } diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..3521236 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,70 @@ +// Package app reads the pixad command line and runs the server. +package app + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/spf13/cobra" + "go.uber.org/fx" + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/handlers" + "sneak.berlin/go/pixa/internal/healthcheck" + "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/middleware" + "sneak.berlin/go/pixa/internal/server" +) + +var configPath string //nolint:gochecknoglobals // cobra flag + +// Run reads the command line and runs the server until it stops, with +// version as the version pixad logs and reports. It exits the process +// with status 1 when the command line is not valid. +func Run(version string) { + globals.Version = version + + rootCmd := &cobra.Command{ + Use: "pixad", + Short: "Pixa image caching proxy server", + Run: run, + } + + rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file") + + err := rootCmd.Execute() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(_ *cobra.Command, _ []string) { + // Set config path in environment if specified via flag + if configPath != "" { + _ = os.Setenv("PIXA_CONFIG_PATH", configPath) + } + + // A write to a closed stdout or stderr must not end the process. + signal.Ignore(syscall.SIGPIPE) + + fx.New( + fx.Provide( + config.New, + database.New, + globals.New, + handlers.New, + logger.New, + server.New, + middleware.New, + healthcheck.New, + ), + fx.Invoke( + func(log *logger.Logger) { log.Identify() }, + func(*server.Server) {}, + ), + ).Run() +} diff --git a/internal/globals/globals.go b/internal/globals/globals.go index ac00731..7a3d2bb 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -7,8 +7,8 @@ import ( const appname = "pixad" -// Version is populated from main() via ldflags. -var Version string //nolint:gochecknoglobals // set from main +// Version is set by app.Run to the version main was built with. +var Version string //nolint:gochecknoglobals // set by app.Run // Globals holds application-wide constants. type Globals struct { From ae7c3f226d8a161b54a233d61f365f0cb48c4379 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 01:24:41 +0200 Subject: [PATCH 044/131] Keep local config files out of the Docker build context (closes #211) config.yaml and config.dev.yml are kept out of git because they can hold the signing key, but .dockerignore did not leave them out, so a local copy in the working tree reached the build context and, through COPY . ., a build-stage layer. .dockerignore now leaves them out in every directory and in any letter case. configs/config.example.yml is still sent. Model: opus-5-5 --- .dockerignore | 4 ++++ TODO.md | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/.dockerignore b/.dockerignore index 8790b32..6586021 100644 --- a/.dockerignore +++ b/.dockerignore @@ -66,3 +66,7 @@ .gitignore /bin /data + +# Local config files, kept out of git because they can hold the signing key. +**/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL] +**/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL] diff --git a/TODO.md b/TODO.md index f725367..fefbc63 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,13 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 local config files stay out of the Docker build context (closes + #211): `.dockerignore` now leaves out `config.yaml` and `config.dev.yml` in + every directory and in any letter case, the local config files `.gitignore` + keeps out of git because they can hold the signing key. + `configs/config.example.yml` is still sent. `config.yml`, which Getting + Started creates, is in neither file: + https://git.eeqj.de/sneak/pixa/issues/212. - 2026-10-04 `cmd/pixad/main.go` is one call into `internal/` (closes #206): what it did (the command line and its `--config` flag, setting `PIXA_CONFIG_PATH`, ignoring `SIGPIPE`, starting the fx app) is now `Run` in From f77faf13de3844f1a7da048b0375f81e5656f3a2 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 01:58:32 +0200 Subject: [PATCH 045/131] Keep config.yml out of git and the Docker build context (closes #212) Getting Started has you create config.yml at the repository root with a real signing key, but neither .gitignore nor .dockerignore left it out, so it could be committed and, through COPY . ., reach a build-stage layer. .gitignore now ignores it next to config.yaml, and .dockerignore leaves it out in every directory and in any letter case, as it already does config.yaml and config.dev.yml. Model: opus-5-5 --- .dockerignore | 1 + .gitignore | 1 + TODO.md | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/.dockerignore b/.dockerignore index 6586021..c2dfed1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -68,5 +68,6 @@ /data # Local config files, kept out of git because they can hold the signing key. +**/[cC][oO][nN][fF][iI][gG].[yY][mM][lL] **/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL] **/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL] diff --git a/.gitignore b/.gitignore index 24690f7..8549307 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,6 @@ node_modules/ *.sqlite3 # Local dev configs +config.yml config.yaml config.dev.yml diff --git a/TODO.md b/TODO.md index fefbc63..2c39457 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,11 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 `config.yml` stays out of git and the Docker build context (closes + #212): `.gitignore` now ignores `config.yml`, the config file Getting Started + creates with the signing key, and `.dockerignore` leaves it out in every + directory and in any letter case, as it already did `config.yaml` and + `config.dev.yml`. - 2026-10-04 local config files stay out of the Docker build context (closes #211): `.dockerignore` now leaves out `config.yaml` and `config.dev.yml` in every directory and in any letter case, the local config files `.gitignore` From c434581a54a082a87bf3b247a67969fb6f8a42b1 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 02:07:32 +0200 Subject: [PATCH 046/131] Fetch the tags in the CI checkout (closes #208) The standard checkout action clones shallow and fetches no tags, so the version the build takes from `git describe --tags --always` would be a bare commit even on a tagged commit. The checkout step now sets `fetch-depth: 0`, as REPO_POLICIES.md asks of a repo that takes its version from the tags. Model: opus-5-5 --- .gitea/workflows/check.yml | 5 +++++ TODO.md | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index decade9..bf68d24 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,5 +6,10 @@ jobs: steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # The default clone is shallow and has no tags, so the + # version the build takes from `git describe` would be a + # bare commit; this fetches the whole history with its tags. + with: + fetch-depth: 0 - run: script/cibuild - run: script/docker-smoke diff --git a/TODO.md b/TODO.md index 2c39457..c7d14a1 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,10 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 the CI checkout fetches the tags (closes #208): the checkout step + in `.gitea/workflows/check.yml` sets `fetch-depth: 0`, as `REPO_POLICIES.md` + asks of a repo that takes its version from the tags, so a CI build of a tagged + commit stamps the tag from `git describe` instead of a bare commit. - 2026-10-04 `config.yml` stays out of git and the Docker build context (closes #212): `.gitignore` now ignores `config.yml`, the config file Getting Started creates with the signing key, and `.dockerignore` leaves it out in every From 55cf7f4face1b2f0bfebf22b46e0a40449894a45 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 02:58:41 +0200 Subject: [PATCH 047/131] Add script/loadtest to measure throughput, latency and memory (closes #81) script/loadtest [duration [clients]] (make loadtest, defaults 10s and 4) measures pixad in three scenarios: hit (one cached image), miss (a new source image every request) and herd (each new source image asked for by all clients at once). For each it prints vegeta's report (requests per second, p50/p95/p99 latency, status codes), pixad's peak resident memory and how many requests reached the origin. It is a benchmark, not run by script/check. The origin it uses, internal/loadtestorigin behind cmd/loadtest-origin, answers every path with one generated JPEG. Bad arguments are refused before the build. README.md says how to run and read it and keeps 1-5k r/s as a target not yet measured at scale. Model: opus-5-5 --- Makefile | 7 +- README.md | 63 ++++++- TODO.md | 18 +- cmd/loadtest-origin/main.go | 9 + internal/loadtestorigin/loadtestorigin.go | 87 +++++++++ .../loadtestorigin_internal_test.go | 51 +++++ script/loadtest | 177 ++++++++++++++++++ 7 files changed, 406 insertions(+), 6 deletions(-) create mode 100644 cmd/loadtest-origin/main.go create mode 100644 internal/loadtestorigin/loadtestorigin.go create mode 100644 internal/loadtestorigin/loadtestorigin_internal_test.go create mode 100755 script/loadtest diff --git a/Makefile b/Makefile index 708d5e5..ee52f5c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks +.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") LDFLAGS := -X main.Version=$(VERSION) @@ -59,6 +59,11 @@ docker: docker-smoke: @script/docker-smoke +# Measure throughput, latency and peak memory with the default duration and +# number of clients (needs Docker and Go; a benchmark, not part of check) +loadtest: + @script/loadtest + # Build Docker image tagged pixad:$(VERSION) and pixad:latest docker-versioned: docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest . diff --git a/README.md b/README.md index 8597c53..ac26fa2 100644 --- a/README.md +++ b/README.md @@ -146,10 +146,11 @@ name. Multiple source paths may reference the same content blob; the database tracks references rather than using filesystem refcounting. -Toward a target of 1-5k r/s, pixa keeps in memory the content types of -the 10,000 transformed images most recently cached or served, so a -cache hit on one of them reads only the image file from disk and not -the metadata file stored beside it. + +pixa's target is 1-5k r/s, which has not been measured at that rate (see Load +Test). Toward it, pixa keeps in memory the content types of the 10,000 +transformed images most recently cached or served, so a cache hit on one of them +reads only the image file from disk and not the metadata file stored beside it. ### Routes @@ -576,6 +577,8 @@ them. We provide: - `script/check` — run test, lint, and fmt-check - `script/docker` — build the Docker image tagged via `script/projectname` - `script/docker-smoke` — build the image, start it, wait for it to be healthy +- `script/loadtest` — measure pixad's throughput, latency and peak memory; a + benchmark, not part of `script/check` (see Load Test) - `script/cibuild` — CI entrypoint: `docker build .` with a new `CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of coming from the build cache, and a green run implies a green repo @@ -584,6 +587,58 @@ them. We provide: - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` +## Load Test + +`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how +much memory it uses. It is a benchmark, not a check: `script/check` does not run +it. It needs Docker and Go. + +```bash +script/loadtest # 10 seconds per scenario, 4 clients +script/loadtest 30s 32 # 30 seconds per scenario, 32 clients +``` + +It builds the image with `script/docker` and the load tool, +[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario +starts a new pixad container and a new origin container, `cmd/loadtest-origin`: +an upstream host that answers every path with the same generated 1600x1200 JPEG. +vegeta then sends requests from the given number of clients, each sending its +next request as soon as its last one is answered, all for an image resized to +400x300 WebP: + +- `hit`: the same image every time, put in the cache first; +- `miss`: a new source image every time, so pixad fetches and converts each one; +- `herd`: each new source image once per client in a row, so that all clients + ask for it at the same time and share one fetch and one conversion (see + Routes). + +pixad refuses upstream hosts with private or local addresses, so the containers +share a Docker network in `203.0.113.0/24`, a range set aside for documentation. +A second run on the same Docker host while one is going fails, as it cannot +create that network. + +For each scenario the script prints vegeta's report and two lines of its own: + +- `Requests [total, rate, throughput]`: the requests sent, how many were sent + per second, and how many were answered successfully per second; the last is + the number to compare with the target under Storage; +- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request + to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th + percentiles; +- `Status Codes` and `Error Set`: anything other than `200` means the other + numbers are not for the scenario described, such as `503` when pixad was busy; +- `Bytes In`: `0`, as vegeta is told not to keep the images it receives; +- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since + its container started, in kB; for `hit` it includes the request that put the + image in the cache; +- `requests to the origin`: the fetches pixad made: one for `hit`, one per + request for `miss`, and one per image for `herd`, that is the requests sent + divided by the number of clients. + +The numbers depend on the machine and on whatever else runs on it. The first +measurement, made on a shared machine with few clients, is in `TODO.md`; it says +nothing about the target. + ## TODO See [TODO.md](TODO.md) for the full prioritized task list. diff --git a/TODO.md b/TODO.md index c7d14a1..29b9e2e 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,21 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]` + (`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check` + does not run, measures three scenarios, each against a new pixad container and + a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss` + (a new source image every request) and `herd` (each new source image asked for + by all clients at once). For each it prints vegeta's report (requests per + second, latency percentiles, status codes), pixad's peak resident memory and + the requests that reached the origin. `README.md` says how to run it and read + it, and keeps 1-5k r/s as a target not yet measured. First measurement, with + the defaults on a shared 48-CPU machine with other work running: a baseline + for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95 + 8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so + not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100 + MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms, + peak 60 MiB, 188 fetches for 749 requests. - 2026-10-04 the CI checkout fetches the tags (closes #208): the checkout step in `.gitea/workflows/check.yml` sets `fetch-depth: 0`, as `REPO_POLICIES.md` asks of a repo that takes its version from the tags, so a CI build of a tagged @@ -652,4 +667,5 @@ P2: security: per-IP rate limiting on the image routes - optional Sentry error reporting - comprehensive request logging - Prometheus performance metrics - - load tests to verify the 1k to 5k req/s target + - measure the 1k to 5k req/s target with `script/loadtest` on a machine not + shared with other work diff --git a/cmd/loadtest-origin/main.go b/cmd/loadtest-origin/main.go new file mode 100644 index 0000000..5a8a93c --- /dev/null +++ b/cmd/loadtest-origin/main.go @@ -0,0 +1,9 @@ +// Command loadtest-origin is the upstream host script/loadtest points pixad +// at; internal/loadtestorigin says what it does. +package main + +import "sneak.berlin/go/pixa/internal/loadtestorigin" + +func main() { + loadtestorigin.Run() +} diff --git a/internal/loadtestorigin/loadtestorigin.go b/internal/loadtestorigin/loadtestorigin.go new file mode 100644 index 0000000..6119163 --- /dev/null +++ b/internal/loadtestorigin/loadtestorigin.go @@ -0,0 +1,87 @@ +// Package loadtestorigin is the upstream host script/loadtest points pixad +// at, run by cmd/loadtest-origin. It answers every request, whatever its path, +// with the same generated JPEG, so each new path is a new source image for +// pixad to fetch, and it logs one line per request, so its log counts pixad's +// fetches. +package loadtestorigin + +import ( + "bytes" + "image" + "image/color" + "image/jpeg" + "log/slog" + "math" + "net/http" + "os" + "time" +) + +const ( + listenAddress = ":80" + readHeaderTimeout = 10 * time.Second + imageWidth = 1600 + imageHeight = 1200 + jpegQuality = 85 +) + +// Run makes the image and serves it on port 80 until the server fails, then +// exits the process with status 1. +func Run() { + photo, err := makeJPEG() + if err != nil { + slog.Error("cannot make the image", "error", err) + os.Exit(1) + } + + server := &http.Server{ + Addr: listenAddress, + Handler: newHandler(photo), + ReadHeaderTimeout: readHeaderTimeout, + } + + err = server.ListenAndServe() + slog.Error("server stopped", "error", err) + os.Exit(1) +} + +// newHandler answers every request with photo and logs the request's path. +func newHandler(photo []byte) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + slog.Info("request", "path", r.URL.Path) + w.Header().Set("Content-Type", "image/jpeg") + _, _ = w.Write(photo) + }) +} + +// makeJPEG draws colour gradients crossed with a fine pattern, so the image +// has detail to decode and does not compress to almost nothing. +func makeJPEG() ([]byte, error) { + img := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight)) + + // red and green count up from 0 to 255 and wrap around, along each row + // and down the image. + var green uint8 + + for y := range imageHeight { + var red uint8 + + for x := range imageWidth { + img.SetRGBA(x, y, color.RGBA{ + R: red, G: green, B: red ^ green, A: math.MaxUint8, + }) + red++ + } + + green++ + } + + var buf bytes.Buffer + + err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/internal/loadtestorigin/loadtestorigin_internal_test.go b/internal/loadtestorigin/loadtestorigin_internal_test.go new file mode 100644 index 0000000..b97b32f --- /dev/null +++ b/internal/loadtestorigin/loadtestorigin_internal_test.go @@ -0,0 +1,51 @@ +package loadtestorigin + +import ( + "bytes" + "image/jpeg" + "net/http" + "net/http/httptest" + "testing" +) + +// TestEveryPathServesTheSameJPEG checks that the origin answers any path with +// 200 and the same JPEG, so every new path script/loadtest asks pixad for is +// a valid source image. +func TestEveryPathServesTheSameJPEG(t *testing.T) { + t.Parallel() + + photo, err := makeJPEG() + if err != nil { + t.Fatalf("makeJPEG: %v", err) + } + + size, err := jpeg.DecodeConfig(bytes.NewReader(photo)) + if err != nil { + t.Fatalf("the image does not decode as a JPEG: %v", err) + } + + if size.Width != imageWidth || size.Height != imageHeight { + t.Errorf("the image is %dx%d, want %dx%d", + size.Width, size.Height, imageWidth, imageHeight) + } + + handler := newHandler(photo) + + for _, path := range []string{"/", "/miss/1.jpg", "/herd/2.jpg"} { + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, path, nil)) + + if rec.Code != http.StatusOK { + t.Errorf("%s: status = %d, want %d", path, rec.Code, http.StatusOK) + } + + if ct := rec.Header().Get("Content-Type"); ct != "image/jpeg" { + t.Errorf("%s: Content-Type = %q, want image/jpeg", path, ct) + } + + if !bytes.Equal(rec.Body.Bytes(), photo) { + t.Errorf("%s: the body is not the image", path) + } + } +} diff --git a/script/loadtest b/script/loadtest new file mode 100755 index 0000000..bcc361e --- /dev/null +++ b/script/loadtest @@ -0,0 +1,177 @@ +#!/bin/sh +# script/loadtest: measure pixad's throughput, latency and peak memory. +# +# script/loadtest [duration [clients]] (defaults: 10s and 4) +# +# A benchmark, not a check: script/check does not run it. It needs Docker +# and Go. It builds the image with script/docker and builds vegeta, the +# load tool, from a pinned commit. Each scenario then gets a new pixad +# container and a new origin container (cmd/loadtest-origin, which answers +# every path with the same JPEG), and vegeta sends requests for +# from clients at once, each asking for an image resized to +# 400x300 WebP: +# +# hit the same image every time, put in the cache first +# miss a new source image every time +# herd each new source image once per client in a row, so that all +# clients ask for it at the same time +# +# For each, it prints vegeta's report, pixad's peak resident memory and +# how many requests reached the origin. README.md says how to read them. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +# vegeta v12.13.0, 2026-10-04 +VEGETA_COMMIT=4b240c3089fa4aa10816542d64a74294d974211f + +# pixad refuses upstream hosts with private or local addresses, so the +# containers share a network in 203.0.113.0/24, a range set aside for +# documentation (RFC 5737) that pixad does not refuse and that is never +# routed on the internet. +SUBNET=203.0.113.0/24 + +usage() { + echo "usage: script/loadtest [duration [clients]]" >&2 + exit 2 +} + +main() { + duration="${1:-10s}" + clients="${2:-4}" + # The duration is a whole number, not zero (vegeta takes 0 to mean no + # end), followed by ms, s, m or h. + case "$duration" in + *ms) number="${duration%ms}" ;; + *s | *m | *h) number="${duration%?}" ;; + *) usage ;; + esac + case "$number" in + "" | *[!0-9]*) usage ;; + esac + [ "$number" -gt 0 ] || usage + # The number of clients is a whole number that does not start with 0, + # which also refuses zero: vegeta reads a leading 0 as octal. + case "$clients" in + *[!0-9]* | 0*) usage ;; + esac + + cd "$ROOT" + run="pixa-loadtest-$$" + tmp="$(mktemp -d)" + trap cleanup EXIT + trap 'exit 1' HUP INT TERM + + "$SCRIPT_DIR/docker" + # The image's ID, so a build elsewhere that moves the tag does not + # change what a later scenario starts. + image="$(docker image inspect --format '{{.Id}}' \ + "$("$SCRIPT_DIR/projectname")")" + + GOBIN="$tmp" go install "github.com/tsenart/vegeta/v12@$VEGETA_COMMIT" + # The origin runs in a container, so it is built for the Docker host. + CGO_ENABLED=0 GOOS=linux \ + GOARCH="$(docker version --format '{{.Server.Arch}}')" \ + go build -o "$tmp/loadtest-origin" ./cmd/loadtest-origin + + docker network create --subnet "$SUBNET" "$run" >/dev/null + + start_containers + # Put the image the hit scenario asks for in the cache. + docker exec "$run-pixad" wget -q -O /dev/null \ + "http://localhost:8080/v1/image/origin/hit.jpg/400x300.webp" + attack hit hit_targets + stop_containers + + start_containers + attack miss miss_targets + stop_containers + + start_containers + attack herd herd_targets + stop_containers +} + +# start_containers starts a new origin and a new pixad, and waits up to 30 +# seconds for pixad's health check to pass. +start_containers() { + docker run -d --name "$run-origin" \ + --network "$run" --network-alias origin \ + -v "$tmp/loadtest-origin:/usr/local/bin/loadtest-origin:ro" \ + --entrypoint /usr/local/bin/loadtest-origin "$image" >/dev/null + docker run -d --name "$run-pixad" \ + --network "$run" -p 127.0.0.1::8080 --health-interval=1s \ + -e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \ + -e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \ + "$image" >/dev/null + + waited=0 + until [ "$(docker inspect --format '{{.State.Health.Status}}' \ + "$run-pixad")" = healthy ]; do + if [ "$waited" -ge 30 ]; then + echo "loadtest: pixad not healthy after 30 seconds; its log:" >&2 + docker logs "$run-pixad" >&2 + exit 1 + fi + sleep 1 + waited=$((waited + 1)) + done + pixa="http://$(docker port "$run-pixad" 8080/tcp)" +} + +stop_containers() { + docker rm -f "$run-pixad" "$run-origin" >/dev/null +} + +# attack : send the requests prints and +# report on them. +attack() { + echo + echo "== $1: $clients clients for $duration" + "$2" | "$tmp/vegeta" attack -lazy -rate 0 -workers "$clients" \ + -max-workers "$clients" -duration "$duration" -max-body 0 | + "$tmp/vegeta" report + # pixad is process 1 in its container: the entrypoint execs it. + echo "pixad peak memory (VmHWM):" \ + "$(docker exec "$run-pixad" awk '/^VmHWM:/ { print $2, $3 }' \ + /proc/1/status)" + echo "requests to the origin:" \ + "$(docker logs "$run-origin" 2>&1 | grep -c ' request ')" +} + +# The targets functions print vegeta targets until vegeta stops reading. + +hit_targets() { + while :; do + echo "GET $pixa/v1/image/origin/hit.jpg/400x300.webp" + done +} + +miss_targets() { + i=0 + while :; do + i=$((i + 1)) + echo "GET $pixa/v1/image/origin/miss/$i.jpg/400x300.webp" + done +} + +herd_targets() { + i=0 + while :; do + i=$((i + 1)) + n=0 + while [ "$n" -lt "$clients" ]; do + n=$((n + 1)) + echo "GET $pixa/v1/image/origin/herd/$i.jpg/400x300.webp" + done + done +} + +cleanup() { + docker rm -f "$run-pixad" "$run-origin" >/dev/null 2>&1 || : + docker network rm "$run" >/dev/null 2>&1 || : + rm -rf "$tmp" +} + +main "$@" From a941a80bf9cfc6c13969f4e97dfc1d4be9a87d15 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 03:41:50 +0200 Subject: [PATCH 048/131] Run lint and tests as Dockerfile phases built with --no-cache (closes #202) script/check, cibuild, docker, lint, test, setup and install-precommit are now the sneak/prompts main copies, unchanged: lint and test each build their Dockerfile phase with --no-cache. The lint phase runs golangci-lint from the image REPO_POLICIES.md names, with libvips-dev from apt-get; the test phase runs the tests with a 90-second timeout; the build stage depends on both. script/bootstrap installs the C compiler and image libraries only with --cgo, which the test phase and build stage pass, and refreshes the apt lists before its first apt install. Dockerfile.lint and CHECK_EPOCH are gone, and make docker-versioned and docker-test call the scripts. Without VERSION the build stage still uses git describe, per issue 166. Model: opus-5-5 --- Dockerfile | 68 ++++++++++++++++++++++------------------ Dockerfile.lint | 34 -------------------- Makefile | 18 +++++------ README.md | 35 ++++++++++++++------- TODO.md | 18 +++++++++++ script/bootstrap | 33 ++++++++++++++----- script/check | 5 +-- script/cibuild | 29 +++++++++++------ script/docker | 18 +++++++---- script/install-precommit | 2 +- script/lint | 40 ++++++++--------------- script/setup | 3 +- script/test | 28 ++++++----------- 13 files changed, 175 insertions(+), 156 deletions(-) delete mode 100644 Dockerfile.lint diff --git a/Dockerfile b/Dockerfile index 424ee87..d811825 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,62 @@ -# Lint stage -# Same image as Dockerfile.lint: change both pins together. -# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 -FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint +# Lint phase. script/lint builds it alone. The linter is run directly: +# `make lint` and script/lint are themselves a docker build. +# golangci/golangci-lint:v2.12.2, 2026-10-04 +FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint + +# The linter compiles every package, and govips needs the libvips +# headers for that. REPO_POLICIES.md has the lint phase install them +# itself; this image is Debian, so with apt-get rather than apk. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN golangci-lint run --config .golangci.yml ./... + +# Test phase. script/test builds it alone. +# golang:1.25.4-alpine, 2026-02-25 +FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test WORKDIR /src -# script/bootstrap installs the build dependencies and downloads the Go -# modules. Only script/, go.mod and go.sum are copied first, so this -# layer is reused until one of them changes. +# script/bootstrap --cgo installs the build dependencies (a C compiler +# and the libvips and libheif headers) and downloads the Go modules. COPY script/ ./script/ COPY go.mod go.sum ./ -RUN script/bootstrap +RUN script/bootstrap --cgo -# Copy source code COPY . . -# Tells script/lint it is inside a container, so it runs the linter. -ENV container=docker +# Without -v first; on a failure, again with -v for the details, and +# the step fails even if the second run passes. +RUN go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } -# Run formatting check and linter. script/cibuild and script/docker pass -# a new CHECK_EPOCH on every run, and each check step names it in its -# command, so a new value reruns the step instead of reusing a cached -# success that checked nothing. A plain `docker build .` leaves it empty -# and reuses the check steps only for an identical build context. -ARG CHECK_EPOCH -RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check -RUN echo "check epoch: ${CHECK_EPOCH}" && make lint - -# Build stage +# Build stage. Nothing is wanted from the two phases above: these copies +# make BuildKit build them first, so this stage runs only when lint and +# test passed. # golang:1.25.4-alpine, 2026-02-25 FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder -# Depend on lint stage passing COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null WORKDIR /src -# Build dependencies and Go modules, as in the lint stage +# Build dependencies and Go modules, as in the test phase COPY script/ ./script/ COPY go.mod go.sum ./ -RUN script/bootstrap +RUN script/bootstrap --cgo # Copy source code COPY . . -# Run tests; a new CHECK_EPOCH reruns them, as in the lint stage. -ARG CHECK_EPOCH -RUN echo "check epoch: ${CHECK_EPOCH}" && make test - # VERSION is declared here, not earlier: a new value reruns only the -# build, not script/bootstrap or the tests. Given none, the version is +# build, not script/bootstrap. Given none, the version is # `git describe --tags --always` of the .git in the build context (git # comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH # after one, the short commit when no tag is reachable. A context that @@ -68,7 +75,8 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \ -ldflags "-s -w -X main.Version=${version}" \ -o /pixad ./cmd/pixad -# Runtime stage +# Runtime stage, and the last one: a plain `docker build .` builds this +# stage and what it depends on, and nothing else. # alpine:3.21, 2026-02-25 FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 diff --git a/Dockerfile.lint b/Dockerfile.lint deleted file mode 100644 index 41e350b..0000000 --- a/Dockerfile.lint +++ /dev/null @@ -1,34 +0,0 @@ -# Dockerfile.lint: the container script/lint builds to run golangci-lint, -# which is never installed on the host. Pinned to the same image as the -# Dockerfile lint stage: change both pins together, or the two run -# different linter versions. -# -# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 -FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 - -WORKDIR /src - -# pixa is CGO/libvips: the type-aware linters compile every package, so -# this image needs the same C libraries the build does. script/bootstrap -# installs them and downloads the Go modules. Only script/, go.mod and -# go.sum are copied first; they settle this layer's result, so it may -# safely be reused between runs. -COPY script/ ./script/ -COPY go.mod go.sum ./ -RUN script/bootstrap - -COPY . . - -# Tells script/lint it is inside a container, so it runs the linter. -ENV container=docker - -# script/lint passes a different CACHEBUST on every run, and BuildKit -# keys every RUN after this ARG on its value, so the lint step always -# runs instead of returning a cached success that linted nothing. -# -# Go's and golangci-lint's caches (/root/.cache, hundreds of MB) go on a -# tmpfs that is discarded after the step. Written into the layer, they -# would pile up as build cache on every run, since no later run, with -# its new CACHEBUST, can reuse that layer. -ARG CACHEBUST -RUN --mount=type=tmpfs,target=/root/.cache script/lint diff --git a/Makefile b/Makefile index ee52f5c..188f643 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev LDFLAGS := -X main.Version=$(VERSION) # Use nix-shell to provide CGO dependencies unless they are already available -# (e.g. inside a Docker build or an existing nix-shell). +# (e.g. inside an existing nix-shell). HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null) ifdef HAS_PKGCONFIG NIX_RUN_PREFIX = @@ -32,11 +32,11 @@ fmt-check: fmt: @script/fmt -# Run linter +# Run linter (the lint phase of the Dockerfile) lint: @script/lint -# Run tests (30-second timeout) +# Run tests (the test phase of the Dockerfile) test: @script/test @@ -64,20 +64,20 @@ docker-smoke: loadtest: @script/loadtest -# Build Docker image tagged pixad:$(VERSION) and pixad:latest +# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION) docker-versioned: - docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest . + @script/docker + docker tag pixa pixa:$(VERSION) -# Run tests in Docker (needed for CGO/libvips) +# Run tests in Docker, as `make test` does docker-test: - docker build --target builder --build-arg VERSION=$(VERSION) -t pixad-builder . - docker run --rm pixad-builder sh -c "CGO_ENABLED=1 GOTOOLCHAIN=auto go test -v ./..." + @script/test # Run local dev server in Docker devserver: docker-versioned devserver-stop docker run -d --name pixad-dev -p 8080:8080 \ -v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \ - pixad:latest + pixa:latest @echo "pixad running at http://localhost:8080" # Stop dev server diff --git a/README.md b/README.md index ac26fa2..ce00d83 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,11 @@ another part of pixa failed to stop. A request not finished by then is cut off. Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as it uses libvips through CGO; building it also needs their development files, -`pkg-config` and a C compiler. `script/bootstrap` installs all of these with -nix, apt, brew or apk. +`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these, +as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which +`script/setup` and `script/cibuild` run, installs only git, make and Go: the +checks compile pixa in Docker, so the host needs none of the C libraries. Docker +itself must already be installed. ## Running under upaas @@ -565,28 +568,38 @@ standard: normalized scripts in `script/` are the entrypoints for the development workflow, and the Makefile targets are thin shims that call them. We provide: -- `script/bootstrap` — install all dependencies (idempotent) +- `script/bootstrap` — install git, make and Go and download the Go modules + (idempotent); with `--cgo`, also the C compiler and the libvips and libheif + libraries that compiling pixa needs - `script/setup` — make a fresh clone ready for development (bootstrap, then install-precommit) - `script/projectname` — output the project name ("pixa") -- `script/test` — run the test suite -- `script/lint` — run golangci-lint, always in a container (builds - `Dockerfile.lint` when run outside one) +- `script/test` — run the test suite: build the `test` phase of the + `Dockerfile`, tagged `pixa-test` +- `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`, + tagged `pixa-lint`; the linter never runs on the host - `script/fmt` — format all code (writes) -- `script/fmt-check` — check formatting (read-only) +- `script/fmt-check` — check formatting (read-only), on the host - `script/check` — run test, lint, and fmt-check -- `script/docker` — build the Docker image tagged via `script/projectname` +- `script/docker` — build the Docker image tagged via `script/projectname`, with + the version from `git describe`; the image's build stage depends on the `lint` + and `test` phases, so this runs them too - `script/docker-smoke` — build the image, start it, wait for it to be healthy - `script/loadtest` — measure pixad's throughput, latency and peak memory; a benchmark, not part of `script/check` (see Load Test) -- `script/cibuild` — CI entrypoint: `docker build .` with a new - `CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of - coming from the build cache, and a green run implies a green repo +- `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`), + then `script/check`, then build the image as `script/docker` does - `script/precommit` — pre-commit checks (`go mod tidy` guard, then `script/check`) - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` +Every `docker build` in these scripts passes `--no-cache`, so the lint and test +phases run on every build instead of coming from the build cache. +`script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`, +`script/setup` and `script/install-precommit` are the standard copies from +`sneak/prompts`, kept identical to them. + ## Load Test `script/loadtest` (or `make loadtest`) measures how fast pixad answers and how diff --git a/TODO.md b/TODO.md index 29b9e2e..ffff926 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,24 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-05 lint and tests run as the `lint` and `test` phases of the + `Dockerfile`, built with `--no-cache` (closes #202): `script/check`, + `script/cibuild`, `script/docker`, `script/lint`, `script/test`, + `script/setup` and `script/install-precommit` are now the copies from + `sneak/prompts` `main`, unchanged. The `lint` phase runs golangci-lint from + the image `REPO_POLICIES.md` names, with `libvips-dev` from `apt-get`; the + `test` phase runs the tests with a 90-second timeout; the build stage depends + on both. `Dockerfile.lint` and the `CHECK_EPOCH` build argument are gone, the + formatting check runs on the host, and `make docker-versioned` and + `make docker-test` call the scripts. `script/bootstrap`, `script/fmt`, + `script/fmt-check`, `script/precommit` and `script/projectname` stay pixa's + own. `script/bootstrap` installs git, make and Go, refreshing apt's package + lists before its first apt install; with `--cgo`, which only the `test` phase + and the build stage pass, it also installs the C compiler and the libvips and + libheif libraries. The stage that compiles still takes the version from + `git describe` when no `VERSION` is given, per + https://git.eeqj.de/sneak/pixa/issues/166, so the copied scripts' comment that + `.dockerignore` leaves out `.git` does not hold for pixa. - 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]` (`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check` does not run, measures three scenarios, each against a new pixad container and diff --git a/script/bootstrap b/script/bootstrap index 54c04ea..e826a28 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -4,11 +4,15 @@ # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes NOTHING is present (not git, # make, or go). The linter is never installed on the host: golangci-lint -# runs only inside a container, Dockerfile.lint or the Dockerfile lint -# stage (see script/lint). A C compiler and the CGO image libraries -# (pkg-config, vips, libheif) are installed for the govips bindings. -# Both Dockerfiles run this script too, so their build dependencies are -# the ones listed here. +# runs only in the lint phase of the Dockerfile (see script/lint). +# +# script/bootstrap git, make and Go, all the host needs: the +# checks compile pixa in Docker +# script/bootstrap --cgo also a C compiler and the CGO image +# libraries (pkg-config, vips, libheif) for +# the govips bindings, to compile pixa; the +# Dockerfile's test phase and build stage +# run this set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -35,6 +39,10 @@ detect_pkgmgr() { if [ "$(id -u)" != "0" ]; then SUDO="sudo" fi + # This runs before the first install only. A fresh image, such + # as a CI runner's, has no package lists, and apt-get install + # finds no package without them. + $SUDO apt-get update fi } @@ -71,7 +79,16 @@ ensure_cgo_deps() { fi } +usage() { + echo "usage: script/bootstrap [--cgo]" >&2 + exit 2 +} + main() { + case "$*" in + "" | --cgo) ;; + *) usage ;; + esac cd "$ROOT" # Base tooling @@ -81,8 +98,10 @@ main() { # Go toolchain if missing go; then pkg_install go golang go go; fi - # CGO image libraries - ensure_cgo_deps + # CGO image libraries, only where pixa is compiled + if [ "$*" = "--cgo" ]; then + ensure_cgo_deps + fi go mod download diff --git a/script/check b/script/check index cc046f7..92875f7 100755 --- a/script/check +++ b/script/check @@ -1,7 +1,8 @@ #!/bin/sh # script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. -# Generic: usually needs no adaptation. +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/cibuild b/script/cibuild index 948134c..688299f 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,18 +1,29 @@ #!/bin/sh -# script/cibuild: run the CI build. The Dockerfile runs the checks -# (make fmt-check, lint, test) as build steps. This script passes a new -# CHECK_EPOCH on every run, so Docker runs those steps instead of -# reusing cached results: a successful run means the checks ran and -# passed on this tree. Generic: needs no adaptation. The Gitea workflow -# runs this on push. +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - epoch="$(date +%s)$$" - docker build --build-arg CHECK_EPOCH="$epoch" . + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@" diff --git a/script/docker b/script/docker index b9907d8..c4688e8 100755 --- a/script/docker +++ b/script/docker @@ -1,9 +1,8 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. -# Identical in all repos; the tag comes from script/projectname. Like -# script/cibuild, it passes a new CHECK_EPOCH, so the build runs the -# checks instead of reusing cached results. Generic: needs no -# adaptation. +# Identical in all repos; the tag comes from script/projectname. +# --no-cache because the gate phases the final stage depends on are RUN +# steps, and a cached one is a check that did not run. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -11,8 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - epoch="$(date +%s)$$" - docker build --build-arg CHECK_EPOCH="$epoch" \ + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ -t "$("$SCRIPT_DIR/projectname")" . } diff --git a/script/install-precommit b/script/install-precommit index 49da044..bef6406 100755 --- a/script/install-precommit +++ b/script/install-precommit @@ -1,13 +1,13 @@ #!/bin/sh # script/install-precommit: install the git pre-commit hook that runs # script/precommit. Our own extension to scripts-to-rule-them-all. -# Generic: needs no adaptation. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" + hook=".git/hooks/pre-commit" printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit chmod +x .git/hooks/pre-commit echo "pre-commit hook installed: runs script/precommit" diff --git a/script/lint b/script/lint index 96aa46d..2d8b075 100755 --- a/script/lint +++ b/script/lint @@ -1,37 +1,23 @@ #!/bin/sh -# script/lint: run golangci-lint over the whole tree. This is the only -# way the linter is run, everywhere; it is never installed on the host. +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. # -# Inside a container it runs the linter. Anywhere else it builds -# Dockerfile.lint, whose last step runs this script again inside that -# container. -# -# Dockerfile.lint and the Dockerfile lint stage set container=docker -# (the systemd convention for marking a container) to say where we are. -# /.dockerenv cannot: it is missing inside build steps, and present on -# hosts that are themselves containers. +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - if [ "${container:-}" = docker ]; then - # `golangci-lint config verify` is not run: it fetches its JSON - # schema over an unpinned live HTTPS call, which REPO_POLICIES.md - # forbids. - echo "Running linter..." - golangci-lint run --config .golangci.yml ./... - else - # A new CACHEBUST on every run means the lint step is never - # served from cache (see Dockerfile.lint). The cacheonly output - # leaves no image behind. - docker build \ - --progress=plain \ - --build-arg CACHEBUST="$(date +%s)-$$" \ - --output=type=cacheonly \ - -f Dockerfile.lint . - fi + docker build --no-cache \ + --target lint \ + -t "$("$SCRIPT_DIR/projectname")-lint" . } main "$@" diff --git a/script/setup b/script/setup index 53327ba..4cc5b6b 100755 --- a/script/setup +++ b/script/setup @@ -1,7 +1,6 @@ #!/bin/sh # script/setup: set up the repo for development after a fresh clone: -# installs dependencies (script/bootstrap) and the git pre-commit hook. -# Add any repo-specific initialization (db init, .env template) here. +# installs dependencies and the git pre-commit hook. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/test b/script/test index c0bcc75..cd239f2 100755 --- a/script/test +++ b/script/test @@ -1,27 +1,19 @@ #!/bin/sh -# script/test: run the test suite. CGO dependencies (pkg-config, vips, -# libheif) come from nix-shell when not already available (e.g. inside -# a Docker build or an existing nix-shell). +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -run_with_cgo_deps() { - if command -v pkg-config >/dev/null 2>&1; then - sh -c "$1" - else - nix-shell -p pkg-config vips libheif git --run "$1" - fi -} +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - echo "Running tests..." - # Run without -v first for clean output on success; on failure rerun - # with -v for full diagnostics, then exit non-zero (REPO_POLICIES.md - # conditional-verbose-rerun pattern). The first run already proved the - # tests broken, so the build fails even if the rerun happens to pass. - run_with_cgo_deps "CGO_ENABLED=1 go test -timeout 30s -race -cover ./... || { echo '--- Rerunning with -v for details ---'; CGO_ENABLED=1 go test -timeout 30s -race -v ./...; exit 1; }" + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . } main "$@" From 01823d27dba433d4412b7446cff2d56e224bb01d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 04:24:45 +0200 Subject: [PATCH 049/131] Format the markdown with prettier in script/fmt and script/fmt-check (closes #100) script/fmt and script/fmt-check run prettier 3.8.1 on the markdown after gofmt, with the same yarn helper and arguments as the copies in sneak/prompts. prettier is pinned in package.json and yarn.lock; .prettierrc sets four-space tabs and proseWrap always, and .prettierignore keeps prettier off REPO_POLICIES.md and vendor/. Plain script/bootstrap installs Node and Yarn the way the one in sneak/prompts does; with --cgo it does not, as the Dockerfile stages that pass it format nothing. The HTML templates stay out: prettier cannot parse a Go template action inside a tag. This commit also holds the reflow that make fmt then produced (lines rewrapped, bullets as dashes, no word changed), which the PR kept as a separate commit for review. Model: opus-5-5 --- .prettierignore | 7 + .prettierrc | 4 + CLAUDE.md | 105 ++++---- README.md | 346 ++++++++++++------------ TODO.md | 682 +++++++++++++++++++++++------------------------ package.json | 6 + script/bootstrap | 101 ++++++- script/fmt | 20 ++ script/fmt-check | 20 ++ yarn.lock | 8 + 10 files changed, 710 insertions(+), 589 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc create mode 100644 package.json create mode 100644 yarn.lock diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..888b018 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,7 @@ +node_modules/ +yarn.lock + +# A byte-for-byte copy of the one in sneak/prompts. +REPO_POLICIES.md + +vendor/ diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..8af31cd --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "proseWrap": "always" +} diff --git a/CLAUDE.md b/CLAUDE.md index c9041c8..af2e77c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,73 +4,68 @@ Last Updated 2026-01-08 These rules MUST be followed at all times, it is very important. -* Never use `git add -A` - add specific changes to a deliberate commit. A - commit should contain one change. After each change, make a commit with a - good one-line summary. +- Never use `git add -A` - add specific changes to a deliberate commit. A commit + should contain one change. After each change, make a commit with a good + one-line summary. -* NEVER modify the linter config without asking first. +- NEVER modify the linter config without asking first. -* NEVER modify tests to exclude special cases or otherwise get them to pass - without asking first. In almost all cases, the code should be changed, - NOT the tests. If you think the test needs to be changed, make your case - for that and ask for permission to proceed, then stop. You need explicit - user approval to modify existing tests. (You do not need user approval - for writing NEW tests.) +- NEVER modify tests to exclude special cases or otherwise get them to pass + without asking first. In almost all cases, the code should be changed, NOT the + tests. If you think the test needs to be changed, make your case for that and + ask for permission to proceed, then stop. You need explicit user approval to + modify existing tests. (You do not need user approval for writing NEW tests.) -* When linting, assume the linter config is CORRECT, and that each item - output by the linter is something that legitimately needs fixing in the - code. +- When linting, assume the linter config is CORRECT, and that each item output + by the linter is something that legitimately needs fixing in the code. -* When running tests, use `make test`. +- When running tests, use `make test`. -* Before commits, run `make check`. This runs `make lint` and `make test` - and `make check-fmt`. Any issues discovered MUST be resolved before - committing unless explicitly told otherwise. +- Before commits, run `make check`. This runs `make lint` and `make test` and + `make check-fmt`. Any issues discovered MUST be resolved before committing + unless explicitly told otherwise. -* When fixing a bug, write a failing test for the bug FIRST. Add - appropriate logging to the test to ensure it is written correctly. Commit - that. Then go about fixing the bug until the test passes (without - modifying the test further). Then commit that. +- When fixing a bug, write a failing test for the bug FIRST. Add appropriate + logging to the test to ensure it is written correctly. Commit that. Then go + about fixing the bug until the test passes (without modifying the test + further). Then commit that. -* When adding a new feature, do the same - implement a test first (TDD). It - doesn't have to be super complex. Commit the test, then commit the - feature. +- When adding a new feature, do the same - implement a test first (TDD). It + doesn't have to be super complex. Commit the test, then commit the feature. -* When adding a new feature, use a feature branch. When the feature is - completely finished and the code is up to standards (passes `make check`) - then and only then can the feature branch be merged into `main` and the - branch deleted. +- When adding a new feature, use a feature branch. When the feature is + completely finished and the code is up to standards (passes `make check`) then + and only then can the feature branch be merged into `main` and the branch + deleted. -* Write godoc documentation comments for all exported types and functions as - you go along. +- Write godoc documentation comments for all exported types and functions as you + go along. -* ALWAYS be consistent in naming. If you name something one thing in one - place, name it the EXACT SAME THING in another place. +- ALWAYS be consistent in naming. If you name something one thing in one place, + name it the EXACT SAME THING in another place. -* Be descriptive and specific in naming. `wl` is bad; - `SourceHostWhitelist` is good. `ConnsPerHost` is bad; - `MaxConnectionsPerHost` is good. +- Be descriptive and specific in naming. `wl` is bad; `SourceHostWhitelist` is + good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good. -* This is not prototype or teaching code - this is designed for production. - Any security issues (such as denial of service) or other web - vulnerabilities are P1 bugs and must be added to TODO.md at the top. +- This is not prototype or teaching code - this is designed for production. Any + security issues (such as denial of service) or other web vulnerabilities are + P1 bugs and must be added to TODO.md at the top. -* As this is production code, no stubbing of implementations unless - specifically instructed. We need working implementations. +- As this is production code, no stubbing of implementations unless specifically + instructed. We need working implementations. -* NEVER silently fall back to a different setting when a user's parameter - explicitly specifies a value. If a user requests format=webp and WebP - encoding is not supported, return an error - do NOT silently output PNG - instead. If a user specifies fit=invalid and that fit mode doesn't exist, - return an error - do NOT silently default to "cover". Silent fallbacks - violate the principle of least surprise and mask bugs. The only acceptable - defaults are for OMITTED parameters, never for INVALID explicit values. +- NEVER silently fall back to a different setting when a user's parameter + explicitly specifies a value. If a user requests format=webp and WebP encoding + is not supported, return an error - do NOT silently output PNG instead. If a + user specifies fit=invalid and that fit mode doesn't exist, return an error - + do NOT silently default to "cover". Silent fallbacks violate the principle of + least surprise and mask bugs. The only acceptable defaults are for OMITTED + parameters, never for INVALID explicit values. -* Avoid vendoring deps unless specifically instructed to. NEVER commit - the vendor directory, NEVER commit compiled binaries. If these - directories or files exist, add them to .gitignore (and commit the - .gitignore) if they are not already in there. Keep the entire git - repository (with history) small - under 20MiB, unless you specifically - must commit larger files (e.g. test fixture example media files). Only - OUR source code and immediately supporting files (such as test examples) - goes into the repo/history. +- Avoid vendoring deps unless specifically instructed to. NEVER commit the + vendor directory, NEVER commit compiled binaries. If these directories or + files exist, add them to .gitignore (and commit the .gitignore) if they are + not already in there. Keep the entire git repository (with history) small - + under 20MiB, unless you specifically must commit larger files (e.g. test + fixture example media files). Only OUR source code and immediately supporting + files (such as test examples) goes into the repo/history. diff --git a/README.md b/README.md index ce00d83..95d9079 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ # pixa -pixa is a GPL-3.0-licensed Go web server by -[@sneak](https://sneak.berlin) that proxies images from upstream -sources, optionally resizing or transforming them, and serves the -results. Both source and transformed images are cached to disk so that -subsequent requests are served without origin fetches or additional +pixa is a GPL-3.0-licensed Go web server by [@sneak](https://sneak.berlin) that +proxies images from upstream sources, optionally resizing or transforming them, +and serves the results. Both source and transformed images are cached to disk so +that subsequent requests are served without origin fetches or additional processing. ## Getting Started @@ -27,12 +26,11 @@ make docker docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest ``` -A container takes its settings from environment variables (see -Configuration below for the list). Only `PIXA_SIGNING_KEY` is required; if -it is unset the container exits at startup naming the variable. Everything -else has a built-in default. A config file mounted at `/etc/pixa/config.yml` -is optional: it is read when present, and an environment variable wins over -the same setting in it. +A container takes its settings from environment variables (see Configuration +below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the +container exits at startup naming the variable. Everything else has a built-in +default. A config file mounted at `/etc/pixa/config.yml` is optional: it is read +when present, and an environment variable wins over the same setting in it. ## Deployment @@ -94,41 +92,40 @@ Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as it uses libvips through CGO; building it also needs their development files, `pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these, as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which -`script/setup` and `script/cibuild` run, installs only git, make and Go: the -checks compile pixa in Docker, so the host needs none of the C libraries. Docker -itself must already be installed. +`script/setup` and `script/cibuild` run, installs git, make and Go, and Node, +Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but +none of the C libraries: the checks compile pixa in Docker. Docker itself must +already be installed. ## Running under upaas What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: - **Port:** pixa listens on container port `8080`. -- **Volume:** container path `/var/lib/pixa`, where pixa keeps its - database and cache. Creating the host directory when it is missing is - upaas's job, tracked in https://git.eeqj.de/sneak/upaas/issues/235. +- **Volume:** container path `/var/lib/pixa`, where pixa keeps its database and + cache. Creating the host directory when it is missing is upaas's job, tracked + in https://git.eeqj.de/sneak/upaas/issues/235. - **Environment variables:** - - `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs - and login, 32+ characters, for example from - `openssl rand -base64 32` + - `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and + login, 32+ characters, for example from `openssl rand -base64 32` - `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature, comma-separated - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it; default 75% of (free space + what the cache holds) - the rest are in the table under Configuration below - **Health check:** the image's `HEALTHCHECK` requests - `/.well-known/healthcheck.json`. upaas reads the container's health 60 - seconds after a deploy and marks the deploy failed unless it is - `healthy`. The probe uses the port from `PORT` (default `8080`), so a - port changed only in a mounted config file is not seen by it: change - the port with `PORT`. + `/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds + after a deploy and marks the deploy failed unless it is `healthy`. The probe + uses the port from `PORT` (default `8080`), so a port changed only in a + mounted config file is not seen by it: change the port with `PORT`. ## Rationale -Image-heavy web applications need a fast, caching reverse proxy that -can resize and transcode images on the fly. pixa fills that role as a -single, self-contained binary with no external runtime dependencies -beyond libvips. It supports HMAC-SHA256 signed URLs with expiration to -prevent abuse, and allowlisted source hosts for open access. +Image-heavy web applications need a fast, caching reverse proxy that can resize +and transcode images on the fly. pixa fills that role as a single, +self-contained binary with no external runtime dependencies beyond libvips. It +supports HMAC-SHA256 signed URLs with expiration to prevent abuse, and +allowlisted source hosts for open access. ## Design @@ -137,8 +134,8 @@ prevent abuse, and allowlisted source hosts for open access. - **Source content**: `/cache/sources///` - **Source metadata**: - `/cache/metadata//.json` - (host, path and query, content hash, upstream status and headers, fetch time) + `/cache/metadata//.json` (host, + path and query, content hash, upstream status and headers, fetch time) - **Database**: `/state.sqlite3` (SQLite) - **Transformed images**: `/cache/variants///`, @@ -147,8 +144,8 @@ prevent abuse, and allowlisted source hosts for open access. `` and `` are the first and second pairs of characters of the file's name. -Multiple source paths may reference the same content blob; the -database tracks references rather than using filesystem refcounting. +Multiple source paths may reference the same content blob; the database tracks +references rather than using filesystem refcounting. pixa's target is 1-5k r/s, which has not been measured at that rate (see Load Test). Toward it, pixa keeps in memory the content types of the 10,000 @@ -163,8 +160,8 @@ answers any method as it answers `GET`. A browser's CORS preflight request (`OPTIONS` with `Origin` and `Access-Control-Request-Method` headers) to any path under `/v1/` answers 200, in maintenance mode too. -- `GET /` — the login page, or the URL generator page with a login session - (see Encrypted URLs). Needs: nothing. Answers: 200. +- `GET /` — the login page, or the URL generator page with a login session (see + Encrypted URLs). Needs: nothing. Answers: 200. - `POST /` — log in with the signing key typed into the login page. Needs: the login page's form (below). Answers: 303 to `/` with a login session cookie that lasts 30 days for the right key; 200 with the login page and an error for @@ -197,11 +194,10 @@ path under `/v1/` answers 200, in maintenance mode too. - `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: nothing. Answers: 200. - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, - `uptime_seconds`, `uptime_human`, `version`, `appname` and - `maintenance_mode`. Needs: nothing. Answers: 200, always. -- `GET /static/` — the stylesheet and script the login and generator - pages load. Needs: nothing. Answers: 200, or 404 for a file that does not - exist. + `uptime_seconds`, `uptime_human`, `version`, `appname` and `maintenance_mode`. + Needs: nothing. Answers: 200, always. +- `GET /static/` — the stylesheet and script the login and generator pages + load. Needs: nothing. Answers: 200, or 404 for a file that does not exist. - `GET /metrics` — Prometheus metrics (see Architecture). Needs: HTTP basic authentication with `metrics.username` and `metrics.password`. Answers: 200; 401 without them; 404 when they are not set, as the route then does not exist. @@ -225,9 +221,9 @@ HTTP is for development on the browser's own machine: the login session cookie is always marked `Secure`, and over plain HTTP a browser keeps such a cookie only for its own machine (`localhost`), if at all. A form is also refused with 403 when the page's host is not the `Host` header pixa receives, so a reverse -proxy in front of pixa must pass that header on unchanged. A form body over -1 MiB is refused with 413. The image routes answer the errors listed for them -with JSON holding `error`, `status` and `timestamp`. +proxy in front of pixa must pass that header on unchanged. A form body over 1 +MiB is refused with 413. The image routes answer the errors listed for them with +JSON holding `error`, `status` and `timestamp`. An image URL has this form: @@ -239,8 +235,8 @@ Images are only fetched from origins using TLS with valid certificates, unless `allow_http` is set: then pixa fetches every image over plain HTTP, which is for testing only. -A request whose query string cannot be decoded, or gives any parameter more -than once, is refused with 400. +A request whose query string cannot be decoded, or gives any parameter more than +once, is refused with 400. - ``: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, `avif`, `gif` @@ -248,39 +244,37 @@ than once, is refused with 400. - `sig` and `exp`: the signature and its expiry, needed unless the host is allowlisted (see Signature Specification) - `q` and `fit`: the output quality and how the image is fitted to ``, - both optional (values under Signature Specification). Both are part of what - is cached, so each value of either is a separate cached image. + both optional (values under Signature Specification). Both are part of what is + cached, so each value of either is a separate cached image. An image is served with `Cache-Control: public, max-age=, immutable`. -When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), -`max-age` is the whole seconds left until then, at most one year, so no browser -or proxy cache keeps the image after pixa would refuse the URL. A URL with no -expiry gets one year. `immutable` only stops a client revalidating while its -copy is fresh. +When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), `max-age` +is the whole seconds left until then, at most one year, so no browser or proxy +cache keeps the image after pixa would refuse the URL. A URL with no expiry gets +one year. `immutable` only stops a client revalidating while its copy is fresh. -When several requests for the same image, size, format, quality and fit miss -the cache at once, they share one upstream fetch (or one read of the cached -source) and one transcode: the first request does the work, and the others wait -for its image or its error, holding no upstream connection or processing slot -of their own. A waiting request stops waiting when its own client goes away. -The work goes on for the others even if the first request's client goes away, -until that request's `downstream_timeout` ends. The shared fetch sends the first -request's ID upstream, and the lines logged for the fetch and the transcode -carry that ID. +When several requests for the same image, size, format, quality and fit miss the +cache at once, they share one upstream fetch (or one read of the cached source) +and one transcode: the first request does the work, and the others wait for its +image or its error, holding no upstream connection or processing slot of their +own. A waiting request stops waiting when its own client goes away. The work +goes on for the others even if the first request's client goes away, until that +request's `downstream_timeout` ends. The shared fetch sends the first request's +ID upstream, and the lines logged for the fetch and the transcode carry that ID. The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is refused with 429 and a `Retry-After` header. Behind a reverse proxy the client address comes from `X-Forwarded-For` only when the address pixa sees for requests that come through the proxy is in `trusted_proxies`; otherwise all -users behind the proxy are counted as one client. That address is not always -the proxy's own: a proxy on the Docker host that connects to pixa over -`127.0.0.1` is seen as the gateway of the container's Docker network, such as -`172.17.0.1` on the default bridge, and one that connects through another of the -host's addresses is seen with that address. To be sure, read it as `remoteIP` in -pixa's request log while it is not in `trusted_proxies` (see `trusted_proxies` -under Configuration). With the default `trusted_proxies` (the RFC 1918 ranges), -a client with a private address can choose the address it is counted by through +users behind the proxy are counted as one client. That address is not always the +proxy's own: a proxy on the Docker host that connects to pixa over `127.0.0.1` +is seen as the gateway of the container's Docker network, such as `172.17.0.1` +on the default bridge, and one that connects through another of the host's +addresses is seen with that address. To be sure, read it as `remoteIP` in pixa's +request log while it is not in `trusted_proxies` (see `trusted_proxies` under +Configuration). With the default `trusted_proxies` (the RFC 1918 ranges), a +client with a private address can choose the address it is counted by through its own `X-Forwarded-For`, whether it connects directly or through the proxy, because its own address is trusted too. Setting `trusted_proxies` to only the address pixa sees for requests that come through the proxy closes this. @@ -305,18 +299,18 @@ nor change what it asks for. with `http` instead while `debug` is on. The name after the token is ignored and only gives the URL a file extension, `jpg` for `orig`. -The token holds the source's host, path and query and the size, format, -quality, fit and expiry, encrypted with a key derived from `signing_key`. The -source URL's scheme is not kept: the image is fetched like any other (see -Routes), and the blocked networks still apply. +The token holds the source's host, path and query and the size, format, quality, +fit and expiry, encrypted with a key derived from `signing_key`. The source +URL's scheme is not kept: the image is fetched like any other (see Routes), and +the blocked networks still apply. -How long the URL lasts is chosen on the page, from 1 minute to 1 year, or -never. The expiry is fixed in the token when the URL is made and cannot be -changed or revoked afterwards. Until then the image is served with a `max-age` -that ends at the expiry (see Routes); after it the URL answers 410 -`URL has expired`. A URL made to last forever stops working only when -`signing_key` changes: changing it makes every encrypted URL already handed out -answer 400, and ends every login session. +How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never. +The expiry is fixed in the token when the URL is made and cannot be changed or +revoked afterwards. Until then the image is served with a `max-age` that ends at +the expiry (see Routes); after it the URL answers 410 `URL has expired`. A URL +made to last forever stops working only when `signing_key` changes: changing it +makes every encrypted URL already handed out answer 400, and ends every login +session. ### Image Metadata @@ -335,16 +329,15 @@ turned off. ### Source Hosts -Source hosts may be allowlisted in the configuration. Non-allowlisted -hosts require an HMAC-SHA256 signature. +Source hosts may be allowlisted in the configuration. Non-allowlisted hosts +require an HMAC-SHA256 signature. #### Signature Specification -Signatures use HMAC-SHA256 and include an expiration timestamp to -prevent replay attacks. Signatures are **exact match only**: every -component (host, path, query, dimensions, format, expiration, quality, -fit) must match exactly what was signed. No suffix matching, wildcard -matching, or partial matching is supported. +Signatures use HMAC-SHA256 and include an expiration timestamp to prevent replay +attacks. Signatures are **exact match only**: every component (host, path, +query, dimensions, format, expiration, quality, fit) must match exactly what was +signed. No suffix matching, wildcard matching, or partial matching is supported. **Signed data format** (colon-separated): @@ -361,24 +354,24 @@ Where: - `height` — requested height in pixels, `0` for original - `format` — output format, one of those listed under Routes, with `original` signed as `orig` and `jpg` as `jpeg` -- `expiration` — the URL's `exp` query parameter, the Unix timestamp when - the signature expires; a request whose `exp` is not a whole number, an - empty `exp=` included, is refused with 400 -- `quality` — the URL's `q` query parameter, a whole number from 1 to 100, - or `85` when the URL has no `q`; a request whose `q` is anything else is - refused with 400 +- `expiration` — the URL's `exp` query parameter, the Unix timestamp when the + signature expires; a request whose `exp` is not a whole number, an empty + `exp=` included, is refused with 400 +- `quality` — the URL's `q` query parameter, a whole number from 1 to 100, or + `85` when the URL has no `q`; a request whose `q` is anything else is refused + with 400 - `fit` — the URL's `fit` query parameter (cover, contain, fill, inside, outside), or `cover` when the URL has no `fit`; a request whose `fit` is anything else, an empty `fit=` included, is refused with 400 -The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet -of RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa +The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of +RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa compares it exactly, so a signature encoded without padding, as Node's `base64url` and Go's `base64.RawURLEncoding` do, is refused with 401. **Example:** with the signing key `example-signing-key-for-documentation`, -resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP with -expiration 1704067200, default quality and fit: +resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP with expiration +1704067200, default quality and fit: 1. Build input: `cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover` @@ -406,29 +399,29 @@ or a `*.` wildcard, aborts startup. ### Configuration -Every setting can be given as an environment variable, in a YAML config -file (`--config`), or both. A variable present in the environment wins over -the file, even when it is empty, and the file wins over the built-in -default. The one exception is a variable named in the file's `env:` section: -it is set while the file loads, so it overrides both the environment the -process was started with and the file's own key. A variable's value is -parsed as the same text in the file would be. The three lists take -comma-separated entries, with the spaces around each trimmed; an empty -variable is an empty list. A value that does not parse or is invalid aborts -startup, naming the variable. A variable whose name starts with `PIXA_` but -is not in the table below, such as a misspelled one or `PIXA_PORT`, aborts -startup naming it, as an unknown config key does. The one other accepted -name is `PIXA_CONFIG_PATH`, the config file's path (like `--config`). The -variables set by the file's `env:` section are checked the same way. +Every setting can be given as an environment variable, in a YAML config file +(`--config`), or both. A variable present in the environment wins over the file, +even when it is empty, and the file wins over the built-in default. The one +exception is a variable named in the file's `env:` section: it is set while the +file loads, so it overrides both the environment the process was started with +and the file's own key. A variable's value is parsed as the same text in the +file would be. The three lists take comma-separated entries, with the spaces +around each trimmed; an empty variable is an empty list. A value that does not +parse or is invalid aborts startup, naming the variable. A variable whose name +starts with `PIXA_` but is not in the table below, such as a misspelled one or +`PIXA_PORT`, aborts startup naming it, as an unknown config key does. The one +other accepted name is `PIXA_CONFIG_PATH`, the config file's path (like +`--config`). The variables set by the file's `env:` section are checked the same +way. pixa reads at most one config file: the one given with `--config` (or `-c`), otherwise the one `PIXA_CONFIG_PATH` names, otherwise the first of these that pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`, -`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml` -and `config.yaml` in the working directory. A named file that does not exist, -cannot be read or does not parse aborts startup. Of the files pixa looks for on -its own, only one that does not exist is passed over, without a message. One -that pixa cannot read or parse aborts startup, naming the file. So does one in a +`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml` and +`config.yaml` in the working directory. A named file that does not exist, cannot +be read or does not parse aborts startup. Of the files pixa looks for on its +own, only one that does not exist is passed over, without a message. One that +pixa cannot read or parse aborts startup, naming the file. So does one in a directory pixa may not enter, whether or not it is there, since pixa cannot tell. With no file, pixa uses the environment and the defaults. @@ -463,11 +456,11 @@ Key settings in more detail: of the image routes, `/v1/image/` and `/v1/e/`, sent as the CORS `Access-Control-Allow-Origin` header; no other route sends it. `*`, the default, is any site; otherwise one `http` or `https` origin such as - `https://example.com`, whose host is a lowercase host name (letters, - digits, hyphens and dots, with a letter in its last part) or an IP address - (IPv6 in brackets, in its shortest form), with an optional port 1-65535 - that has no leading zero and is not the scheme's default. Any other value, - including another scheme such as a browser extension's, aborts startup + `https://example.com`, whose host is a lowercase host name (letters, digits, + hyphens and dots, with a letter in its last part) or an IP address (IPv6 in + brackets, in its shortest form), with an optional port 1-65535 that has no + leading zero and is not the scheme's default. Any other value, including + another scheme such as a browser extension's, aborts startup - `allowlist_hosts` — list of allowed upstream hosts - `referer_blocklist` — list of hosts whose pages may not show pixa's images, to stop other sites hotlinking them. Entries are written and matched as for @@ -476,40 +469,37 @@ Key settings in more detail: `/v1/e/` whose `Referer` header names a listed host is refused with 403 before its signature or token is checked and before the cache or the upstream host is used, so it fetches nothing, and it is refused even when the image is cached. - A request with no `Referer`, or one that does not parse as a URL - with a host, is served, as many clients send none. So this is easily got - around: a site whose pages send no `Referer` (for example with + A request with no `Referer`, or one that does not parse as a URL with a host, + is served, as many clients send none. So this is easily got around: a site + whose pages send no `Referer` (for example with `Referrer-Policy: no-referrer`) is not stopped. It does not apply to the login and generator pages. Default: empty -- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, - added to the always-enforced built-in ranges (loopback, private, - link-local, CGNAT, benchmark, NAT64, and the like); an invalid CIDR - aborts startup -- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front - of pixa. `X-Forwarded-For` is believed only when the direct peer falls - inside one of these ranges; the logged and login-recorded client - address is then the rightmost forwarded entry that is not itself a - trusted proxy. Otherwise the direct peer address is used and the header - is ignored, so a client connecting directly from an address outside - these ranges cannot spoof its address. - An omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`, - `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a - proxy on a private network; an explicitly empty list (`[]`) trusts no - one, and an explicit list replaces the default. An invalid CIDR aborts - startup. Set this to the address pixa sees for requests that come through - your proxy, such as `172.17.0.1/32`, when the defaults do not cover it, or - to trust nothing else (see the login limit under Routes). For a proxy on - the Docker host that connects to pixa over `127.0.0.1`, that address is the - gateway of the container's Docker network (`172.17.0.1` on the default - bridge), not the proxy's own address; a proxy that connects through another of - the host's addresses is seen with that address. To be sure which address it - is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request - through the proxy, and read `remoteIP` in pixa's request log line for it -- `upstream_fetch_timeout` — time allowed for one fetch from an upstream - host, as a duration such as `30s` (the default) or `2m` -- `upstream_max_response_size` — largest upstream response accepted, in - bytes; default `52428800` (50 MiB). It also limits the image data pixa - decodes +- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added + to the always-enforced built-in ranges (loopback, private, link-local, CGNAT, + benchmark, NAT64, and the like); an invalid CIDR aborts startup +- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of + pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one + of these ranges; the logged and login-recorded client address is then the + rightmost forwarded entry that is not itself a trusted proxy. Otherwise the + direct peer address is used and the header is ignored, so a client connecting + directly from an address outside these ranges cannot spoof its address. An + omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`, + `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a + private network; an explicitly empty list (`[]`) trusts no one, and an + explicit list replaces the default. An invalid CIDR aborts startup. Set this + to the address pixa sees for requests that come through your proxy, such as + `172.17.0.1/32`, when the defaults do not cover it, or to trust nothing else + (see the login limit under Routes). For a proxy on the Docker host that + connects to pixa over `127.0.0.1`, that address is the gateway of the + container's Docker network (`172.17.0.1` on the default bridge), not the + proxy's own address; a proxy that connects through another of the host's + addresses is seen with that address. To be sure which address it is, set this + to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request through the + proxy, and read `remoteIP` in pixa's request log line for it +- `upstream_fetch_timeout` — time allowed for one fetch from an upstream host, + as a duration such as `30s` (the default) or `2m` +- `upstream_max_response_size` — largest upstream response accepted, in bytes; + default `52428800` (50 MiB). It also limits the image data pixa decodes - `downstream_timeout` — time allowed for answering one client request, as a duration; default `60s`. The upstream fetch counts toward it, and so do the waits for an upstream connection and for a processing slot (up to 10 seconds @@ -521,11 +511,11 @@ Key settings in more detail: so a write that finds another in progress waits up to five seconds for it instead of failing. WAL mode comes only from the URL: keep `_pragma=journal_mode(WAL)` in one you set -- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the - disk cache entirely; omitted defaults to 75% of the sum of the free space on - the filesystem containing `/cache/` and the bytes of source and - transformed images the cache already holds, worked out at startup (minimum - 500 MiB) +- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk + cache entirely; omitted defaults to 75% of the sum of the free space on the + filesystem containing `/cache/` and the bytes of source and + transformed images the cache already holds, worked out at startup (minimum 500 + MiB) - `upstream_connections` — the most connections to upstream hosts at once, all hosts together, on top of `upstream_connections_per_host`; default `64`. A fetch holds its connection until its image has been processed. A fetch that @@ -540,10 +530,10 @@ Key settings in more detail: - `maintenance_mode` — while `true`, the image routes (`/v1/image/` and `/v1/e/`) answer every request for an image with 503, a `Retry-After` header and a JSON error body. The health check (`/.well-known/healthcheck.json`) - still answers 200 and reports `"maintenance_mode": true`. It stays 200 - because the image's Docker `HEALTHCHECK` requests it: a 503 there would make - the container unhealthy, and upaas marks a deploy failed when its container - is unhealthy. The login and URL generator pages and `/metrics` keep working + still answers 200 and reports `"maintenance_mode": true`. It stays 200 because + the image's Docker `HEALTHCHECK` requests it: a 503 there would make the + container unhealthy, and upaas marks a deploy failed when its container is + unhealthy. The login and URL generator pages and `/metrics` keep working See `configs/config.example.yml` for all options with defaults. @@ -565,21 +555,23 @@ See `configs/config.example.yml` for all options with defaults. This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) standard: normalized scripts in `script/` are the entrypoints for the -development workflow, and the Makefile targets are thin shims that call -them. We provide: +development workflow, and the Makefile targets are thin shims that call them. We +provide: -- `script/bootstrap` — install git, make and Go and download the Go modules - (idempotent); with `--cgo`, also the C compiler and the libvips and libheif - libraries that compiling pixa needs -- `script/setup` — make a fresh clone ready for development - (bootstrap, then install-precommit) +- `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and + download the Go modules (idempotent); with `--cgo`, the C compiler and the + libvips and libheif libraries that compiling pixa needs instead of Node, Yarn + and prettier +- `script/setup` — make a fresh clone ready for development (bootstrap, then + install-precommit) - `script/projectname` — output the project name ("pixa") - `script/test` — run the test suite: build the `test` phase of the `Dockerfile`, tagged `pixa-test` - `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`, tagged `pixa-lint`; the linter never runs on the host -- `script/fmt` — format all code (writes) -- `script/fmt-check` — check formatting (read-only), on the host +- `script/fmt` — format the Go code with gofmt and the markdown with prettier + (writes) +- `script/fmt-check` — check the same formatting (read-only), on the host - `script/check` — run test, lint, and fmt-check - `script/docker` — build the Docker image tagged via `script/projectname`, with the version from `git describe`; the image's build stage depends on the `lint` @@ -591,14 +583,18 @@ them. We provide: then `script/check`, then build the image as `script/docker` does - `script/precommit` — pre-commit checks (`go mod tidy` guard, then `script/check`) -- `script/install-precommit` — install the git pre-commit hook that - runs `script/precommit` +- `script/install-precommit` — install the git pre-commit hook that runs + `script/precommit` Every `docker build` in these scripts passes `--no-cache`, so the lint and test phases run on every build instead of coming from the build cache. `script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`, `script/setup` and `script/install-precommit` are the standard copies from -`sneak/prompts`, kept identical to them. +`sneak/prompts`, kept identical to them. `script/fmt` and `script/fmt-check` are +the standard copies with pixa's `gofmt` step kept before prettier. prettier +formats the markdown only: not the HTML templates, as it cannot parse a Go +template action inside a tag, and not `REPO_POLICIES.md` (see +`.prettierignore`), a copy of the one in `sneak/prompts`. ## Load Test diff --git a/TODO.md b/TODO.md index ffff926..cabe9e2 100644 --- a/TODO.md +++ b/TODO.md @@ -1,28 +1,27 @@ # Workflow -* branch per issue from `next` -* do the work in Next Step -* move Next Step to the top of Completed Steps -* `TODO.md` merges with git's union merge (`.gitattributes`), which never +- branch per issue from `next` +- do the work in Next Step +- move Next Step to the top of Completed Steps +- `TODO.md` merges with git's union merge (`.gitattributes`), which never reports a conflict: read the merged entries after every merge or rebase -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* open a PR based on `next` -* an independent reviewer who did not write the change gates it -* the manager squash-merges the PR into `next` once review passes -* `next` stays green and mergeable to `main` at any time; only the owner - merges `next` into `main`, via the single milestone PR -* push +- move the top item of Future Steps into Next Step +- commit (`TODO.md` changes in the same commit as the work) +- open a PR based on `next` +- an independent reviewer who did not write the change gates it +- the manager squash-merges the PR into `next` once review passes +- `next` stays green and mergeable to `main` at any time; only the owner merges + `next` into `main`, via the single milestone PR +- push # Status -pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work -lands on `next`, and `main` receives only the milestone PR that the -owner merges. `next` is at the canonical `golangci-lint` v2.12.2 config -and is green. Recent work extracted the internal/magic, -internal/allowlist, internal/httpfetcher, and internal/signature -packages. The gosec findings from the 2026-07-06 survey are resolved. -The disk cache is now size-bounded with LRU eviction +pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work lands on +`next`, and `main` receives only the milestone PR that the owner merges. `next` +is at the canonical `golangci-lint` v2.12.2 config and is green. Recent work +extracted the internal/magic, internal/allowlist, internal/httpfetcher, and +internal/signature packages. The gosec findings from the 2026-07-06 survey are +resolved. The disk cache is now size-bounded with LRU eviction (`cache_max_bytes`), closing the unbounded disk growth DoS vector. # Next Step @@ -31,6 +30,16 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-05 the markdown is formatted with prettier (closes #100): `script/fmt` + and `script/fmt-check` run prettier 3.8.1, pinned in `package.json` and + `yarn.lock`, on `**/*.md` after `gofmt`, with four-space tabs and + `proseWrap: always` as `.prettierrc` says; `.prettierignore` keeps it off + `REPO_POLICIES.md`, the copy from `sneak/prompts`, and `vendor/`. Plain + `script/bootstrap` installs Node and Yarn as the one in `sneak/prompts` does + and then prettier; `script/bootstrap --cgo` does not, as the `Dockerfile` + stages that run it format nothing. The HTML templates stay unformatted: + prettier cannot parse a Go template action inside a tag. The markdown was + reflowed in a commit of its own. - 2026-10-05 lint and tests run as the `lint` and `test` phases of the `Dockerfile`, built with `--no-cache` (closes #202): `script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`, @@ -103,13 +112,12 @@ P2: security: per-IP rate limiting on the image routes (the workflow's `script/docker-smoke` step), https://git.eeqj.de/sneak/pixa/issues/204 (`.claude/` in `.gitignore`), https://git.eeqj.de/sneak/pixa/issues/205 (`.dockerignore` patterns at every - depth), https://git.eeqj.de/sneak/pixa/issues/206 (a thin - `cmd/pixad/main.go`) and https://git.eeqj.de/sneak/pixa/issues/208 - (`fetch-depth: 0` on the CI checkout, so the build sees the tags). Its rule - that no build stage runs `git describe` is not followed: pixa takes the - version from the `.git` in the build context, per - https://git.eeqj.de/sneak/pixa/issues/166, as the copy on `sneak/prompts` - `next` already says. + depth), https://git.eeqj.de/sneak/pixa/issues/206 (a thin `cmd/pixad/main.go`) + and https://git.eeqj.de/sneak/pixa/issues/208 (`fetch-depth: 0` on the CI + checkout, so the build sees the tags). Its rule that no build stage runs + `git describe` is not followed: pixa takes the version from the `.git` in the + build context, per https://git.eeqj.de/sneak/pixa/issues/166, as the copy on + `sneak/prompts` `next` already says. - 2026-10-04 an integration test of the image proxy flow (closes #80): `TestImageProxyFlow` in `internal/server` starts the database, handlers and middleware from the constructors `pixad` uses, with a fresh state directory, @@ -125,13 +133,12 @@ P2: security: per-IP rate limiting on the image routes `httpfetcher.Config.DialContext` connects in place of the dialer that refuses internal addresses, the URL and redirect checks still running, and `handlers.Params.Fetcher` replaces the fetcher the handlers build. -- 2026-10-04 a URL made on the generator page with a `ttl` is tested to - expire (closes #199): a new test in `internal/handlers` makes a URL on the - generator page with a `ttl` of one second, checks that `/v1/e/` serves it at - once, waits two seconds and checks that it then answers 410. The test waits - for real, as pixa reads the clock directly when it makes and checks a URL; it - waits two seconds because the time a URL expires is kept in whole seconds. - Test only. +- 2026-10-04 a URL made on the generator page with a `ttl` is tested to expire + (closes #199): a new test in `internal/handlers` makes a URL on the generator + page with a `ttl` of one second, checks that `/v1/e/` serves it at once, waits + two seconds and checks that it then answers 410. The test waits for real, as + pixa reads the clock directly when it makes and checks a URL; it waits two + seconds because the time a URL expires is kept in whole seconds. Test only. - 2026-10-04 referer blocklist (closes #90): `referer_blocklist` (`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for `allowlist_hosts` with the same matcher; an entry of either list that is @@ -149,30 +156,29 @@ P2: security: per-IP rate limiting on the image routes `README.md`, the comments in `internal/config/config.go` and the startup error for the placeholder signing key name the new path; `scripts/manual-test.sh` and its directory are deleted, as the handler tests in `internal/handlers` - cover every check it made except two: fetching a real image from the - internet, and a URL made on the generator page with a `ttl` answering 410 once - the `ttl` has passed (https://git.eeqj.de/sneak/pixa/issues/199); - `CONVENTIONS.md` is deleted, as `REPO_POLICIES.md` links the canonical Go HTTP - server conventions. + cover every check it made except two: fetching a real image from the internet, + and a URL made on the generator page with a `ttl` answering 410 once the `ttl` + has passed (https://git.eeqj.de/sneak/pixa/issues/199); `CONVENTIONS.md` is + deleted, as `REPO_POLICIES.md` links the canonical Go HTTP server conventions. - 2026-10-04 SQLite writes no longer fail with "database is locked" (closes #198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write that finds another in progress on another connection waits up to five seconds for it, and the default `db_url` turns on WAL mode with `_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a parameter the driver reads, so the database was never in WAL mode. -- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup` - only passes through a periodic pass (closes #189): it slept for three - eviction intervals before writing its file, and a startup pass still running - then could adopt the file itself. It now holds the test database's only - connection until the startup pass waits for it after walking the empty - variant directory, writes the file and lets the connection go, as +- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup` only + passes through a periodic pass (closes #189): it slept for three eviction + intervals before writing its file, and a startup pass still running then could + adopt the file itself. It now holds the test database's only connection until + the startup pass waits for it after walking the empty variant directory, + writes the file and lets the connection go, as `TestEvictionRunsOnPeriodicSchedule` does, so only a periodic reconciliation pass can adopt the file. Test only. - 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have - handler tests (closes #77): new tests in `internal/handlers`, with no - network, check that `GET /` without a login session shows the login form; a - wrong key shows it again with an error and sets no session cookie; the right - key answers 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and + handler tests (closes #77): new tests in `internal/handlers`, with no network, + check that `GET /` without a login session shows the login form; a wrong key + shows it again with an error and sets no session cookie; the right key answers + 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and `SameSite=Strict`, with which `GET /` shows the generator page; `GET /logout` answers 303 to `/` with an empty session cookie sent with `Max-Age=0`; `POST /generate` without a login session answers 303 to `/`; `/v1/e/` serves @@ -182,11 +188,11 @@ P2: security: per-IP rate limiting on the image routes - 2026-10-04 `TODO.md` merges with git's union merge (closes #190): a root `.gitattributes`, copied from `sneak/prompts`, marks it `merge=union`, so two branches that each add an entry at the top of Completed Steps merge without a - conflict and keep both entries. Git now never reports a conflict in - `TODO.md`: a real one keeps both versions of the lines, and two entries that - share an identical line can end up one inside the other, which a rebase can - do to an entry already on `next`. The Workflow above says to read the merged - entries after every merge or rebase. + conflict and keep both entries. Git now never reports a conflict in `TODO.md`: + a real one keeps both versions of the lines, and two entries that share an + identical line can end up one inside the other, which a rebase can do to an + entry already on `next`. The Workflow above says to read the merged entries + after every merge or rebase. - 2026-10-04 the default `cache_max_bytes` no longer shrinks as the cache fills (closes #184): for an omitted key, the cache works out the limit when it opens, after the database is open, as 75% of the sum of the free space on the @@ -210,10 +216,10 @@ P2: security: per-IP rate limiting on the image routes that pixa may not enter, aborts startup naming the file, as a file that does not parse already did. - 2026-10-04 `.golangci.yml` re-vendored from the canonical copy (closes #57): - the deprecated `gomodguard` is switched off, so lint runs print no - deprecation warning; its successor `gomodguard_v2` runs with the shared - module block list, and `depguard` keeps `net/http/httptest` out of files that - are not tests. The tree needed no code changes. + the deprecated `gomodguard` is switched off, so lint runs print no deprecation + warning; its successor `gomodguard_v2` runs with the shared module block list, + and `depguard` keeps `net/http/httptest` out of files that are not tests. The + tree needed no code changes. - 2026-10-04 the Content-Security-Policy allows no inline script or style (closes #125): `script-src` and `style-src` are `'self'` only. The generator page's two inline `onclick` handlers moved into @@ -232,30 +238,30 @@ P2: security: per-IP rate limiting on the image routes counts, the health check for a load balancer, what a stop does and its exit codes, and what running outside Docker needs; `configs/Caddyfile` is the example, checked with `caddy validate`. -- 2026-10-04 the metrics basic auth, CORS preflight, request logging and - metrics recording have tests (closes #79): `MetricsAuth` on its own answers - 401 with a challenge without credentials or with a wrong username or password - and lets the configured ones through; a preflight request gets `*` for any - origin when `access_control_allow_origin` is `*` and no - `Access-Control-Allow-Origin` from another origin than the configured one; a - `POST /` carrying the signing key leaves no trace of it in the request log - line, and the login handler's own log lines leave out the submitted key; the - metrics middleware on its own records a request it served, and the router - records nothing while no metrics username is set. Not tested: that the router - puts the basic auth in front of `/metrics` and records requests when a - metrics username is set. Only one test per package can set up `/metrics`, and - in `internal/server` that is `TestMaintenanceModeKeepsOtherRoutes`, which - needs the owner's approval to change; #180 holds it. Tests only; the basic - auth library already compares the password in constant time. +- 2026-10-04 the metrics basic auth, CORS preflight, request logging and metrics + recording have tests (closes #79): `MetricsAuth` on its own answers 401 with a + challenge without credentials or with a wrong username or password and lets + the configured ones through; a preflight request gets `*` for any origin when + `access_control_allow_origin` is `*` and no `Access-Control-Allow-Origin` from + another origin than the configured one; a `POST /` carrying the signing key + leaves no trace of it in the request log line, and the login handler's own log + lines leave out the submitted key; the metrics middleware on its own records a + request it served, and the router records nothing while no metrics username is + set. Not tested: that the router puts the basic auth in front of `/metrics` + and records requests when a metrics username is set. Only one test per package + can set up `/metrics`, and in `internal/server` that is + `TestMaintenanceModeKeepsOtherRoutes`, which needs the owner's approval to + change; #180 holds it. Tests only; the basic auth library already compares the + password in constant time. - 2026-10-04 the image route's signature check and error answers are tested (closes #76): new tests in `internal/handlers`, with no network, check the status and JSON error body for a missing, wrong, unpadded, upper-case or - expired signature on a host not on the allowlist, or a valid one sent for - its parent domain, a sibling host, a subdomain or the host with another - domain appended (401), an unparseable path (400), `localhost` as the - upstream host (403) and an upstream error (502); that an allowlisted host is - served without a signature, another host only with a valid one; and the - answers of `/robots.txt` and the health check. No code changes. + expired signature on a host not on the allowlist, or a valid one sent for its + parent domain, a sibling host, a subdomain or the host with another domain + appended (401), an unparseable path (400), `localhost` as the upstream host + (403) and an upstream error (502); that an allowlisted host is served without + a signature, another host only with a valid one; and the answers of + `/robots.txt` and the health check. No code changes. - 2026-10-04 request IDs returned and passed on, and `/v1/e/` revalidates (closes #84): pixa's own `RequestID` middleware, in place of chi's, gives each request an ID, its own `X-Request-ID` when that is at most 64 letters, digits, @@ -272,26 +278,26 @@ P2: security: per-IP rate limiting on the image routes `Vary: Accept` is left to #88. - 2026-10-04 routes, encrypted URLs and config file documented (closes #75): "Routes" in `README.md` lists every route with its method, purpose, what it - needs and the status codes it answers with, and says `q` and `fit` are part - of what is cached; "Encrypted URLs" covers logging in, making one on the + needs and the status codes it answers with, and says `q` and `fit` are part of + what is cached; "Encrypted URLs" covers logging in, making one on the generator page, how long it lasts and the 410 once it has expired; "Configuration" gives the order in which pixa looks for its config file; `config.example.yml` lists `db_url` and `env` and gives every key's default; `scripts/manual-test.sh` is left to #97. - 2026-10-04 shutdown stops cache eviction in progress (closes #102): `StartEviction` runs the eviction goroutine with its own context, which - `StopEviction` cancels, so a pass in progress stops at its next database - call, file, row or eviction candidate instead of running to completion, and - no pass starts after it, so a stop logs at most one warning; - `StopEviction` takes a context and, when that context ends before the - goroutine exits, stops waiting and returns its error; the handlers' stop hook - passes fx's stop context, so an eviction still running when fx's stop - deadline ends fails the stop and makes the exit code 1. + `StopEviction` cancels, so a pass in progress stops at its next database call, + file, row or eviction candidate instead of running to completion, and no pass + starts after it, so a stop logs at most one warning; `StopEviction` takes a + context and, when that context ends before the goroutine exits, stops waiting + and returns its error; the handlers' stop hook passes fx's stop context, so an + eviction still running when fx's stop deadline ends fails the stop and makes + the exit code 1. - 2026-10-04 dead code in `internal/imgcache` is gone (closes #73): `Purge`, which only returned an error and which nothing called, is no longer part of - the `ImageCache` interface or `Service`; the `SignatureValidator`, - `Allowlist` and `Storage` interfaces, which nothing implemented or used, are - deleted. Nothing else changes. + the `ImageCache` interface or `Service`; the `SignatureValidator`, `Allowlist` + and `Storage` interfaces, which nothing implemented or used, are deleted. + Nothing else changes. - 2026-10-04 upstream host semaphores and variant `.meta` files no longer outlive their use (closes #87): the fetcher counts the fetches holding or waiting for a slot of each upstream host's semaphore and removes the host's @@ -303,21 +309,20 @@ P2: security: per-IP rate limiting on the image routes cache directories pixa uses (`cache/sources`, `cache/metadata`, `cache/variants`) and how files are named in each, and the comments in `001_schema.sql` name the same paths; the routes and the signature section - list the same output formats, `jpg` and `original` included; the TLS - sentence names `allow_http` as its exception; "Metrics" says only generic - HTTP and Go runtime metrics exist, measured and served only when the metrics - username and password are set. -- 2026-10-03 shutdown sets the exit code and waits for image processing - (closes #86): fx alone handles SIGINT and SIGTERM, and the server's own - signal handler is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's - code: 0 for a signal, 1 when the HTTP server cannot listen or the app fails - to start or to stop; the server's stop hook, which fx waits for, stops the - HTTP server, waits for the images still being processed, both within 5 - seconds, then flushes Sentry; images still being processed after that are - logged with their count and make the exit code 1; a Sentry DSN that cannot be - used fails startup, so the stop hooks of what had already started run, - instead of exiting the process from a goroutine; the eviction loop is left to - #102. + list the same output formats, `jpg` and `original` included; the TLS sentence + names `allow_http` as its exception; "Metrics" says only generic HTTP and Go + runtime metrics exist, measured and served only when the metrics username and + password are set. +- 2026-10-03 shutdown sets the exit code and waits for image processing (closes + #86): fx alone handles SIGINT and SIGTERM, and the server's own signal handler + is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's code: 0 for a + signal, 1 when the HTTP server cannot listen or the app fails to start or to + stop; the server's stop hook, which fx waits for, stops the HTTP server, waits + for the images still being processed, both within 5 seconds, then flushes + Sentry; images still being processed after that are logged with their count + and make the exit code 1; a Sentry DSN that cannot be used fails startup, so + the stop hooks of what had already started run, instead of exiting the process + from a goroutine; the eviction loop is left to #102. - 2026-10-03 every `script/cibuild` and `script/docker` run executes the checks (closes #101): the `Dockerfile` declares `CHECK_EPOCH` above `make fmt-check` and `make lint` in the lint stage and above `make test` in the build stage, @@ -332,11 +337,11 @@ P2: security: per-IP rate limiting on the image routes upstream fetch or cached source read and one transcode through `golang.org/x/sync/singleflight`; the first request's processing ignores its cancellation but keeps its deadline, and the others wait for its image or - error holding no upstream connection or processing slot, and stop waiting - when their own context ends; the request doing the processing waits for it - even then, up to its deadline; a request whose context has already ended - starts nothing; each request counts one miss, and the processing counts its - fetch and transcode once; a panic while processing is reported to Sentry when + error holding no upstream connection or processing slot, and stop waiting when + their own context ends; the request doing the processing waits for it even + then, up to its deadline; a request whose context has already ended starts + nothing; each request counts one miss, and the processing counts its fetch and + transcode once; a panic while processing is reported to Sentry when `sentry_dsn` is set and becomes an error for every waiting request instead of stopping pixad; documented in `README.md`. - 2026-09-29 only the image routes send CORS headers (closes #98): the CORS @@ -345,20 +350,20 @@ P2: security: per-IP rate limiting on the image routes still answers a preflight `OPTIONS` request; the login and URL generator pages, `/metrics` and the other routes send no `Access-Control-Allow-Origin`; documented in `README.md` and `config.example.yml`. -- 2026-10-02 a plain `docker build .` stamps the tag or short commit, not - `dev` (closes #166): `.dockerignore` lets `.git` into the build context, - without `.git/config`; with no `VERSION` build argument the `Dockerfile` - takes the version from `git describe --tags --always`, and fails the build if - the context carries `.git` and no version comes out; `ARG VERSION` has no - default; pixad logs its version, with its name and architecture, as its first - log line at startup. -- 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes - #159): `deploy/docker-entrypoint.sh` creates the directory if it is missing, - gives the directory and everything in it to `pixad` when the directory or one - of its top-level entries belongs to another user or group, sets its mode to - `750`, then runs the server as `pixad`; data left by an earlier run under - another uid is taken over this way; "Running under upaas" in `README.md` no - longer tells the operator to create or chown the host directory. +- 2026-10-02 a plain `docker build .` stamps the tag or short commit, not `dev` + (closes #166): `.dockerignore` lets `.git` into the build context, without + `.git/config`; with no `VERSION` build argument the `Dockerfile` takes the + version from `git describe --tags --always`, and fails the build if the + context carries `.git` and no version comes out; `ARG VERSION` has no default; + pixad logs its version, with its name and architecture, as its first log line + at startup. +- 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes #159): + `deploy/docker-entrypoint.sh` creates the directory if it is missing, gives + the directory and everything in it to `pixad` when the directory or one of its + top-level entries belongs to another user or group, sets its mode to `750`, + then runs the server as `pixad`; data left by an earlier run under another uid + is taken over this way; "Running under upaas" in `README.md` no longer tells + the operator to create or chown the host directory. - 2026-09-29 variant content types kept in memory (closes #70): `Cache.metaCache` holds the content types of up to 10,000 variants in an LRU (`github.com/hashicorp/golang-lru/v2`), filled by `StoreVariant` and by @@ -397,8 +402,8 @@ P2: security: per-IP rate limiting on the image routes `ARG VERSION` sits just above the build, so a new version reruns neither `script/bootstrap` nor the tests. - 2026-09-29 migrations at the path `REPO_POLICIES.md` sets (closes #96): the - migration files moved, contents unchanged, from `internal/database/schema/` - to `internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the + migration files moved, contents unchanged, from `internal/database/schema/` to + `internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the `internal/db/migrations` package embeds them and `internal/database` reads them through its `FS()`; the `internal/database` package itself stays; the version still comes from the filename prefix, so a database that has recorded @@ -413,8 +418,8 @@ P2: security: per-IP rate limiting on the image routes `=` padding kept, and gives the example's `sig` for a stated signing key. - 2026-09-29 fixed uid and gid for `pixad` (closes #151): the image creates the `pixad` group with gid 65532 and the `pixad` user with uid 65532, instead of - the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad` - is not owned on the host by a person's login account; the first-run step of + the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad` is + not owned on the host by a person's login account; the first-run step of "Running under upaas" in `README.md` names the uid and gid. - 2026-09-29 `max-age` never outlives an expiring URL (closes #63): both image routes build `Cache-Control` from the request's `Expires`, which an encrypted @@ -423,29 +428,27 @@ P2: security: per-IP rate limiting on the image routes that is sooner, never negative; an allowlisted host's URL that has an `exp` follows it too; `immutable` stays, as freshness now ends at the expiry; documented in `README.md`. -- 2026-09-28 add the four settings `README.md` documented but pixa did not - have, which aborted startup as unknown keys (closes #61): +- 2026-09-28 add the four settings `README.md` documented but pixa did not have, + which aborted startup as unknown keys (closes #61): `access_control_allow_origin` (default `*`, the CORS origin), `upstream_fetch_timeout` (default `30s`), `upstream_max_response_size` - (default 50 MiB) and `downstream_timeout` (default `60s`, both the - server's write timeout and the per-request timeout); each has a - `PIXA_` variable; durations are positive Go duration strings, the size a - whole number of bytes up to 1 GiB, the origin `*` or one `http` or - `https` origin as `README.md` describes it; an invalid value - aborts startup naming the key and the value; documented in - `config.example.yml` and `README.md`. -- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` - counts the cached source images and processed variants (`source_content` - plus `variant_content`) and takes their size from `Cache.UsageBytes`, - instead of reading `request_cache` and `output_content`, which nothing - writes; those two tables are left in the schema; a disabled disk cache - reports no items and no size. A hit is counted even when the request - context has ended. A miss is counted after it is served or fails, also - when the request context has ended by then, with the bytes it read from - upstream, so `upstream_fetch_count` and `upstream_fetch_bytes` move, - including for an upstream body that fails partway or a fetched source - that then fails the magic byte check; `transform_count` counts each image - the image processor transcodes. + (default 50 MiB) and `downstream_timeout` (default `60s`, both the server's + write timeout and the per-request timeout); each has a `PIXA_` variable; + durations are positive Go duration strings, the size a whole number of bytes + up to 1 GiB, the origin `*` or one `http` or `https` origin as `README.md` + describes it; an invalid value aborts startup naming the key and the value; + documented in `config.example.yml` and `README.md`. +- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` counts + the cached source images and processed variants (`source_content` plus + `variant_content`) and takes their size from `Cache.UsageBytes`, instead of + reading `request_cache` and `output_content`, which nothing writes; those two + tables are left in the schema; a disabled disk cache reports no items and no + size. A hit is counted even when the request context has ended. A miss is + counted after it is served or fails, also when the request context has ended + by then, with the bytes it read from upstream, so `upstream_fetch_count` and + `upstream_fetch_bytes` move, including for an upstream body that fails partway + or a fetched source that then fails the magic byte check; `transform_count` + counts each image the image processor transcodes. - 2026-09-28 strip metadata from processed images (closes #82): every output is exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC profile; the image is first turned upright with `AutoRotate` (before sizes are @@ -456,220 +459,201 @@ P2: security: per-IP rate limiting on the image routes attempts per minute per client address, and an attempt over the limit is refused with 429 and a `Retry-After` header; the address is the one `internal/clientip` resolves through `trusted_proxies`, an IPv6 client is - counted by its /64, and an IPv4-mapped address as the IPv4 address it - carries; the limit is a `RateLimit` middleware in `internal/middleware` on + counted by its /64, and an IPv4-mapped address as the IPv4 address it carries; + the limit is a `RateLimit` middleware in `internal/middleware` on `github.com/go-chi/httprate`, which the image routes can reuse; the library keeps counts for the current and the previous minute only; documented in `README.md`. -- 2026-09-28 refuse an unparseable `exp` on `/v1/image/` and log swallowed - cache errors (closes #72): an `exp` in the URL that is not a whole - number, an empty `exp=` included, is a 400 naming `exp` and the value, - instead of being ignored and answered with 401 as if the URL had no - `exp`; only an `exp` missing from the URL is unchanged; `README.md` says - so where it documents `exp`. A failed variant `.meta` write, source - metadata JSON write, `Stats` count query, stats counter update, negative - cache write or expired negative cache delete is now logged at `warn` - with the path or key and the error, and stays non-fatal. -- 2026-09-28 refuse an empty `fit` on `/v1/image/` (closes #139): a - `fit` in the URL with an empty value (`fit=`) is a 400 naming `fit`, - instead of being served as `cover` and verified against a signature - made for `cover`; only a `fit` missing from the URL is still `cover`; - any other value still goes through the existing fit-mode check; - `README.md` says so where it documents `fit`. -- 2026-09-28 refuse an invalid `q` on `/v1/image/` (closes #134): a `q` - that is not a whole number from 1 to 100, an empty `q` included, is a - 400 naming `q` and the value, instead of being served at the default - 85; the route reads `q` with the generator's quality check - (`parseFormInt` with `minQuality` and `maxQuality`); only a `q` missing - from the URL is still 85; a query string that cannot be decoded, such - as `q=80%`, is a 400 showing it; any query parameter given more than - once (`q`, `fit`, `sig`, `exp` alike) is a 400 naming it, so none is - read from its first value only; `README.md` states the range and both - query-string rules. -- 2026-09-28 unknown `PIXA_` environment variables abort startup (closes - #133): a variable whose name starts with `PIXA_` but is neither a - setting's variable nor `PIXA_CONFIG_PATH` aborts startup naming it, as - an unknown config key does, and `PIXA_PORT` is named with a pointer to - `PORT`; the check runs after the config file loads, so the variables - the file's `env:` section sets are checked too; documented in - `README.md`. -- 2026-09-28 start on a fresh upaas volume (closes #129): the image - starts as root only to give `/var/lib/pixa` to `pixad` when `pixad` - does not own it (`deploy/docker-entrypoint.sh`), then runs the server - as `pixad` through `su-exec`, so a root-owned host directory - bind-mounted there no longer stops the container at startup; - `README.md` gains a "Running under upaas" section. -- 2026-09-28 run all linting in Docker via `Dockerfile.lint` + - `script/lint` (closes #104): `make lint` calls `script/lint`, the only - way the linter is run; inside a container (both Dockerfiles set - `container=docker`) it runs `golangci-lint`, anywhere else it builds the - hash-pinned `Dockerfile.lint`, whose last step runs `script/lint` again; - the `Dockerfile` lint stage runs `make lint`; no host or nix-shell - `golangci-lint` path remains (`script/bootstrap` installs no linter); - a per-run `CACHEBUST` build-arg keeps the lint step from being served - from cache, and a tmpfs mount on that step keeps Go's and - golangci-lint's caches out of its layer, so a run leaves no large build - cache behind; `golangci-lint config verify` stays out, as it fetches its - schema over an unpinned live HTTPS call -- 2026-09-28 every setting as an environment variable (closes #128, also - covers #99): each config key can be set by `PIXA_` plus the key in upper - case (`.` written as `_`), and the port by `PORT`; a variable present in - the environment, even empty, wins over the config file, which wins over - the default; the typed getters read the variable first, so every existing - check applies to it and a bad value aborts startup naming the variable; - lists are comma-separated, and an empty variable (or `""` in the file) is - an empty list; the Docker image no longer bakes in `config.docker.yml` or - passes `--config`, and its `HEALTHCHECK` probes `PORT` (default `8080`); - the config file is looked for under `/etc/pixa` and `~/.config/pixa` - instead of the daemon name `pixad`; documented in `README.md` and - `config.example.yml`. -- 2026-09-28 quality and fit in the URL signature (closes #60): the signed - data is now `host:path:query:width:height:format:expiration:quality:fit`, - using `85` and `cover` when the URL has no `q` or `fit`, so one signed - URL can no longer be replayed across other quality and fit values to - create unauthorized cache entries and transcodes; the known-answer - vectors in `internal/signature/golden_test.go` and the README signature - specification describe the new format. -- 2026-09-28 Docker image healthcheck (closes #111): a `HEALTHCHECK` in - the runtime stage probing `/.well-known/healthcheck.json` with busybox - `wget`; `script/docker-smoke` (`make docker-smoke`) builds the image, - starts it with a throwaway `PIXA_SIGNING_KEY`, and passes only once - Docker reports it healthy within 30 seconds, removing the container on - exit; the Gitea workflow runs it after `script/cibuild`. +- 2026-09-28 refuse an unparseable `exp` on `/v1/image/` and log swallowed cache + errors (closes #72): an `exp` in the URL that is not a whole number, an empty + `exp=` included, is a 400 naming `exp` and the value, instead of being ignored + and answered with 401 as if the URL had no `exp`; only an `exp` missing from + the URL is unchanged; `README.md` says so where it documents `exp`. A failed + variant `.meta` write, source metadata JSON write, `Stats` count query, stats + counter update, negative cache write or expired negative cache delete is now + logged at `warn` with the path or key and the error, and stays non-fatal. +- 2026-09-28 refuse an empty `fit` on `/v1/image/` (closes #139): a `fit` in the + URL with an empty value (`fit=`) is a 400 naming `fit`, instead of being + served as `cover` and verified against a signature made for `cover`; only a + `fit` missing from the URL is still `cover`; any other value still goes + through the existing fit-mode check; `README.md` says so where it documents + `fit`. +- 2026-09-28 refuse an invalid `q` on `/v1/image/` (closes #134): a `q` that is + not a whole number from 1 to 100, an empty `q` included, is a 400 naming `q` + and the value, instead of being served at the default 85; the route reads `q` + with the generator's quality check (`parseFormInt` with `minQuality` and + `maxQuality`); only a `q` missing from the URL is still 85; a query string + that cannot be decoded, such as `q=80%`, is a 400 showing it; any query + parameter given more than once (`q`, `fit`, `sig`, `exp` alike) is a 400 + naming it, so none is read from its first value only; `README.md` states the + range and both query-string rules. +- 2026-09-28 unknown `PIXA_` environment variables abort startup (closes #133): + a variable whose name starts with `PIXA_` but is neither a setting's variable + nor `PIXA_CONFIG_PATH` aborts startup naming it, as an unknown config key + does, and `PIXA_PORT` is named with a pointer to `PORT`; the check runs after + the config file loads, so the variables the file's `env:` section sets are + checked too; documented in `README.md`. +- 2026-09-28 start on a fresh upaas volume (closes #129): the image starts as + root only to give `/var/lib/pixa` to `pixad` when `pixad` does not own it + (`deploy/docker-entrypoint.sh`), then runs the server as `pixad` through + `su-exec`, so a root-owned host directory bind-mounted there no longer stops + the container at startup; `README.md` gains a "Running under upaas" section. +- 2026-09-28 run all linting in Docker via `Dockerfile.lint` + `script/lint` + (closes #104): `make lint` calls `script/lint`, the only way the linter is + run; inside a container (both Dockerfiles set `container=docker`) it runs + `golangci-lint`, anywhere else it builds the hash-pinned `Dockerfile.lint`, + whose last step runs `script/lint` again; the `Dockerfile` lint stage runs + `make lint`; no host or nix-shell `golangci-lint` path remains + (`script/bootstrap` installs no linter); a per-run `CACHEBUST` build-arg keeps + the lint step from being served from cache, and a tmpfs mount on that step + keeps Go's and golangci-lint's caches out of its layer, so a run leaves no + large build cache behind; `golangci-lint config verify` stays out, as it + fetches its schema over an unpinned live HTTPS call +- 2026-09-28 every setting as an environment variable (closes #128, also covers + #99): each config key can be set by `PIXA_` plus the key in upper case (`.` + written as `_`), and the port by `PORT`; a variable present in the + environment, even empty, wins over the config file, which wins over the + default; the typed getters read the variable first, so every existing check + applies to it and a bad value aborts startup naming the variable; lists are + comma-separated, and an empty variable (or `""` in the file) is an empty list; + the Docker image no longer bakes in `config.docker.yml` or passes `--config`, + and its `HEALTHCHECK` probes `PORT` (default `8080`); the config file is + looked for under `/etc/pixa` and `~/.config/pixa` instead of the daemon name + `pixad`; documented in `README.md` and `config.example.yml`. +- 2026-09-28 quality and fit in the URL signature (closes #60): the signed data + is now `host:path:query:width:height:format:expiration:quality:fit`, using + `85` and `cover` when the URL has no `q` or `fit`, so one signed URL can no + longer be replayed across other quality and fit values to create unauthorized + cache entries and transcodes; the known-answer vectors in + `internal/signature/golden_test.go` and the README signature specification + describe the new format. +- 2026-09-28 Docker image healthcheck (closes #111): a `HEALTHCHECK` in the + runtime stage probing `/.well-known/healthcheck.json` with busybox `wget`; + `script/docker-smoke` (`make docker-smoke`) builds the image, starts it with a + throwaway `PIXA_SIGNING_KEY`, and passes only once Docker reports it healthy + within 30 seconds, removing the container on exit; the Gitea workflow runs it + after `script/cibuild`. - 2026-09-21 trusted-proxy client IP resolution (closes #94): a `trusted_proxies` config key taking a list of CIDRs, parsed by the same - `net/netip` list parser as `blocked_networks` (an invalid entry aborts - startup naming the key and value; an omitted key defaults to the RFC 1918 - private ranges, an explicitly empty list trusts no one, and an explicit - list replaces the default); a new - `internal/clientip` package resolves the client address by honoring - `X-Forwarded-For` only when the direct peer is a trusted proxy, walking - the chain right-to-left to the rightmost non-proxy entry, so a client - connecting directly cannot spoof its address; the resolved address is - stored in the request context by a new middleware and used by the - request-logging middleware and the login-attempt logs in place of the - raw peer address; documented in `README.md` and `config.example.yml`. + `net/netip` list parser as `blocked_networks` (an invalid entry aborts startup + naming the key and value; an omitted key defaults to the RFC 1918 private + ranges, an explicitly empty list trusts no one, and an explicit list replaces + the default); a new `internal/clientip` package resolves the client address by + honoring `X-Forwarded-For` only when the direct peer is a trusted proxy, + walking the chain right-to-left to the rightmost non-proxy entry, so a client + connecting directly cannot spoof its address; the resolved address is stored + in the request context by a new middleware and used by the request-logging + middleware and the login-attempt logs in place of the raw peer address; + documented in `README.md` and `config.example.yml`. - 2026-09-21 blocked networks configuration extending SSRF protection: a - `blocked_networks` config key taking a list of CIDRs (parsed with - `net/netip`, an invalid entry aborts startup naming the key and value), - added to the built-in blocklist rather than replacing it; the built-in - ranges extended to CGNAT `100.64.0.0/10`, IETF protocol assignments - `192.0.0.0/24`, benchmark `198.18.0.0/15`, and NAT64 `64:ff9b::/96` - (IPv4-mapped forms covered); enforcement stays in the dial-time - re-resolution so the DNS-rebinding window remains closed; documented in - `README.md` and `config.example.yml`. -- 2026-09-21 validate dimensions and fit mode on the encrypted-URL - route and the token generator (closes #62): `imgcache.ValidateDimension` - alone holds the `MaxDimension` bound and is used by the path parser, by - the new `ValidateImageRequest` (which also applies `ValidateFitMode`) - and by the generator; both the `/v1/image/` and `/v1/e/` routes call - `ValidateImageRequest`, so an over-limit size or an unknown fit mode is a - 400 rather than an out-of-memory or a 500 from the processor; the URL - generator answers 400 naming the field for a `width` or `height` that is - not a number or fails the shared check, a `quality` that is not a number - from 1 to 100, a `ttl` that is not a number from 0 to the largest number - of seconds the expiry calculation can hold, or an unknown `fit`; an empty - `quality` is 85 and - an empty `ttl` never expires; the form's width and height inputs stop at - 8192 -- 2026-09-21 http.Server hardening (closes #92): added - `HTTPReadHeaderTimeout` (10s, bounds the slowloris header dribble) and - `HTTPIdleTimeout` (120s, bounds keep-alive reuse) alongside the - existing timeouts and wired them onto the server; added a `LimitBody` - middleware capping the two form POST bodies (`POST /`, `POST /generate`) - at `MaxFormBytes` (1 MiB) and returning 413, applied ahead of the CSRF - middleware so an oversized body is refused as 413 rather than being read - as a missing CSRF token (403); left `WriteTimeout` at 60s unchanged -- 2026-08-07 update golangci-lint to v2.12.2 with the canonical - `.golangci.yml` (v2 schema, `default: all` minus six disabled - linters, `lll` 88, tests included): bumped the pinned - `golangci/golangci-lint:v2.12.2-alpine` image in `Dockerfile` and the - release-archive sha256 pins in `script/bootstrap`; fixed the findings - the stricter config surfaced (notably `paralleltest`, `wsl_v5`, + `blocked_networks` config key taking a list of CIDRs (parsed with `net/netip`, + an invalid entry aborts startup naming the key and value), added to the + built-in blocklist rather than replacing it; the built-in ranges extended to + CGNAT `100.64.0.0/10`, IETF protocol assignments `192.0.0.0/24`, benchmark + `198.18.0.0/15`, and NAT64 `64:ff9b::/96` (IPv4-mapped forms covered); + enforcement stays in the dial-time re-resolution so the DNS-rebinding window + remains closed; documented in `README.md` and `config.example.yml`. +- 2026-09-21 validate dimensions and fit mode on the encrypted-URL route and the + token generator (closes #62): `imgcache.ValidateDimension` alone holds the + `MaxDimension` bound and is used by the path parser, by the new + `ValidateImageRequest` (which also applies `ValidateFitMode`) and by the + generator; both the `/v1/image/` and `/v1/e/` routes call + `ValidateImageRequest`, so an over-limit size or an unknown fit mode is a 400 + rather than an out-of-memory or a 500 from the processor; the URL generator + answers 400 naming the field for a `width` or `height` that is not a number or + fails the shared check, a `quality` that is not a number from 1 to 100, a + `ttl` that is not a number from 0 to the largest number of seconds the expiry + calculation can hold, or an unknown `fit`; an empty `quality` is 85 and an + empty `ttl` never expires; the form's width and height inputs stop at 8192 +- 2026-09-21 http.Server hardening (closes #92): added `HTTPReadHeaderTimeout` + (10s, bounds the slowloris header dribble) and `HTTPIdleTimeout` (120s, bounds + keep-alive reuse) alongside the existing timeouts and wired them onto the + server; added a `LimitBody` middleware capping the two form POST bodies + (`POST /`, `POST /generate`) at `MaxFormBytes` (1 MiB) and returning 413, + applied ahead of the CSRF middleware so an oversized body is refused as 413 + rather than being read as a missing CSRF token (403); left `WriteTimeout` at + 60s unchanged +- 2026-08-07 update golangci-lint to v2.12.2 with the canonical `.golangci.yml` + (v2 schema, `default: all` minus six disabled linters, `lll` 88, tests + included): bumped the pinned `golangci/golangci-lint:v2.12.2-alpine` image in + `Dockerfile` and the release-archive sha256 pins in `script/bootstrap`; fixed + the findings the stricter config surfaced (notably `paralleltest`, `wsl_v5`, `goconst`, `lll`, `noinlineerr`, `err113`, `errcheck`, `testpackage` — - white-box test files renamed to `*_internal_test.go`), including #55's - code absorbed after it merged, iterating the pinned linter to - `0 issues.`; no single finding total is substantiable, since - golangci-lint's `uniq-by-line` reveals new findings on a line as - others there are fixed — the documented re-measurements were 81 after - the #53 merge and 149 after the #55 merge; three behavior changes, so - not a pure no-op: `Cache.StoreVariant` now takes a `context.Context` - (`noctx`), so a cancelled request skips its best-effort accounting - row; `MetadataStorage.Store`'s cleanup defer was dead on `main` and - leaked `.tmp-*.json` on failure, now fixed with explicit removals; and - the `signing_key` validation error text gained `value too short: `; - the eviction loop's uncancellable context is deferred to #102 under a - `//nolint:contextcheck`; three `//nolint:tagliatelle` directives keep - the snake_case JSON wire/disk formats unchanged; `make check` green -- 2026-08-07 implement cache size management and eviction (closes - #51): new `cache_max_bytes` config key validated by the startup - framework (explicit values used exactly with no floor, `0` disables - the disk cache entirely, omitted defaults to max(75% of free space - on the filesystem containing `/cache/`, 500 MiB), logged - at startup); processed variants are now tracked in the database (a - new `variant_content` table and an LRU timestamp on `source_content`) - so total usage is two SUMs, never a directory scan on the hot path; a - background goroutine evicts globally least-recently-used entries - (variants and source blobs merged) to the limit, woken by a periodic - ticker and by write-pressure notifications from stores; a source - blob and ALL of its `source_metadata` references are deleted in one - transaction before the file is unlinked, so multi-referenced blobs - are never removed while referenced and rows never point at deleted - files; a startup and periodic reconciliation pass adopts untracked - variant files, drops rows for missing files, removes unreachable - source blobs, and sweeps stale temp files -- 2026-08-07 validate configuration on startup, fail fast on bad - config (closes #52): a config value that is set but unparseable or - invalid aborts startup naming the key and value (defaults apply only - to omitted keys), unknown config keys abort startup, a malformed - config file aborts instead of being skipped, and `state_dir` is - verified creatable and writable before the listener binds -- 2026-08-07 manual test pass of the auth and encrypted URL flows - against a locally built and running `pixad` (built from `main` at - `6573b9d`, port 18099, local throwaway config); all six checks - passed, plus all nine tests in `scripts/manual-test.sh` (closes #49): - - [x] visit `/` and see the login form: HTTP 200, `Pixa - Login` - page with `name="key"` password form - - [x] wrong key shows an error: POST `/` with `key=wrong-key` - returned HTTP 200 login page containing "Invalid signing key" - - [x] correct signing key shows the generator form: POST `/` - returned HTTP 303 to `/` with - `Set-Cookie: pixa_session=...; HttpOnly; Secure; SameSite=Strict`; - GET `/` with that cookie rendered `Pixa - URL Generator` with the - `/generate` form and logout link + white-box test files renamed to `*_internal_test.go`), including #55's code + absorbed after it merged, iterating the pinned linter to `0 issues.`; no + single finding total is substantiable, since golangci-lint's `uniq-by-line` + reveals new findings on a line as others there are fixed — the documented + re-measurements were 81 after the #53 merge and 149 after the #55 merge; three + behavior changes, so not a pure no-op: `Cache.StoreVariant` now takes a + `context.Context` (`noctx`), so a cancelled request skips its best-effort + accounting row; `MetadataStorage.Store`'s cleanup defer was dead on `main` and + leaked `.tmp-*.json` on failure, now fixed with explicit removals; and the + `signing_key` validation error text gained `value too short: `; the eviction + loop's uncancellable context is deferred to #102 under a + `//nolint:contextcheck`; three `//nolint:tagliatelle` directives keep the + snake_case JSON wire/disk formats unchanged; `make check` green +- 2026-08-07 implement cache size management and eviction (closes #51): new + `cache_max_bytes` config key validated by the startup framework (explicit + values used exactly with no floor, `0` disables the disk cache entirely, + omitted defaults to max(75% of free space on the filesystem containing + `/cache/`, 500 MiB), logged at startup); processed variants are now + tracked in the database (a new `variant_content` table and an LRU timestamp on + `source_content`) so total usage is two SUMs, never a directory scan on the + hot path; a background goroutine evicts globally least-recently-used entries + (variants and source blobs merged) to the limit, woken by a periodic ticker + and by write-pressure notifications from stores; a source blob and ALL of its + `source_metadata` references are deleted in one transaction before the file is + unlinked, so multi-referenced blobs are never removed while referenced and + rows never point at deleted files; a startup and periodic reconciliation pass + adopts untracked variant files, drops rows for missing files, removes + unreachable source blobs, and sweeps stale temp files +- 2026-08-07 validate configuration on startup, fail fast on bad config (closes + #52): a config value that is set but unparseable or invalid aborts startup + naming the key and value (defaults apply only to omitted keys), unknown config + keys abort startup, a malformed config file aborts instead of being skipped, + and `state_dir` is verified creatable and writable before the listener binds +- 2026-08-07 manual test pass of the auth and encrypted URL flows against a + locally built and running `pixad` (built from `main` at `6573b9d`, port 18099, + local throwaway config); all six checks passed, plus all nine tests in + `scripts/manual-test.sh` (closes #49): + - [x] visit `/` and see the login form: HTTP 200, `Pixa - Login` page with + `name="key"` password form + - [x] wrong key shows an error: POST `/` with `key=wrong-key` returned HTTP + 200 login page containing "Invalid signing key" + - [x] correct signing key shows the generator form: POST `/` returned HTTP + 303 to `/` with + `Set-Cookie: pixa_session=...; HttpOnly; Secure; SameSite=Strict`; GET + `/` with that cookie rendered `Pixa - URL Generator` with the + `/generate` form and logout link - [x] a generated encrypted URL serves the image: POST `/generate` - (ttl=3600) produced a `/v1/e//img.jpeg` URL that returned - HTTP 200, `Content-Type: image/jpeg`, an 800x600 baseline JPEG of - 61706 bytes - - [x] an expired URL (short TTL) returns 410: a ttl=1 URL fetched - after 3 s returned HTTP 410 Gone with - `{"error":"URL has expired","status":410,...}` - - [x] logout redirects back to login: GET `/logout` returned HTTP - 303 to `/` with `Set-Cookie: pixa_session=; Max-Age=0`; - subsequent GET `/` rendered the login form again -- 2026-08-07 fix the two remaining gosec findings (G124 in - internal/session): session cookies now always carry - Secure/HttpOnly/SameSite=Strict on both the set and clear paths; - `make check` green (closes #47) -- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, - Makefile shims, README Entrypoints section + (ttl=3600) produced a `/v1/e//img.jpeg` URL that returned HTTP + 200, `Content-Type: image/jpeg`, an 800x600 baseline JPEG of 61706 + bytes + - [x] an expired URL (short TTL) returns 410: a ttl=1 URL fetched after 3 s + returned HTTP 410 Gone with + `{"error":"URL has expired","status":410,...}` + - [x] logout redirects back to login: GET `/logout` returned HTTP 303 to `/` + with `Set-Cookie: pixa_session=; Max-Age=0`; subsequent GET `/` + rendered the login form again +- 2026-08-07 fix the two remaining gosec findings (G124 in internal/session): + session cookies now always carry Secure/HttpOnly/SameSite=Strict on both the + set and clear paths; `make check` green (closes #47) +- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile + shims, README Entrypoints section - 2026-04-07 extract magic byte detection into internal/magic (#42) - 2026-03-25 extract allowlist package from internal/imgcache (#41) - 2026-03-25 move schema_migrations table creation into 000.sql (#36) -- 2026-03-20 enforce and document exact-match-only signature - verification (#40) -- 2026-03-20 bound imageprocessor.Process input read to prevent - unbounded memory use (#37); consolidate appname into an - internal/globals constant (#34) +- 2026-03-20 enforce and document exact-match-only signature verification (#40) +- 2026-03-20 bound imageprocessor.Process input read to prevent unbounded memory + use (#37); consolidate appname into an internal/globals constant (#34) - 2026-03-18 parse version prefix from migration filenames (#33) - 2026-03-15 QA audit fixes for 1.0/MVP readiness (#25) -- 2026-03-02 split Dockerfile with pre-built golangci-lint stage for - faster CI (#23) +- 2026-03-02 split Dockerfile with pre-built golangci-lint stage for faster CI + (#23) - 2026-02-25 repo policy compliance: CI workflow, hash-pinned images, - golangci-lint and gosec fixes of that date (#14); arm64 Docker build - fix (#16) -- 2026-01-08 WebP and AVIF encoding support via govips (both former P0 - image processing items, now done) + golangci-lint and gosec fixes of that date (#14); arm64 Docker build fix (#16) +- 2026-01-08 WebP and AVIF encoding support via govips (both former P0 image + processing items, now done) # Future Steps diff --git a/package.json b/package.json new file mode 100644 index 0000000..bab3bb1 --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "license": "GPL-3.0", + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/script/bootstrap b/script/bootstrap index e826a28..31abd17 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,20 +3,33 @@ # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes NOTHING is present (not git, -# make, or go). The linter is never installed on the host: golangci-lint -# runs only in the lint phase of the Dockerfile (see script/lint). +# make, or go). Node is used directly if installed; otherwise it is +# installed at a pinned version via nvm (installing nvm itself first, +# from a hash-verified release archive, never curl | sh). The linter is +# never installed on the host: golangci-lint runs only in the lint phase +# of the Dockerfile (see script/lint). # -# script/bootstrap git, make and Go, all the host needs: the -# checks compile pixa in Docker -# script/bootstrap --cgo also a C compiler and the CGO image -# libraries (pkg-config, vips, libheif) for -# the govips bindings, to compile pixa; the -# Dockerfile's test phase and build stage -# run this +# script/bootstrap git, make, Go, and Node, Yarn and the +# prettier in yarn.lock for script/fmt and +# script/fmt-check: all the host needs, as +# the checks compile pixa in Docker +# script/bootstrap --cgo git, make, Go, and a C compiler and the +# CGO image libraries (pkg-config, vips, +# libheif) for the govips bindings instead +# of Node: to compile pixa, in the +# Dockerfile's test phase and build stage, +# which format nothing set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Pinned versions, 2026-07-06 +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + PKGMGR="" SUDO="" @@ -61,6 +74,69 @@ missing() { ! command -v "$1" >/dev/null 2>&1 } +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +ensure_node() { + if ! missing node; then return 0; fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" +} + +ensure_yarn() { + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + elif [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + else + yarn install --frozen-lockfile + fi +} + # CGO dependencies for govips (image processing) ensure_cgo_deps() { # cgo compiles with gcc on Linux; build-base and build-essential @@ -98,9 +174,14 @@ main() { # Go toolchain if missing go; then pkg_install go golang go go; fi - # CGO image libraries, only where pixa is compiled + # CGO image libraries where pixa is compiled; elsewhere Node, Yarn + # and prettier if [ "$*" = "--cgo" ]; then ensure_cgo_deps + else + ensure_node + ensure_yarn + install_js_deps fi go mod download diff --git a/script/fmt b/script/fmt index 976c3a2..c7b584a 100755 --- a/script/fmt +++ b/script/fmt @@ -4,11 +4,31 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# script/bootstrap installs node and yarn under nvm and leaves neither +# on the PATH of the shell that called it, so resolve the pinned +# toolchain here the way bootstrap's own install step does. nvm is a +# bash script, hence the subshell. +run_yarn() { + if command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + main() { cd "$ROOT" echo "Formatting code..." # shellcheck disable=SC2046 # word splitting of file list is wanted gofmt -w $(find . -name '*.go' -not -path './vendor/*') + run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/script/fmt-check b/script/fmt-check index 848fdc6..b8d3f00 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -5,6 +5,25 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# script/bootstrap installs node and yarn under nvm and leaves neither +# on the PATH of the shell that called it, so resolve the pinned +# toolchain here the way bootstrap's own install step does. nvm is a +# bash script, hence the subshell. +run_yarn() { + if command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt-check: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + main() { cd "$ROOT" echo "Checking formatting..." @@ -13,6 +32,7 @@ main() { gofmt -l . | grep -v '^vendor/' exit 1 fi + run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always } main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg== From 15d95436a5c8589c71924e44121bdd056aaf9c66 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 5 Oct 2026 05:24:50 +0200 Subject: [PATCH 050/131] Serve the format auto, chosen from the Accept header (closes #88) auto is a format in the /v1/image/ path, an encrypted URL's token and the generator page. Once the signature or token is checked, pixa chooses AVIF when Accept names image/avif, else WebP when it names image/webp, else JPEG when the most specific of image/jpeg, image/* and */* allows it or Accept is absent. q=0 refuses a format; a header allowing none of the three answers 406, one that does not parse 400. The signature and token cover auto itself; the cache key and ETag use the chosen format. Answers from then on carry Vary: Accept. Model: opus-5-5 --- README.md | 42 ++- TODO.md | 13 +- internal/handlers/auth.go | 2 +- internal/handlers/format_auto.go | 131 ++++++++ .../handlers/format_auto_internal_test.go | 310 ++++++++++++++++++ internal/handlers/image.go | 5 + internal/handlers/imageenc.go | 2 +- internal/imgcache/imgcache.go | 5 + internal/imgcache/urlparser.go | 2 + internal/imgcache/urlparser_internal_test.go | 15 + internal/server/format_auto_internal_test.go | 46 +++ internal/templates/generator.html | 1 + 12 files changed, 560 insertions(+), 14 deletions(-) create mode 100644 internal/handlers/format_auto.go create mode 100644 internal/handlers/format_auto_internal_test.go create mode 100644 internal/server/format_auto_internal_test.go diff --git a/README.md b/README.md index 95d9079..bfa0451 100644 --- a/README.md +++ b/README.md @@ -175,12 +175,14 @@ path under `/v1/` answers 200, in maintenance mode too. - `GET` or `HEAD` `/v1/image///.` — an image, fetched, resized and converted (below). Needs: a signature, unless the host is allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches - the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a - missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when - the request's `Referer` names a host in `referer_blocklist`, checked before - the signature, the cache and the upstream fetch; 403 when the upstream host, - or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`, - or has an address in a blocked network (see `blocked_networks`); 502 when the + the image's `ETag`; 400 for a URL or parameter that is not valid, or for the + format `auto` an `Accept` header that is not valid; 406 for the format `auto` + when `Accept` allows none of the formats it chooses from; 401 for a missing or + wrong signature, a missing `exp` or an `exp` in the past; 403 when the + request's `Referer` names a host in `referer_blocklist`, checked before the + signature, the cache and the upstream fetch; 403 when the upstream host, or a + host it redirects to, is `localhost`, ends in `.localhost` or `.local`, or has + an address in a blocked network (see `blocked_networks`); 502 when the upstream answered with an error status, and for 5 minutes after that for the same source URL; 503 when pixa is busy or in maintenance mode; 500 for any other failure. @@ -190,7 +192,8 @@ path under `/v1/` answers 200, in maintenance mode too. decrypt, or that asks for a size or fit that is not valid; 410 once it has expired; 504 when the upstream has not sent its response headers within `upstream_fetch_timeout`, but 500 when that time runs out while the image - itself is still arriving; 403, 502, 503 and 500 as for `/v1/image/`. + itself is still arriving; 400 for an `Accept` header that is not valid, and + 406, 403, 502, 503 and 500, as for `/v1/image/`. - `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: nothing. Answers: 200. - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, @@ -239,7 +242,7 @@ A request whose query string cannot be decoded, or gives any parameter more than once, is refused with 400. - ``: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, - `avif`, `gif` + `avif`, `gif`, or `auto` (below) - ``: `orig` or `x` (e.g. `800x600`) - `sig` and `exp`: the signature and its expiry, needed unless the host is allowlisted (see Signature Specification) @@ -247,6 +250,24 @@ once, is refused with 400. both optional (values under Signature Specification). Both are part of what is cached, so each value of either is a separate cached image. +With the format `auto`, pixa chooses the format for each request from its +`Accept` header, in this order: + +1. AVIF, when the header names `image/avif`; +2. WebP, when it names `image/webp`; +3. JPEG, when the first of `image/jpeg`, `image/*` and `*/*` that it names + allows it, or when there is no `Accept` header or it is empty. + +An entry with `q=0` refuses its format; other `q` values do not change the +order. AVIF and WebP must be named, as clients that cannot show them also send +`image/*` and `*/*`. pixa never sends a format the client refused: when the +header allows none of the three, the answer is 406, and a header that does not +parse, or has a `q` that is not a number from 0 to 1, is refused with 400. The +signature, or the token of an encrypted URL, covers `auto` itself, so one URL +serves every client. Each format chosen is cached as a separate image, and every +answer that depends on `Accept` (the image, a 304, and the 400 and 406 above) +carries `Vary: Accept`, so a shared cache keeps the formats apart too. + An image is served with `Cache-Control: public, max-age=, immutable`. When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), `max-age` is the whole seconds left until then, at most one year, so no browser or proxy @@ -297,7 +318,7 @@ nor change what it asks for. 3. The page shows the URL, `https:///v1/e//img.`, and when it expires. `` is the host the page was opened on, and the URL starts with `http` instead while `debug` is on. The name after the token is ignored - and only gives the URL a file extension, `jpg` for `orig`. + and only gives the URL a file extension, `jpg` for `orig` and `auto`. The token holds the source's host, path and query and the size, format, quality, fit and expiry, encrypted with a key derived from `signing_key`. The source @@ -353,7 +374,8 @@ Where: - `width` — requested width in pixels, `0` for original - `height` — requested height in pixels, `0` for original - `format` — output format, one of those listed under Routes, with `original` - signed as `orig` and `jpg` as `jpeg` + signed as `orig` and `jpg` as `jpeg`; `auto` is signed as `auto`, not as the + format chosen for the request - `expiration` — the URL's `exp` query parameter, the Unix timestamp when the signature expires; a request whose `exp` is not a whole number, an empty `exp=` included, is refused with 400 diff --git a/TODO.md b/TODO.md index cabe9e2..3cc0295 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-05 the format `auto` (closes #88): a format in the `/v1/image/` path, + an encrypted URL's token and the generator page's format choice, chosen for + each request from `Accept` once the signature or token is checked: AVIF when + the header names `image/avif`, else WebP when it names `image/webp`, else JPEG + when the first of `image/jpeg`, `image/*` and `*/*` that it names allows it, + or when it names nothing; `q=0` refuses a format. AVIF and WebP must be named, + as clients that cannot show them send the wildcards too. A header that allows + none of the three answers 406, one that does not parse 400. The signature and + the token cover `auto` itself; the cache key and `ETag` use the format chosen. + Answers from the point the format is chosen carry `Vary: Accept`, next to the + CORS `Vary: Origin`; fixed-format answers do not. - 2026-10-05 the markdown is formatted with prettier (closes #100): `script/fmt` and `script/fmt-check` run prettier 3.8.1, pinned in `package.json` and `yarn.lock`, on `**/*.md` after `gofmt`, with four-space tabs and @@ -661,8 +672,6 @@ P2: security: per-IP rate limiting on the image routes - per-origin rate limiting - P2: HTTP response handling - Last-Modified headers - - Vary header for content negotiation -- P2: auto format selection (format=auto based on Accept header) - P2: configuration - YAML config file support - P2: operational diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index a0ad493..8184ce9 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -371,7 +371,7 @@ func (s *Handlers) buildGeneratedURL(r *http.Request, token, format string) stri // Determine file extension for the trailing filename ext := format - if ext == "" || ext == "orig" { + if ext == "" || ext == "orig" || ext == "auto" { ext = "jpg" // Default extension } diff --git a/internal/handlers/format_auto.go b/internal/handlers/format_auto.go new file mode 100644 index 0000000..485faed --- /dev/null +++ b/internal/handlers/format_auto.go @@ -0,0 +1,131 @@ +package handlers + +import ( + "errors" + "fmt" + "mime" + "net/http" + "strconv" + "strings" + + "sneak.berlin/go/pixa/internal/imgcache" +) + +// Errors for an Accept header that an auto URL cannot be served for. +var ( + errInvalidAccept = errors.New("invalid Accept header") + errNotAcceptable = errors.New( + "not acceptable: auto serves image/avif, image/webp or image/jpeg") +) + +// chooseAutoFormat replaces the format auto in req with the format +// formatForAccept chooses from r's Accept header, and adds Vary: Accept to the +// response, which then depends on that header. It answers 400 for an Accept +// header that is not valid and 406 for one that allows none of the formats, +// and reports whether req can be served. Any other format is left as it is. +func (s *Handlers) chooseAutoFormat( + w http.ResponseWriter, r *http.Request, req *imgcache.ImageRequest, +) bool { + if req.Format != imgcache.FormatAuto { + return true + } + + w.Header().Add("Vary", "Accept") + + format, err := formatForAccept(strings.Join(r.Header.Values("Accept"), ",")) + if errors.Is(err, errNotAcceptable) { + s.respondError(w, err.Error(), http.StatusNotAcceptable) + + return false + } + + if err != nil { + s.respondError(w, err.Error(), http.StatusBadRequest) + + return false + } + + req.Format = format + + return true +} + +// formatForAccept returns the format an auto URL is served in for the Accept +// header accept: AVIF when it names image/avif, else WebP when it names +// image/webp, else JPEG when its most specific entry of image/jpeg, image/* +// and */* allows it, or when it names nothing. A q of 0 refuses a format. +// AVIF and WebP must be named, as clients that cannot show them send image/* +// and */* too. +func formatForAccept(accept string) (imgcache.ImageFormat, error) { + qualities, err := parseAccept(accept) + if err != nil { + return "", err + } + + if len(qualities) == 0 { + return imgcache.FormatJPEG, nil + } + + if qualities["image/avif"] > 0 { + return imgcache.FormatAVIF, nil + } + + if qualities["image/webp"] > 0 { + return imgcache.FormatWebP, nil + } + + // For JPEG, the most specific entry the header has decides + quality, named := qualities["image/jpeg"] + if !named { + quality, named = qualities["image/*"] + } + + if !named { + quality = qualities["*/*"] + } + + if quality > 0 { + return imgcache.FormatJPEG, nil + } + + return "", errNotAcceptable +} + +// parseAccept returns the q of each media range the Accept header accept +// names, 1 where it gives none. A media range named more than once keeps its +// lowest q, so a refusal is never overridden. A media range that does not +// parse, or a q that is not a number from 0 to 1, is an error. +func parseAccept(accept string) (map[string]float64, error) { + qualities := make(map[string]float64) + + for entry := range strings.SplitSeq(accept, ",") { + // A header field list may hold empty entries + if strings.TrimSpace(entry) == "" { + continue + } + + mediaRange, params, err := mime.ParseMediaType(entry) + if err != nil { + return nil, fmt.Errorf("%w: %q: %w", errInvalidAccept, entry, err) + } + + quality := 1.0 + + if qParam, given := params["q"]; given { + quality, err = strconv.ParseFloat(qParam, 64) + inRange := quality >= 0 && quality <= 1 + + if err != nil || !inRange { + return nil, fmt.Errorf("%w: %q: q is not a number from 0 to 1", + errInvalidAccept, entry) + } + } + + previous, named := qualities[mediaRange] + if !named || quality < previous { + qualities[mediaRange] = quality + } + } + + return qualities, nil +} diff --git a/internal/handlers/format_auto_internal_test.go b/internal/handlers/format_auto_internal_test.go new file mode 100644 index 0000000..9b28d13 --- /dev/null +++ b/internal/handlers/format_auto_internal_test.go @@ -0,0 +1,310 @@ +package handlers + +import ( + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// The content types the tests below expect. +const ( + avifType = "image/avif" + webpType = "image/webp" + jpegType = "image/jpeg" + jsonType = "application/json" +) + +// TestFormatForAccept verifies the format chosen for the format auto from each +// Accept header below, and the error for one that allows none of AVIF, WebP +// and JPEG or is not valid. +func TestFormatForAccept(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + accept string + want imgcache.ImageFormat + wantErr error + }{ + {"AVIF-capable browser", + "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + imgcache.FormatAVIF, nil}, + {"WebP-capable browser", + "image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5", + imgcache.FormatWebP, nil}, + {"WebP only", webpType, imgcache.FormatWebP, nil}, + {"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imgcache.FormatJPEG, nil}, + {"wildcard only", "*/*", imgcache.FormatJPEG, nil}, + {"image wildcard only", "image/*", imgcache.FormatJPEG, nil}, + {"absent", "", imgcache.FormatJPEG, nil}, + {"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imgcache.FormatWebP, nil}, + {"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*", + imgcache.FormatJPEG, nil}, + {"upper case and spaces", " Image/AVIF ; Q=0.5 ", imgcache.FormatAVIF, nil}, + {"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable}, + {"q=0 on everything", "*/*;q=0", "", errNotAcceptable}, + {"PNG only", "image/png", "", errNotAcceptable}, + {"malformed media range", "image/", "", errInvalidAccept}, + {"q not a number", "image/avif;q=high", "", errInvalidAccept}, + {"q above 1", "image/avif;q=2", "", errInvalidAccept}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := formatForAccept(tt.accept) + t.Logf("Accept %q: %q, %v", tt.accept, got, err) + + if got != tt.want || !errors.Is(err, tt.wantErr) { + t.Errorf("formatForAccept(%q) = %q, %v, want %q, %v", + tt.accept, got, err, tt.want, tt.wantErr) + } + }) + } +} + +// autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL, +// both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in +// the format auto, made with h's image service and generator. +func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { + t.Helper() + + signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ + SourceHost: signedHost, + SourcePath: photoPath, + Size: imgcache.Size{Width: 50, Height: 50}, + Format: imgcache.FormatAuto, + }, time.Minute) + if err != nil { + t.Fatalf("GenerateSignedURL() error = %v", err) + } + + token, err := h.encGen.Generate(&encurl.Payload{ + SourceHost: signedHost, + SourcePath: photoPath, + Width: 50, + Height: 50, + Format: imgcache.FormatAuto, + ExpiresAt: time.Now().Add(time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + return signedURL, "/v1/e/" + token + "/img.jpg" +} + +// requestImage sends method for target to srv with an Accept header line for +// each of accept, and returns the response. +func requestImage( + t *testing.T, srv http.Handler, method, target string, accept ...string, +) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), method, target, nil) + + for _, value := range accept { + req.Header.Add("Accept", value) + } + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + t.Logf("%s %s with Accept %q: %d, Content-Type %s, Vary %v, X-Pixa-Cache %s", + method, target, accept, rec.Code, rec.Header().Get("Content-Type"), + rec.Header().Values("Vary"), rec.Header().Get("X-Pixa-Cache")) + + return rec +} + +// TestFormatAuto_ChosenFromAccept requests an auto URL on each image route +// with each Accept below, and checks the answer and that it carries +// Vary: Accept. The signed URL is signed for auto, so it is valid whatever +// Accept chooses. +func TestFormatAuto_ChosenFromAccept(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + signedURL, encryptedURL := autoPhotoURLs(t, h) + + tests := []struct { + name string + accept []string + wantStatus int + wantType string + }{ + {"AVIF accepted", []string{"image/avif,image/webp,*/*;q=0.8"}, + http.StatusOK, avifType}, + {"WebP accepted", []string{"image/webp,*/*;q=0.8"}, http.StatusOK, webpType}, + {"no Accept", nil, http.StatusOK, jpegType}, + {"two Accept lines", []string{"image/png", webpType}, http.StatusOK, webpType}, + {"none of the three", []string{"image/gif"}, + http.StatusNotAcceptable, jsonType}, + {"not valid", []string{"image/avif;q=high"}, http.StatusBadRequest, jsonType}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + for _, target := range []string{signedURL, encryptedURL} { + rec := requestImage(t, srv, http.MethodGet, target, tt.accept...) + gotType := rec.Header().Get("Content-Type") + + if rec.Code != tt.wantStatus || gotType != tt.wantType { + t.Errorf("%s: %d %s, want %d %s; body %s", target, + rec.Code, gotType, tt.wantStatus, tt.wantType, rec.Body) + } + + if !slices.Contains(rec.Header().Values("Vary"), "Accept") { + t.Errorf("%s: Vary = %v, want Accept in it", + target, rec.Header().Values("Vary")) + } + } + }) + } +} + +// TestFormatAuto_SignatureCoversAuto verifies that a /v1/image/ URL with the +// format auto is checked against a signature for auto, not for the format +// chosen: a URL signed for avif, with auto put in its path, is refused for a +// client whose Accept chooses AVIF. +func TestFormatAuto_SignatureCoversAuto(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ + SourceHost: signedHost, + SourcePath: photoPath, + Size: imgcache.Size{Width: 50, Height: 50}, + Format: imgcache.FormatAVIF, + }, time.Minute) + if err != nil { + t.Fatalf("GenerateSignedURL() error = %v", err) + } + + target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1) + if target == signedForAVIF { + t.Fatalf("no /50x50.avif? in %s", signedForAVIF) + } + + rec := requestImage(t, srv, http.MethodGet, target, avifType) + if rec.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", rec.Code, http.StatusUnauthorized) + } +} + +// TestFormatAuto_CachesEachFormatApart requests an auto URL for AVIF, then +// JPEG, then both again. Each format is processed once and then served from +// the cache, with an ETag of its own, so a client never gets the other format +// from the cache. +func TestFormatAuto_CachesEachFormatApart(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + signedURL, _ := autoPhotoURLs(t, h) + + steps := []struct { + wantType string + wantCache string + }{ + {avifType, "MISS"}, + {jpegType, "MISS"}, + {avifType, "HIT"}, + {jpegType, "HIT"}, + } + + etags := make(map[string]string) + + for _, step := range steps { + rec := requestImage(t, srv, http.MethodGet, signedURL, step.wantType) + gotType := rec.Header().Get("Content-Type") + gotCache := rec.Header().Get("X-Pixa-Cache") + + if rec.Code != http.StatusOK || gotType != step.wantType || + gotCache != step.wantCache { + t.Fatalf("Accept %s: %d %s %s, want 200 %s %s", step.wantType, + rec.Code, gotType, gotCache, step.wantType, step.wantCache) + } + + etag := rec.Header().Get("ETag") + if previous, seen := etags[gotType]; seen && previous != etag { + t.Errorf("%s ETag changed from %s to %s", gotType, previous, etag) + } + + etags[gotType] = etag + } + + if etags[avifType] == etags[jpegType] { + t.Errorf("AVIF and JPEG have the same ETag %s", etags[avifType]) + } +} + +// TestFormatAuto_Vary verifies that on each image route a HEAD answer and a +// 304 for an auto URL carry Vary: Accept, and that the answer for a URL with +// a fixed format does not. +func TestFormatAuto_Vary(t *testing.T) { + t.Parallel() + + h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + srv := chi.NewRouter() + srv.Get("/v1/image/*", h.HandleImage()) + srv.Head("/v1/image/*", h.HandleImage()) + srv.Get("/v1/e/{token}/*", h.HandleImageEnc()) + srv.Head("/v1/e/{token}/*", h.HandleImageEnc()) + + signedURL, encryptedURL := autoPhotoURLs(t, h) + + for _, urls := range [][2]string{ + {signedURL, signedPhotoURL(t, h)}, + {encryptedURL, encPhotoURL(t, h)}, + } { + autoURL, fixedURL := urls[0], urls[1] + + head := requestImage(t, srv, http.MethodHead, autoURL, webpType) + checkVaryAccept(t, head, http.StatusOK, true) + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, + autoURL, nil) + req.Header.Set("Accept", webpType) + req.Header.Set("If-None-Match", head.Header().Get("ETag")) + + notModified := httptest.NewRecorder() + srv.ServeHTTP(notModified, req) + checkVaryAccept(t, notModified, http.StatusNotModified, true) + + fixed := requestImage(t, srv, http.MethodGet, fixedURL) + checkVaryAccept(t, fixed, http.StatusOK, false) + } +} + +// checkVaryAccept fails the test unless rec answered wantStatus and, as +// wantVary says, has or has not Accept in its Vary header. +func checkVaryAccept( + t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantVary bool, +) { + t.Helper() + + vary := rec.Header().Values("Vary") + t.Logf("status %d, Vary %v", rec.Code, vary) + + if rec.Code != wantStatus { + t.Errorf("status = %d, want %d", rec.Code, wantStatus) + } + + if slices.Contains(vary, "Accept") != wantVary { + t.Errorf("Vary = %v, want Accept in it: %v", vary, wantVary) + } +} diff --git a/internal/handlers/image.go b/internal/handlers/image.go index e099799..b3c3c31 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -43,6 +43,11 @@ func (s *Handlers) HandleImage() http.HandlerFunc { return } + // The signature covers the format auto, not the format chosen + if !s.chooseAutoFormat(w, r, req) { + return + } + // Get cache key for logging cacheKey := imgcache.CacheKey(req) diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index a9c672f..bf79054 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -30,7 +30,7 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { start := time.Now() req, ok := s.parseImageEncRequest(w, r) - if !ok { + if !ok || !s.chooseAutoFormat(w, r, req) { return } diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index 1647658..7069f9e 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -22,6 +22,11 @@ const ( FormatWebP ImageFormat = "webp" FormatAVIF ImageFormat = "avif" FormatGIF ImageFormat = "gif" + + // FormatAuto stands for AVIF, WebP or JPEG, chosen for each request + // from its Accept header once the URL's signature or token has been + // checked; it is never processed or cached as itself. + FormatAuto ImageFormat = "auto" ) // Size represents requested image dimensions diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index a1a424c..b37d6ff 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -277,6 +277,8 @@ func parseFormat(s string) (ImageFormat, error) { return FormatAVIF, nil case "gif": return FormatGIF, nil + case "auto": + return FormatAuto, nil default: return "", fmt.Errorf("%w: %s", ErrInvalidFormat, s) } diff --git a/internal/imgcache/urlparser_internal_test.go b/internal/imgcache/urlparser_internal_test.go index 2d617ec..9f4e68d 100644 --- a/internal/imgcache/urlparser_internal_test.go +++ b/internal/imgcache/urlparser_internal_test.go @@ -111,6 +111,21 @@ func TestParseImageURL(t *testing.T) { } } +// TestParseImageURL_AutoFormat verifies that the format auto in an image URL +// parses as FormatAuto. +func TestParseImageURL_AutoFormat(t *testing.T) { + t.Parallel() + + got, err := ParseImageURL("/v1/image/example.com/photo.jpg/200x200.auto") + if err != nil { + t.Fatalf("ParseImageURL() error = %v", err) + } + + if got.Format != FormatAuto { + t.Errorf("Format = %q, want %q", got.Format, FormatAuto) + } +} + func TestParseImageURL_Errors(t *testing.T) { t.Parallel() diff --git a/internal/server/format_auto_internal_test.go b/internal/server/format_auto_internal_test.go new file mode 100644 index 0000000..c409725 --- /dev/null +++ b/internal/server/format_auto_internal_test.go @@ -0,0 +1,46 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "slices" + "testing" +) + +// TestFormatAutoVaryNextToOrigin requests an image URL with the format auto +// through the server's routes and verifies that the answer carries +// Vary: Accept next to the Vary: Origin the CORS middleware sends. +func TestFormatAutoVaryNextToOrigin(t *testing.T) { + t.Parallel() + + source := encodeTestPNG(t, 64, 48) + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(source) + })) + t.Cleanup(upstream.Close) + + s, _, _ := startImageProxy(t, upstream) + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, + "/v1/image/"+upstreamHost+"/photo.png/32x24.auto", nil) + req.Header.Set("Origin", "https://app.example.com") + req.Header.Set("Accept", "image/webp") + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + vary := rec.Header().Values("Vary") + t.Logf("status %d, Content-Type %s, Vary %v", + rec.Code, rec.Header().Get("Content-Type"), vary) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + + if want := []string{"Origin", "Accept"}; !slices.Equal(vary, want) { + t.Errorf("Vary = %v, want %v", vary, want) + } +} diff --git a/internal/templates/generator.html b/internal/templates/generator.html index 0f39502..8e6458e 100644 --- a/internal/templates/generator.html +++ b/internal/templates/generator.html @@ -95,6 +95,7 @@ - + +
From db91ab29e6d19801c35284e88b14927bf14289ba Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 12:49:56 +0200 Subject: [PATCH 056/131] Serve JPEG XL when a request names no format (closes #222) A /v1/image/ URL whose last segment is a size with no format, such as 800x600 or orig, is served as JPEG XL and signed as jxl, so it shares the signature of the same URL ending in .jxl. An encrypted URL whose token holds no format is served as JPEG XL, as encurl.DefaultFormat is now jxl. The generator page selects JPEG XL by default, and a form with an empty format, or none, makes a URL whose name ends in .jxl. The image processor no longer takes an empty format as orig: both routes give every request a format, so it refuses a request with none instead of keeping a second default. auto still ends with JPEG. Model: opus-5-5 --- README.md | 51 +++--- TODO.md | 9 + internal/encurl/encurl.go | 4 +- internal/handlers/auth.go | 11 +- internal/handlers/image.go | 3 +- .../handlers/jpegxl_default_internal_test.go | 162 ++++++++++++++++++ .../empty_format_internal_test.go | 21 +++ internal/imageprocessor/imageprocessor.go | 4 +- internal/imgcache/service.go | 4 +- internal/imgcache/urlparser.go | 28 ++- internal/server/routes.go | 3 +- internal/templates/generator.html | 2 +- 12 files changed, 259 insertions(+), 43 deletions(-) create mode 100644 internal/handlers/jpegxl_default_internal_test.go create mode 100644 internal/imageprocessor/empty_format_internal_test.go diff --git a/README.md b/README.md index dbbde09..8b6b032 100644 --- a/README.md +++ b/README.md @@ -175,20 +175,21 @@ path under `/v1/` answers 200, in maintenance mode too. with the page naming a field that is not valid; 500 when the URL cannot be made. - `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`. -- `GET` or `HEAD` `/v1/image///.` — an image, fetched, - resized and converted (below). Needs: a signature, unless the host is - allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches - the image's `ETag`; 400 for a URL or parameter that is not valid, or for the - format `auto` an `Accept` header that is not valid; 406 for the format `auto` - when `Accept` allows none of the formats it chooses from; 401 for a missing or - wrong signature, a missing `exp` or an `exp` in the past; 403 when the - request's `Referer` names a host in `referer_blocklist`, checked before the - signature, the cache and the upstream fetch; 403 when the upstream host, or a - host it redirects to, is `localhost`, ends in `.localhost` or `.local`, or has - an address in a blocked network (see `blocked_networks`); 502 when the - upstream answered with an error status, and for 5 minutes after that for the - same source URL; 503 when pixa is busy or in maintenance mode; 500 for any - other failure. +- `GET` or `HEAD` `/v1/image///.`, or + `/v1/image///` with no format — an image, fetched, resized + and converted (below). Needs: a signature, unless the host is allowlisted (see + Source Hosts). Answers: 200; 304 when `If-None-Match` matches the image's + `ETag`; 400 for a URL or parameter that is not valid, or for the format `auto` + an `Accept` header that is not valid; 406 for the format `auto` when `Accept` + allows none of the formats it chooses from; 401 for a missing or wrong + signature, a missing `exp` or an `exp` in the past; 403 when the request's + `Referer` names a host in `referer_blocklist`, checked before the signature, + the cache and the upstream fetch; 403 when the upstream host, or a host it + redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an + address in a blocked network (see `blocked_networks`); 502 when the upstream + answered with an error status, and for 5 minutes after that for the same + source URL; 503 when pixa is busy or in maintenance mode; 500 for any other + failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when `If-None-Match` matches the image's `ETag`; 400 for a token that does not @@ -231,10 +232,11 @@ proxy in front of pixa must pass that header on unchanged. A form body over 1 MiB is refused with 413. The image routes answer the errors listed for them with JSON holding `error`, `status` and `timestamp`. -An image URL has this form: +An image URL has one of these forms, the second with no format: ``` /v1/image///.?sig=&exp=&q=&fit= +/v1/image///?sig=&exp=&q=&fit= ``` Images are only fetched from origins using TLS with valid certificates, unless @@ -245,7 +247,9 @@ A request whose query string cannot be decoded, or gives any parameter more than once, is refused with 400. - ``: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, - `avif`, `jxl` (JPEG XL), `gif`, or `auto` (below) + `avif`, `jxl` (JPEG XL), `gif`, or `auto` (below). A URL with no format (the + second form, with no dot after the size) is served as JPEG XL, the default, as + with `jxl` - ``: `orig` or `x` (e.g. `800x600`) - `sig` and `exp`: the signature and its expiry, needed unless the host is allowlisted (see Signature Specification) @@ -321,13 +325,15 @@ nor change what it asks for. lasts 30 days, or until `/logout`. 2. On the generator page, give the source image's URL, the width and height, the format, quality and fit, and how long the URL lasts, then submit the form - (`POST /generate`). Width and height both empty or `0` keep the original - size; if only one of them is empty or `0`, that side is scaled to keep the - image's proportions. + (`POST /generate`). The format is JPEG XL unless another is chosen; a form + sent with an empty format, or none, also makes a JPEG XL URL. Width and + height both empty or `0` keep the original size; if only one of them is empty + or `0`, that side is scaled to keep the image's proportions. 3. The page shows the URL, `https:///v1/e//img.`, and when it expires. `` is the host the page was opened on, and the URL starts with `http` instead while `debug` is on. The name after the token is ignored - and only gives the URL a file extension, `jpg` for `orig` and `auto`. + and only gives the URL a file extension, `jpg` for `orig` and `auto`, and + `jxl` for a form with no format. The token holds the source's host, path and query and the size, format, quality, fit and expiry, encrypted with a key derived from `signing_key`. The source @@ -387,8 +393,9 @@ Where: - `width` — requested width in pixels, `0` for original - `height` — requested height in pixels, `0` for original - `format` — output format, one of those listed under Routes, with `original` - signed as `orig` and `jpg` as `jpeg`; `auto` is signed as `auto`, not as the - format chosen for the request + signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no + format has the signature of the same URL ending in `.jxl`; `auto` is signed as + `auto`, not as the format chosen for the request - `expiration` — the URL's `exp` query parameter, the Unix timestamp when the signature expires; a request whose `exp` is not a whole number, an empty `exp=` included, is refused with 400 diff --git a/TODO.md b/TODO.md index 4622779..4bea044 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-08 JPEG XL is the default output (closes #222): a `/v1/image/` URL + whose last segment is a size with no format, such as `800x600` or `orig`, is + served as JPEG XL and signed as `jxl`, so it has the signature of the same URL + ending in `.jxl`. An encrypted URL whose token holds no format is served as + JPEG XL (`encurl.DefaultFormat`). The generator page selects JPEG XL until + another format is chosen, and a form with an empty format, or none, makes a + JPEG XL URL whose name ends in `.jxl`. The image processor no longer takes an + empty format as `orig`: both routes give every request a format, and it + refuses a request with none. `auto` still ends with JPEG. - 2026-10-08 JPEG XL as an input and output format (part of #222): a source whose bytes start with either JPEG XL signature, the bare codestream's `FF 0A` or the container's, is detected as `image/jxl`, which the upstream fetch diff --git a/internal/encurl/encurl.go b/internal/encurl/encurl.go index 8815bf0..13a5772 100644 --- a/internal/encurl/encurl.go +++ b/internal/encurl/encurl.go @@ -14,7 +14,7 @@ import ( // Default values for optional fields. const ( DefaultQuality = 85 - DefaultFormat = imgcache.FormatOriginal + DefaultFormat = imgcache.FormatJXL DefaultFitMode = imgcache.FitCover // HKDF salt for URL encryption key derivation @@ -37,7 +37,7 @@ type Payload struct { SourceQuery string `cbor:"q,omitempty"` // optional Width int `cbor:"w,omitempty"` // 0 = original Height int `cbor:"ht,omitempty"` // 0 = original - Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: orig + Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: jxl Quality int `cbor:"ql,omitempty"` // default: 85 FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index 8184ce9..052ccd2 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -369,10 +369,15 @@ func (s *Handlers) buildGeneratedURL(r *http.Request, token, format string) stri scheme = "http" } - // Determine file extension for the trailing filename + // Determine file extension for the trailing filename. A form with no + // format makes a token with none, which is served as encurl.DefaultFormat. ext := format - if ext == "" || ext == "orig" || ext == "auto" { - ext = "jpg" // Default extension + + switch format { + case "": + ext = string(encurl.DefaultFormat) + case "orig", "auto": + ext = "jpg" } return scheme + "://" + r.Host + "/v1/e/" + url.PathEscape(token) + "/img." + ext diff --git a/internal/handlers/image.go b/internal/handlers/image.go index b3c3c31..1aee1cc 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -18,7 +18,8 @@ import ( ) // HandleImage handles the main image proxy route: -// /v1/image///x. +// /v1/image///x., or with no format +// /v1/image///x func (s *Handlers) HandleImage() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if s.refuseBlockedReferer(w, r) { diff --git a/internal/handlers/jpegxl_default_internal_test.go b/internal/handlers/jpegxl_default_internal_test.go new file mode 100644 index 0000000..1d22bca --- /dev/null +++ b/internal/handlers/jpegxl_default_internal_test.go @@ -0,0 +1,162 @@ +package handlers + +import ( + "fmt" + "log/slog" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/davidbyttow/govips/v2/vips" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/signature" +) + +// requireJPEGXL requires that rec answers 200 with a JPEG XL image. +func requireJPEGXL(t *testing.T, rec *httptest.ResponseRecorder) { + t.Helper() + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body %q", + rec.Code, http.StatusOK, rec.Body.String()) + } + + if got := rec.Header().Get("Content-Type"); got != jxlType { + t.Errorf("Content-Type = %q, want %s", got, jxlType) + } + + if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL { + t.Errorf("body is %s, want jxl", vips.ImageTypes[got]) + } +} + +// TestImageWithoutFormat_ServesJPEGXL verifies that a /v1/image/ URL whose +// last segment is a size with no format, 50x50 or orig, answers JPEG XL. +func TestImageWithoutFormat_ServesJPEGXL(t *testing.T) { + t.Parallel() + + route := newImageRoute(t, newPhotoFetcher(t, allowlistedHost)) + + for _, size := range []string{"50x50", "orig"} { + target := "/v1/image/" + allowlistedHost + photoPath + "/" + size + requireJPEGXL(t, sendGet(t, route, target)) + } +} + +// TestImageWithoutFormat_SignedAsJXL verifies that a /v1/image/ URL with no +// format is signed as jxl: the signature made for the URL ending in .jxl is +// accepted for the same URL without .jxl. +func TestImageWithoutFormat_SignedAsJXL(t *testing.T) { + t.Parallel() + + route := newImageRoute(t, newPhotoFetcher(t, signedHost)) + expires := time.Now().Add(time.Hour) + + sig := signature.New(testSigningKey).Sign(&signature.Request{ + SourceHost: signedHost, + SourcePath: photoPath, + Width: 50, + Height: 50, + Format: string(imgcache.FormatJXL), + Quality: encurl.DefaultQuality, + FitMode: string(imgcache.FitCover), + Expires: expires, + }) + query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix()) + + for _, size := range []string{"50x50.jxl", "50x50"} { + target := "/v1/image/" + signedHost + photoPath + "/" + size + query + requireJPEGXL(t, sendGet(t, route, target)) + } +} + +// TestImageEncWithoutFormat_ServesJPEGXL verifies that an encrypted URL whose +// token holds no format answers JPEG XL. +func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) { + t.Parallel() + + h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + + token, err := h.encGen.Generate(&encurl.Payload{ + SourceHost: signedHost, + SourcePath: photoPath, + Width: 50, + Height: 50, + }) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + requireJPEGXL(t, getEncToken(srv, token)) +} + +// TestGeneratorPage_SelectsJPEGXL verifies that the generator page's format +// choice is JPEG XL until another is chosen. +func TestGeneratorPage_SelectsJPEGXL(t *testing.T) { + t.Parallel() + + h, srv := newCSRFTestRouter(t) + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + req.AddCookie(newSessionCookie(t, h)) + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if !strings.Contains(rec.Body.String(), `
From bdde021b45035e7227ecaef02b211e4a8db59d25 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 14:13:25 +0200 Subject: [PATCH 057/131] Save PNG compressed, WebP at effort 4 and AVIF at effort 1 (closes #232) Each output format now has its own govips export with its settings named, in place of govips' generic Export, which sent libvips a zero for some settings it was not given: PNG had no compression and WebP effort 0. PNG now gets libvips' default compression, 6, and WebP its default effort, 4. GIF and JPEG output is unchanged. AVIF was at libvips' default effort, 4, which takes minutes for an 8192x8192 image with one libvips thread, far past the default downstream_timeout. Effort 1, the lowest govips can set, takes about 51 seconds for an image of random pixels, the worst case, and WebP at 4 about 43. A 16-bit source gets 8 bits per sample, not libvips' 12, which take over four times as long. Model: opus-5-5 --- TODO.md | 15 ++ .../encoder_settings_internal_test.go | 145 ++++++++++++++++++ internal/imageprocessor/imageprocessor.go | 114 ++++++++++---- 3 files changed, 246 insertions(+), 28 deletions(-) create mode 100644 internal/imageprocessor/encoder_settings_internal_test.go diff --git a/TODO.md b/TODO.md index 4bea044..7bee655 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,21 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-08 every output format is saved with settings pixa sets on purpose + (closes #232): each format has its own govips export, as JPEG XL does, in + place of govips' generic `Export`, which sent libvips a zero for some settings + it was not given. PNG gets libvips' default compression, 6, where it had none, + and WebP libvips' default effort, 4, where it had 0. GIF was already at + libvips' default effort, 7, and JPEG output is unchanged. AVIF is saved at + effort 1, the lowest govips can set, where it had libvips' default, 4. With + one libvips thread, an 8192x8192 image of random pixels, the worst case, takes + about 51 seconds as AVIF at effort 1 and 43 as WebP at effort 4, against the + default `downstream_timeout` of 60 seconds. On an image of milder noise, which + AVIF at effort 1 saves in about 12 seconds, effort 4 takes minutes. AVIF is + also saved with 8 bits per sample from a 16-bit source, which libvips would + save with 12: a 16-bit 8192x8192 image of milder noise takes about 54 seconds + at effort 1 with 12 bits, nearly all of the default `downstream_timeout`, and + about 12 with 8. - 2026-10-08 JPEG XL is the default output (closes #222): a `/v1/image/` URL whose last segment is a size with no format, such as `800x600` or `orig`, is served as JPEG XL and signed as `jxl`, so it has the signature of the same URL diff --git a/internal/imageprocessor/encoder_settings_internal_test.go b/internal/imageprocessor/encoder_settings_internal_test.go new file mode 100644 index 0000000..90634ce --- /dev/null +++ b/internal/imageprocessor/encoder_settings_internal_test.go @@ -0,0 +1,145 @@ +package imageprocessor + +import ( + "bytes" + "image" + "image/color" + "image/png" + "testing" + + "github.com/davidbyttow/govips/v2/vips" +) + +// processedSize runs input through Process and returns the output's size in +// bytes. +func processedSize(t *testing.T, input []byte, req *Request) int64 { + t.Helper() + + result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), req) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + _ = result.Content.Close() + + return result.ContentLength +} + +// encodePNG encodes img as PNG with Go's encoder. +func encodePNG(t *testing.T, img image.Image) []byte { + t.Helper() + + var buf bytes.Buffer + + err := png.Encode(&buf, img) + if err != nil { + t.Fatalf("failed to encode test PNG: %v", err) + } + + return buf.Bytes() +} + +// decode decodes input with vips, as Process does. +func decode(t *testing.T, input []byte) *vips.ImageRef { + t.Helper() + + img, err := vips.NewImageFromBuffer(input) + if err != nil { + t.Fatalf("failed to decode input: %v", err) + } + + t.Cleanup(img.Close) + + return img +} + +// TestImageProcessor_PNGIsCompressed verifies that a PNG output is +// compressed: a flat 200x150 image, 90,000 bytes of raw pixels, comes out at +// a small fraction of that. +func TestImageProcessor_PNGIsCompressed(t *testing.T) { + t.Parallel() + + const width, height = 200, 150 + + flat := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + flat.Set(x, y, color.RGBA{R: 40, G: 120, B: 200, A: 255}) + } + } + + size := processedSize(t, encodePNG(t, flat), &Request{Format: FormatPNG}) + + const rawBytes = width * height * 3 + if size > rawBytes/10 { + t.Errorf("PNG output is %d bytes, want under a tenth of its %d raw", + size, rawBytes) + } +} + +// TestImageProcessor_WebPAtDefaultEffort verifies that WebP is saved at +// libvips' default effort, 4: the output is the size of the same image saved +// at that effort. +func TestImageProcessor_WebPAtDefaultEffort(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, 200, 150) + + size := processedSize(t, input, &Request{Format: FormatWebP, Quality: 85}) + + want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{ + StripMetadata: true, + Quality: 85, + ReductionEffort: 4, + }) + if err != nil { + t.Fatalf("ExportWebp() error = %v", err) + } + + if size != int64(len(want)) { + t.Errorf("WebP output is %d bytes, want %d, the size at effort 4", + size, len(want)) + } +} + +// TestImageProcessor_AVIFAtEffort1 verifies that AVIF is saved at effort 1 +// with 8 bits per sample: the output is the size of the same image saved +// with those settings. The source is 16-bit, which libvips saves with 12 +// bits when it is not given a bit depth, and 640x480: on a small image, +// such as 64x48, efforts 1 and 2 give the same output. +func TestImageProcessor_AVIFAtEffort1(t *testing.T) { + t.Parallel() + + const width, height = 640, 480 + + source := image.NewRGBA64(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + source.Set(x, y, color.RGBA64{ + R: uint16((x * 65535 / width) & 0xffff), + G: uint16((y * 65535 / height) & 0xffff), + B: 32768, + A: 65535, + }) + } + } + + input := encodePNG(t, source) + + size := processedSize(t, input, &Request{Format: FormatAVIF, Quality: 85}) + + want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{ + StripMetadata: true, + Quality: 85, + Effort: 1, + Bitdepth: 8, + }) + if err != nil { + t.Fatalf("ExportAvif() error = %v", err) + } + + if size != int64(len(want)) { + t.Errorf("AVIF output is %d bytes, want %d, the size at effort 1 "+ + "and 8 bits", size, len(want)) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 7cee229..6d59b34 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -504,36 +504,21 @@ func (p *ImageProcessor) encode( } } - var params vips.ExportParams - switch format { case FormatJPEG: - params = vips.ExportParams{ - Format: vips.ImageTypeJPEG, - Quality: quality, - } + return exportJPEG(img, quality) case FormatPNG: - params = vips.ExportParams{ - Format: vips.ImageTypePNG, - } + return exportPNG(img) case FormatGIF: - params = vips.ExportParams{ - Format: vips.ImageTypeGIF, - } + return exportGIF(img) case FormatWebP: - params = vips.ExportParams{ - Format: vips.ImageTypeWEBP, - Quality: quality, - } + return exportWebP(img, quality) case FormatAVIF: - params = vips.ExportParams{ - Format: vips.ImageTypeAVIF, - Quality: quality, - } + return exportAVIF(img, quality) case FormatJXL: return exportJXL(img, quality) @@ -544,17 +529,90 @@ func (p *ImageProcessor) encode( default: return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format) } +} - // Drop EXIF, XMP, IPTC and the ICC profile. govips ignores this for - // GIF, which carries none of them. - params.StripMetadata = true +// govips sends libvips Go's zero value for some settings an export leaves +// out, such as no compression at all for PNG, so each export below sets +// every setting whose zero value is not what pixa wants. Stripping metadata +// drops EXIF, XMP, IPTC and the ICC profile. - output, _, err := img.Export(¶ms) - if err != nil { - return nil, err - } +// exportJPEG encodes img as JPEG at quality, without metadata. The settings +// it leaves out are at libvips' defaults. +func exportJPEG(img *vips.ImageRef, quality int) ([]byte, error) { + output, _, err := img.ExportJpeg(&vips.JpegExportParams{ + StripMetadata: true, + Quality: quality, + }) - return output, nil + return output, err +} + +// pngCompression is libvips' default PNG compression, from 0 (none) to 9. +const pngCompression = 6 + +// exportPNG encodes img as PNG at libvips' default compression and row +// filter, without metadata. +func exportPNG(img *vips.ImageRef) ([]byte, error) { + output, _, err := img.ExportPng(&vips.PngExportParams{ + StripMetadata: true, + Compression: pngCompression, + Filter: vips.PngFilterNone, + }) + + return output, err +} + +// gifEffort is libvips' default GIF effort, from 1 to 10. +const gifEffort = 7 + +// exportGIF encodes img as GIF at libvips' default effort. govips cannot +// have libvips strip metadata from GIF, which carries none. +func exportGIF(img *vips.ImageRef) ([]byte, error) { + output, _, err := img.ExportGIF(&vips.GifExportParams{Effort: gifEffort}) + + return output, err +} + +// webpEffort is libvips' default WebP effort, from 0 (fastest) to 6. +const webpEffort = 4 + +// exportWebP encodes img as lossy WebP at quality and libvips' default +// effort, without metadata. +func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) { + output, _, err := img.ExportWebp(&vips.WebpExportParams{ + StripMetadata: true, + Quality: quality, + ReductionEffort: webpEffort, + }) + + return output, err +} + +// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest +// govips can set. With one thread, as pixad runs libvips, 1 takes about 51 +// seconds to save an 8192x8192 image of random pixels, the worst case, +// against the default downstream_timeout of 60 seconds. On an image of +// milder noise, which 1 saves in about 12 seconds, 2 takes nearly a minute +// and libvips' default, 4, takes minutes. +const avifEffort = 1 + +// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image. +// libvips would save a 16-bit image with 12, but at avifEffort that takes +// about 54 seconds for a 16-bit 8192x8192 image of milder noise, nearly all +// of the default downstream_timeout, and about 12 seconds with 8. +const avifBitdepth = 8 + +// exportAVIF encodes img as lossy AVIF at quality, avifEffort and +// avifBitdepth, without metadata. +func exportAVIF(img *vips.ImageRef, quality int) ([]byte, error) { + output, _, err := img.ExportAvif(&vips.AvifExportParams{ + StripMetadata: true, + Quality: quality, + Effort: avifEffort, + Bitdepth: avifBitdepth, + }) + + return output, err } // jxlResolution is the resolution every JPEG XL image is saved with, in From 40ad39843b345323852ec593ea2720f528c9f804 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 01:36:18 +0200 Subject: [PATCH 058/131] Limit each client's requests to the image routes (closes #236) The new setting image_requests_per_minute (PIXA_IMAGE_REQUESTS_PER_MINUTE), default 600, caps the GET and HEAD requests one client sends to /v1/image/ and /v1/e/ per minute; the next is answered 429 with Retry-After and the image routes' JSON error body. It uses the login limit's RateLimit middleware, so clients are told apart the same way, behind the CORS middleware, so a preflight is not counted. 0 turns it off, as script/loadtest does; a negative or non-integer value stops startup. TestEnvironmentSetsEveryKey gains the new variable, under the standing approval in https://git.eeqj.de/sneak/pixa/issues/64. To stay within the linters' limits, the sentry_dsn check moves unchanged into validateSentryDSN, and the cache_max_bytes explicit flag is set in one line. Model: opus-5-5 --- README.md | 51 +++-- TODO.md | 12 +- configs/config.example.yml | 8 + internal/config/config.go | 49 ++++- internal/config/env_internal_test.go | 2 + ...image_requests_per_minute_internal_test.go | 107 +++++++++ internal/handlers/image.go | 10 + internal/middleware/middleware.go | 8 +- .../server/image_rate_limit_internal_test.go | 204 ++++++++++++++++++ internal/server/routes.go | 11 +- script/loadtest | 4 +- 11 files changed, 431 insertions(+), 35 deletions(-) create mode 100644 internal/config/image_requests_per_minute_internal_test.go create mode 100644 internal/server/image_rate_limit_internal_test.go diff --git a/README.md b/README.md index 8b6b032..2dac533 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ everything in this list without further settings. The reverse proxy must: builds encrypted URLs from `Host`, and checks `Referer` against `referer_blocklist`; - set `X-Forwarded-For` to the client's address, with `trusted_proxies` set to - the address pixa sees the proxy's requests come from, so the login limit - counts each client by its own address (see `trusted_proxies` under - Configuration); + the address pixa sees the proxy's requests come from, so the login limit and + `image_requests_per_minute` count each client by its own address (see + `trusted_proxies` under Configuration); - wait for pixa's answer for at least `downstream_timeout` (default `60s`), the longest pixa takes to fetch, convert and send an image. @@ -58,8 +58,9 @@ password there. pixa does the rest itself: it checks signatures and encrypted URLs, applies the allowlist, refuses upstream hosts with private or local addresses, limits login -attempts, upstream response size and image dimensions, and sends the security -headers, `Strict-Transport-Security` included, with every response. +attempts, each client's image requests, upstream response size and image +dimensions, and sends the security headers, `Strict-Transport-Security` +included, with every response. The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the database and the disk cache: @@ -186,10 +187,10 @@ path under `/v1/` answers 200, in maintenance mode too. `Referer` names a host in `referer_blocklist`, checked before the signature, the cache and the upstream fetch; 403 when the upstream host, or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an - address in a blocked network (see `blocked_networks`); 502 when the upstream - answered with an error status, and for 5 minutes after that for the same - source URL; 503 when pixa is busy or in maintenance mode; 500 for any other - failure. + address in a blocked network (see `blocked_networks`); 429 over + `image_requests_per_minute` (below); 502 when the upstream answered with an + error status, and for 5 minutes after that for the same source URL; 503 when + pixa is busy or in maintenance mode; 500 for any other failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when `If-None-Match` matches the image's `ETag`; 400 for a token that does not @@ -197,7 +198,7 @@ path under `/v1/` answers 200, in maintenance mode too. expired; 504 when the upstream has not sent its response headers within `upstream_fetch_timeout`, but 500 when that time runs out while the image itself is still arriving; 400 for an `Accept` header that is not valid, and - 406, 403, 502, 503 and 500, as for `/v1/image/`. + 406, 403, 429, 502, 503 and 500, as for `/v1/image/`. - `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: nothing. Answers: 200. - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, @@ -313,6 +314,15 @@ its own `X-Forwarded-For`, whether it connects directly or through the proxy, because its own address is trusted too. Setting `trusted_proxies` to only the address pixa sees for requests that come through the proxy closes this. +One client may send `image_requests_per_minute` requests per minute +(default 600) to the image routes, `/v1/image/` and `/v1/e/` together, `GET` and +`HEAD`, cache hits included; a request over the limit is refused with 429 and a +`Retry-After` header. Clients are counted as for the login limit. A browser's +CORS preflight is not counted. Behind a CDN or proxy whose addresses are not in +`trusted_proxies`, all requests arriving from one proxy or CDN address count as +one client, so such a deployment needs a higher limit, or `0`, which turns the +limit off. + ### Encrypted URLs An encrypted URL is an image URL made on pixa's own web page by someone who @@ -482,6 +492,7 @@ tell. With no file, pixa uses the environment and the defaults. | `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` | | `PIXA_UPSTREAM_CONNECTIONS` | `upstream_connections` | Concurrent connections to all upstream hosts together; default `64` | | `PIXA_MAX_CONCURRENT_PROCESSING` | `max_concurrent_processing` | Images processed at once; default the number of CPUs | +| `PIXA_IMAGE_REQUESTS_PER_MINUTE` | `image_requests_per_minute` | Image requests per client per minute; `0` turns it off; default `600` | | `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` | | `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB | | `PIXA_DOWNSTREAM_TIMEOUT` | `downstream_timeout` | Time allowed for answering one client request; default `60s` | @@ -531,13 +542,13 @@ Key settings in more detail: explicit list replaces the default. An invalid CIDR aborts startup. Set this to the address pixa sees for requests that come through your proxy, such as `172.17.0.1/32`, when the defaults do not cover it, or to trust nothing else - (see the login limit under Routes). For a proxy on the Docker host that - connects to pixa over `127.0.0.1`, that address is the gateway of the - container's Docker network (`172.17.0.1` on the default bridge), not the - proxy's own address; a proxy that connects through another of the host's - addresses is seen with that address. To be sure which address it is, set this - to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request through the - proxy, and read `remoteIP` in pixa's request log line for it + (see the login limit and `image_requests_per_minute` under Routes). For a + proxy on the Docker host that connects to pixa over `127.0.0.1`, that address + is the gateway of the container's Docker network (`172.17.0.1` on the default + bridge), not the proxy's own address; a proxy that connects through another of + the host's addresses is seen with that address. To be sure which address it + is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request + through the proxy, and read `remoteIP` in pixa's request log line for it - `upstream_fetch_timeout` — time allowed for one fetch from an upstream host, as a duration such as `30s` (the default) or `2m` - `upstream_max_response_size` — largest upstream response accepted, in bytes; @@ -570,6 +581,12 @@ Key settings in more detail: container's CPU limit. A request that finds all of them in use waits up to 10 seconds for one to free up; if none does, and `downstream_timeout` has not ended first, it is answered 503 the same way +- `image_requests_per_minute` — how many requests one client may send to the + image routes per minute (see the limit on them under Routes); default `600`. + `0` turns the limit off, and a negative or non-integer value aborts startup. + Behind a CDN or proxy whose addresses are not in `trusted_proxies`, all + requests arriving from one proxy or CDN address count as one client, so raise + the limit there, or set `0` - `maintenance_mode` — while `true`, the image routes (`/v1/image/` and `/v1/e/`) answer every request for an image with 503, a `Retry-After` header and a JSON error body. The health check (`/.well-known/healthcheck.json`) diff --git a/TODO.md b/TODO.md index 7bee655..561b24e 100644 --- a/TODO.md +++ b/TODO.md @@ -26,10 +26,18 @@ resolved. The disk cache is now size-bounded with LRU eviction # Next Step -P2: security: per-IP rate limiting on the image routes +P2: security: per-origin rate limiting # Completed Steps +- 2026-10-12 per-client limit on the image routes (closes #236): the new setting + `image_requests_per_minute` (`PIXA_IMAGE_REQUESTS_PER_MINUTE`), default 600, + is how many `GET` and `HEAD` requests one client may send to `/v1/image/` and + `/v1/e/` together per minute, cache hits included; the next is answered 429 + with `Retry-After`. It uses the login limit's `RateLimit` middleware, so + clients are told apart the same way, and runs after the CORS middleware, so a + preflight is not counted. `0` turns the limit off; a negative or non-integer + value aborts startup. - 2026-10-08 every output format is saved with settings pixa sets on purpose (closes #232): each format has its own govips export, as JPEG XL does, in place of govips' generic `Export`, which sent libvips a zero for some settings @@ -736,8 +744,6 @@ P2: security: per-IP rate limiting on the image routes # Future Steps -- P2: security - - per-origin rate limiting - P2: HTTP response handling - Last-Modified headers - P2: configuration diff --git a/configs/config.example.yml b/configs/config.example.yml index cede448..2c59549 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -115,6 +115,14 @@ upstream_connections: 64 # frees up it is answered 503, unless downstream_timeout has ended first. # max_concurrent_processing: 4 +# How many requests to the image routes (/v1/image/ and /v1/e/) one client +# may send per minute; a request over the limit is answered 429 with a +# Retry-After header. 0 turns the limit off. Clients are told apart as for +# the login limit, so behind a CDN or proxy whose addresses are not in +# trusted_proxies all requests arriving from one proxy or CDN address count +# as one client: raise the limit there, or set 0. (default: 600) +image_requests_per_minute: 600 + # Time allowed for one fetch from an upstream host (default: 30s) upstream_fetch_timeout: 30s diff --git a/internal/config/config.go b/internal/config/config.go index f4e0fc4..d4b78a2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,6 +34,7 @@ const ( DefaultUpstreamFetchTimeout = 30 * time.Second DefaultUpstreamMaxResponseSize = 50 << 20 // 50 MiB DefaultDownstreamTimeout = 60 * time.Second + DefaultImageRequestsPerMinute = 600 ) // Configuration key names. @@ -61,6 +62,7 @@ const ( keyUpstreamFetchTimeout = "upstream_fetch_timeout" keyUpstreamMaxResponseSize = "upstream_max_response_size" keyDownstreamTimeout = "downstream_timeout" + keyImageRequestsPerMinute = "image_requests_per_minute" ) // placeholderSigningKey is the dummy signing_key shipped in @@ -158,6 +160,11 @@ type Config struct { // of the per-request timeout middleware. DownstreamTimeout time.Duration + // ImageRequestsPerMinute is how many requests one client may make to + // the image routes per minute; the next is refused with 429 Too Many + // Requests. 0 turns the limit off. + ImageRequestsPerMinute int + // BlockedNetworks are operator-supplied CIDR ranges to refuse in // addition to the built-in SSRF blocklist. Enforced by the upstream // fetcher's dialer; the built-in ranges always apply. @@ -302,6 +309,8 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin), DownstreamTimeout: loader.durationVal( keyDownstreamTimeout, DefaultDownstreamTimeout), + ImageRequestsPerMinute: loader.intVal( + keyImageRequestsPerMinute, DefaultImageRequestsPerMinute), CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0), BlockedNetworks: blockedNetworks, TrustedProxies: trustedProxies, @@ -311,9 +320,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { // The default for an omitted cache_max_bytes is worked out when // the cache opens; here we only record whether the operator set // the key explicitly. - if _, present := lookupValue(sc, keyCacheMaxBytes); present { - c.CacheMaxBytesExplicit = true - } + _, c.CacheMaxBytesExplicit = lookupValue(sc, keyCacheMaxBytes) // Build DBURL from StateDir if not explicitly set. The derived URL // is a default: it applies only when db_url is omitted, never to an @@ -426,7 +433,7 @@ func isKnownConfigKey(key string) bool { keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks, keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout, keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist, - "env": + keyImageRequestsPerMinute, "env": return true } @@ -461,6 +468,7 @@ func envVarNames() map[string]string { keyUpstreamFetchTimeout: "PIXA_UPSTREAM_FETCH_TIMEOUT", keyUpstreamMaxResponseSize: "PIXA_UPSTREAM_MAX_RESPONSE_SIZE", keyDownstreamTimeout: "PIXA_DOWNSTREAM_TIMEOUT", + keyImageRequestsPerMinute: "PIXA_IMAGE_REQUESTS_PER_MINUTE", } } @@ -613,6 +621,14 @@ func (c *Config) validate() error { settingName(keyCacheMaxBytes), c.CacheMaxBytes, errMustNotBeNegative) } + // Zero is valid (it turns the limit off); only negative values are + // rejected. + if c.ImageRequestsPerMinute < 0 { + return fmt.Errorf("%s: value %d %w", + settingName(keyImageRequestsPerMinute), c.ImageRequestsPerMinute, + errMustNotBeNegative) + } + err = c.validateUpstreamMaxResponseSize() if err != nil { return err @@ -625,12 +641,9 @@ func (c *Config) validate() error { } } - if c.SentryDSN != "" { - parsed, err := url.Parse(c.SentryDSN) - if err != nil || parsed.Scheme == "" || parsed.Host == "" { - return fmt.Errorf("%s: value %q is %w", - settingName(keySentryDSN), c.SentryDSN, errNotAValidURL) - } + err = c.validateSentryDSN() + if err != nil { + return err } if (c.MetricsUsername == "") != (c.MetricsPassword == "") { @@ -642,6 +655,22 @@ func (c *Config) validate() error { return c.validateAccessControlAllowOrigin() } +// validateSentryDSN checks that sentry_dsn, when set, is a URL with a scheme +// and a host. +func (c *Config) validateSentryDSN() error { + if c.SentryDSN == "" { + return nil + } + + parsed, err := url.Parse(c.SentryDSN) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("%s: value %q is %w", + settingName(keySentryDSN), c.SentryDSN, errNotAValidURL) + } + + return nil +} + // validateUpstreamMaxResponseSize checks that upstream_max_response_size // is from 1 byte to 1 GiB. An upstream response is read whole into // memory, and the image processor reads one byte past this limit, which diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go index 6eb79d9..4b23302 100644 --- a/internal/config/env_internal_test.go +++ b/internal/config/env_internal_test.go @@ -70,6 +70,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5") t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10") t.Setenv("PIXA_MAX_CONCURRENT_PROCESSING", "3") + t.Setenv("PIXA_IMAGE_REQUESTS_PER_MINUTE", "30") t.Setenv("PIXA_CACHE_MAX_BYTES", "1024") t.Setenv("PIXA_BLOCKED_NETWORKS", "203.0.113.0/24") t.Setenv("PIXA_TRUSTED_PROXIES", "192.0.2.0/24") @@ -99,6 +100,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) { UpstreamConnectionsPerHost: 5, UpstreamConnections: 10, MaxConcurrentProcessing: 3, + ImageRequestsPerMinute: 30, CacheMaxBytes: 1024, CacheMaxBytesExplicit: true, BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, diff --git a/internal/config/image_requests_per_minute_internal_test.go b/internal/config/image_requests_per_minute_internal_test.go new file mode 100644 index 0000000..12f63ab --- /dev/null +++ b/internal/config/image_requests_per_minute_internal_test.go @@ -0,0 +1,107 @@ +package config + +import "testing" + +// testImageRequestsPerMinuteVar is the variable that sets +// image_requests_per_minute. +const testImageRequestsPerMinuteVar = "PIXA_IMAGE_REQUESTS_PER_MINUTE" + +// TestImageRequestsPerMinuteValues checks that an omitted +// image_requests_per_minute is 600, and that 0, which turns the limit off, +// and any larger whole number are used as given. +func TestImageRequestsPerMinuteValues(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine) + if err != nil { + t.Fatalf("minimal config should be valid, got error: %v", err) + } + + if c.ImageRequestsPerMinute != 600 { + t.Errorf("ImageRequestsPerMinute = %d, want 600", c.ImageRequestsPerMinute) + } + + explicit := []struct { + value string + want int + }{ + {"0", 0}, + {"30", 30}, + } + + for _, tc := range explicit { + c, err := configFromYAML(t, + signingKeyLine+"image_requests_per_minute: "+tc.value+"\n") + if err != nil { + t.Fatalf("image_requests_per_minute: %s should load, got error: %v", + tc.value, err) + } + + if c.ImageRequestsPerMinute != tc.want { + t.Errorf("ImageRequestsPerMinute = %d, want %d", + c.ImageRequestsPerMinute, tc.want) + } + } +} + +// TestInvalidImageRequestsPerMinuteAbortsStartup checks that a value that is +// negative, not a whole number, or null aborts startup naming the key and +// the value, and the variable too where the value could have come from it. +func TestInvalidImageRequestsPerMinuteAbortsStartup(t *testing.T) { + t.Parallel() + + key := keyImageRequestsPerMinute + + runAbortCases(t, []abortCase{ + { + name: "negative", + yaml: signingKeyLine + key + ": -1\n", + wantErrSubstrings: []string{ + key, testImageRequestsPerMinuteVar, "value -1", + }, + }, + { + name: "not a number", + yaml: signingKeyLine + key + ": plenty\n", + wantErrSubstrings: []string{ + key, testImageRequestsPerMinuteVar, "plenty", + }, + }, + { + name: "fractional", + yaml: signingKeyLine + key + ": 2.5\n", + wantErrSubstrings: []string{key, "2.5"}, + }, + { + name: "explicit null", + yaml: signingKeyLine + key + ": null\n", + wantErrSubstrings: []string{key, nullValueText}, + }, + }) +} + +// TestImageRequestsPerMinuteFromEnvironment checks that the variable sets the +// limit over the config file, and that an invalid value in it aborts startup +// naming the variable and the value. +func TestImageRequestsPerMinuteFromEnvironment(t *testing.T) { + t.Setenv(testImageRequestsPerMinuteVar, "0") + + c, err := configFromYAML(t, signingKeyLine+"image_requests_per_minute: 30\n") + if err != nil { + t.Fatalf("limit from the environment should load: %v", err) + } + + if c.ImageRequestsPerMinute != 0 { + t.Errorf("ImageRequestsPerMinute = %d, want 0 from the environment", + c.ImageRequestsPerMinute) + } + + for _, value := range []string{"-5", "ten"} { + t.Run(value, func(t *testing.T) { + t.Setenv(testImageRequestsPerMinuteVar, value) + + _, err := configFromYAML(t, signingKeyLine) + wantStartupError(t, err, testImageRequestsPerMinuteVar, value) + }) + } +} diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 1aee1cc..848a252 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -206,6 +206,16 @@ func parseExpires(query url.Values) (time.Time, error) { return time.Unix(exp, 0), nil } +// HandleTooManyRequests answers 429 Too Many Requests with the JSON error +// body the image routes send for their other errors. The image routes' rate +// limit calls it for a request over the limit. +func (s *Handlers) HandleTooManyRequests() http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + s.respondError(w, "too many requests, try again later", + http.StatusTooManyRequests) + } +} + // respondImageError maps image retrieval errors to HTTP responses. func (s *Handlers) respondImageError( w http.ResponseWriter, req *imgcache.ImageRequest, err error, diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index aeddfd8..f6eb83d 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -98,9 +98,10 @@ func (s *Middleware) ClientIP() func(http.Handler) http.Handler { // holds whole; an IPv4-mapped address (::ffff:a.b.c.d) is counted as the // IPv4 address it carries, since every such address falls in the same /64. // Counts are kept only for the current and the previous window, so memory -// stays bounded. +// stays bounded. limitHandler writes the 429 after Retry-After is set; nil +// keeps the rate limit library's plain-text "Too Many Requests". func (s *Middleware) RateLimit( - requestLimit int, window time.Duration, + requestLimit int, window time.Duration, limitHandler http.HandlerFunc, ) func(http.Handler) http.Handler { return httprate.LimitBy(requestLimit, window, func(r *http.Request) (string, error) { @@ -112,7 +113,8 @@ func (s *Middleware) RateLimit( } return httprate.CanonicalizeIP(ip), nil - }) + }, + httprate.WithLimitHandler(limitHandler)) } // requestIDPattern is what a request's own X-Request-Id must look like to be diff --git a/internal/server/image_rate_limit_internal_test.go b/internal/server/image_rate_limit_internal_test.go new file mode 100644 index 0000000..fb6bfb7 --- /dev/null +++ b/internal/server/image_rate_limit_internal_test.go @@ -0,0 +1,204 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "sneak.berlin/go/pixa/internal/config" +) + +// requestImageFrom sends method path to s from remoteAddr and returns the +// response. The image handlers refuse an unsigned image URL with 401 and a +// token they cannot decrypt with 400, so either status shows the limit let +// the request through. +func requestImageFrom( + t *testing.T, s *Server, method, path, remoteAddr string, +) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), method, path, nil) + req.RemoteAddr = remoteAddr + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + return rec +} + +// TestImageRequestLimitRefusesRequestOverLimit verifies that GET and HEAD on +// both image routes count toward one limit per client, and that the request +// after image_requests_per_minute of them is refused with 429 and a +// Retry-After header. +func TestImageRequestLimitRefusesRequestOverLimit(t *testing.T) { + t.Parallel() + + requests := []struct { + method string + path string + want int + }{ + {http.MethodGet, unsignedImagePath, http.StatusUnauthorized}, + {http.MethodHead, unsignedImagePath, http.StatusUnauthorized}, + {http.MethodGet, encryptedImagePath, http.StatusBadRequest}, + } + + s := newTestServer(t) + s.config.ImageRequestsPerMinute = len(requests) + s.SetupRoutes() + + for _, tc := range requests { + rec := requestImageFrom(t, s, tc.method, tc.path, firstClient) + if rec.Code != tc.want { + t.Fatalf("%s %s within the limit status = %d, want %d", + tc.method, tc.path, rec.Code, tc.want) + } + } + + rec := requestImageFrom(t, s, http.MethodHead, encryptedImagePath, firstClient) + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("request over the limit status = %d, want %d", + rec.Code, http.StatusTooManyRequests) + } + + retryAfter := rec.Header().Get("Retry-After") + + seconds, err := strconv.Atoi(retryAfter) + if err != nil || seconds <= 0 { + t.Errorf("Retry-After = %q, want a positive number of seconds", + retryAfter) + } +} + +// TestImageRequestLimitAnswersWithJSONError verifies that the 429 for a +// request over the limit carries Retry-After and the JSON error body the +// image routes send for their other errors. +func TestImageRequestLimitAnswersWithJSONError(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.ImageRequestsPerMinute = 1 + s.SetupRoutes() + + requestImageFrom(t, s, http.MethodGet, unsignedImagePath, firstClient) + + rec := requestImageFrom(t, s, http.MethodGet, unsignedImagePath, firstClient) + t.Logf("status %d, body %s", rec.Code, rec.Body.String()) + + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("request over the limit status = %d, want %d", + rec.Code, http.StatusTooManyRequests) + } + + retryAfter := rec.Header().Get("Retry-After") + + seconds, err := strconv.Atoi(retryAfter) + if err != nil || seconds <= 0 { + t.Errorf("Retry-After = %q, want a positive number of seconds", + retryAfter) + } + + contentType := rec.Header().Get("Content-Type") + if contentType != "application/json" { + t.Errorf("Content-Type = %q, want %q", contentType, "application/json") + } + + var body struct { + Error string `json:"error"` + Status int `json:"status"` + Timestamp string `json:"timestamp"` + } + + err = json.NewDecoder(rec.Body).Decode(&body) + if err != nil { + t.Fatalf("body is not JSON: %v", err) + } + + if body.Error == "" || body.Status != http.StatusTooManyRequests || + body.Timestamp == "" { + t.Errorf("body = %+v, want an error, status %d and a timestamp", + body, http.StatusTooManyRequests) + } +} + +// TestImageRequestLimitCountsClientsSeparately verifies that one client going +// over the limit does not limit another. +func TestImageRequestLimitCountsClientsSeparately(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.ImageRequestsPerMinute = 1 + s.SetupRoutes() + + rec := requestImageFrom(t, s, http.MethodGet, unsignedImagePath, firstClient) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("first client's first request status = %d, want %d", + rec.Code, http.StatusUnauthorized) + } + + rec = requestImageFrom(t, s, http.MethodGet, unsignedImagePath, firstClient) + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("first client's second request status = %d, want %d", + rec.Code, http.StatusTooManyRequests) + } + + rec = requestImageFrom(t, s, http.MethodGet, unsignedImagePath, secondClient) + if rec.Code != http.StatusUnauthorized { + t.Errorf("second client's first request status = %d, want %d", + rec.Code, http.StatusUnauthorized) + } +} + +// TestImageRequestLimitZeroLetsEveryRequestThrough verifies that with +// image_requests_per_minute set to 0 no image request is refused with 429, +// not even past the default limit. +func TestImageRequestLimitZeroLetsEveryRequestThrough(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.ImageRequestsPerMinute = 0 + s.SetupRoutes() + + for request := range config.DefaultImageRequestsPerMinute + 1 { + rec := requestImageFrom(t, s, http.MethodGet, unsignedImagePath, firstClient) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("request %d status = %d, want %d", + request+1, rec.Code, http.StatusUnauthorized) + } + } +} + +// TestImageRequestLimitDoesNotCountPreflight verifies that a browser's CORS +// preflight request to an image route is answered without being counted, so +// a client that has sent several still gets its image request through. +func TestImageRequestLimitDoesNotCountPreflight(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + s.config.AccessControlAllowOrigin = config.DefaultAccessControlAllowOrigin + s.config.ImageRequestsPerMinute = 1 + s.SetupRoutes() + + for range 3 { + req := httptest.NewRequestWithContext( + t.Context(), http.MethodOptions, unsignedImagePath, nil) + req.RemoteAddr = firstClient + req.Header.Set("Origin", "https://app.example.com") + req.Header.Set("Access-Control-Request-Method", http.MethodGet) + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("preflight status = %d, want %d", rec.Code, http.StatusOK) + } + } + + rec := requestImageFrom(t, s, http.MethodGet, unsignedImagePath, firstClient) + if rec.Code != http.StatusUnauthorized { + t.Errorf("image request after preflights status = %d, want %d", + rec.Code, http.StatusUnauthorized) + } +} diff --git a/internal/server/routes.go b/internal/server/routes.go index ae2680b..104f2e6 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -66,7 +66,7 @@ func (s *Server) SetupRoutes() { r.Use(s.h.LimitBody(handlers.MaxFormBytes)) r.Use(s.h.CSRF()) r.Get("/", s.h.HandleRoot()) - r.With(s.mw.RateLimit(LoginAttemptsPerMinute, time.Minute)). + r.With(s.mw.RateLimit(LoginAttemptsPerMinute, time.Minute, nil)). Post("/", s.h.HandleRoot()) r.Post("/generate", s.h.HandleGenerateURL()) }) @@ -88,6 +88,15 @@ func (s *Server) SetupRoutes() { r.Group(func(r chi.Router) { r.Use(s.refuseDuringMaintenance) + // Limited per client, as the login POST is, but the 429 has + // the image routes' JSON error body; 0 turns the limit off. + // The CORS middleware answers a preflight before it gets + // here, so a preflight is not counted. + if s.config.ImageRequestsPerMinute > 0 { + r.Use(s.mw.RateLimit(s.config.ImageRequestsPerMinute, + time.Minute, s.h.HandleTooManyRequests())) + } + // Main image proxy route // /v1/image///x., or with no // format /v1/image///x diff --git a/script/loadtest b/script/loadtest index bcc361e..77d8808 100755 --- a/script/loadtest +++ b/script/loadtest @@ -94,7 +94,8 @@ main() { } # start_containers starts a new origin and a new pixad, and waits up to 30 -# seconds for pixad's health check to pass. +# seconds for pixad's health check to pass. pixad sees all of vegeta's +# requests come from one address, so its image request limit is off. start_containers() { docker run -d --name "$run-origin" \ --network "$run" --network-alias origin \ @@ -104,6 +105,7 @@ start_containers() { --network "$run" -p 127.0.0.1::8080 --health-interval=1s \ -e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \ -e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \ + -e PIXA_IMAGE_REQUESTS_PER_MINUTE=0 \ "$image" >/dev/null waited=0 From 162a2a33effc81f68f098b4b098d030b65ba38b9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 03:07:23 +0200 Subject: [PATCH 059/131] Keep counts not written by the request's deadline in memory (closes #224) TestService_Get_ReturnsByItsDeadline failed on a busy host because its request, past its deadline, still waited to write its miss count, a write with no deadline; in pixad that write waits for the one database connection every request shares. Each count write now keeps the request's deadline but not its cancellation. A count not written by then is kept in memory, where Stats includes it, and one goroutine of the cache writes those counts one UPDATE at a time, and once more at shutdown before the database closes. The test phase also runs go test with -parallel 4: on a busy host, as many tests at once as there are CPUs wait so long to be scheduled that a timed request can fail. Model: opus-5-5 --- Dockerfile | 9 +- README.md | 10 +- TODO.md | 13 + internal/handlers/handlers.go | 13 +- internal/imgcache/cache.go | 97 +++++- internal/imgcache/pendingcounts.go | 149 ++++++++ .../imgcache/pendingcounts_internal_test.go | 327 ++++++++++++++++++ internal/imgcache/service.go | 32 +- 8 files changed, 615 insertions(+), 35 deletions(-) create mode 100644 internal/imgcache/pendingcounts.go create mode 100644 internal/imgcache/pendingcounts_internal_test.go diff --git a/Dockerfile b/Dockerfile index 8afc3dd..e82d2d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,10 +32,13 @@ RUN script/bootstrap --cgo COPY . . # Without -v first; on a failure, again with -v for the details, and -# the step fails even if the second run passes. -RUN go test -count=1 -timeout 90s -race -cover ./... || \ +# the step fails even if the second run passes. -parallel 4: by default +# a package runs as many of its tests at once as the host has CPUs, and +# on a busy host they then wait so long to be scheduled that a test +# that times a request can see it return a second late. +RUN go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -count=1 -timeout 90s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; } # Build stage. Nothing is wanted from the two phases above: these copies # make BuildKit build them first, so this stage runs only when lint and diff --git a/README.md b/README.md index 2dac533..e8e8df4 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,12 @@ which answers 200 whenever pixa is running, in maintenance mode too (see `maintenance_mode`). On SIGTERM or SIGINT pixa stops accepting connections, gives the requests in -progress and the images being processed 5 seconds to finish, and exits: with 0, -or with 1 when images were still being processed after those 5 seconds or -another part of pixa failed to stop. A request not finished by then is cut off. -`docker stop` waits 10 seconds before it kills the container. +progress and the images being processed 5 seconds to finish, then writes to the +database the counts of cache hits, misses, fetches and conversions that requests +could not write by their deadline, and exits: with 0, or with 1 when images were +still being processed after those 5 seconds, some of those counts could not be +written, or another part of pixa failed to stop. A request not finished by then +is cut off. `docker stop` waits 10 seconds before it kills the container. Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL diff --git a/TODO.md b/TODO.md index 561b24e..66067f4 100644 --- a/TODO.md +++ b/TODO.md @@ -38,6 +38,19 @@ P2: security: per-origin rate limiting clients are told apart the same way, and runs after the CORS middleware, so a preflight is not counted. `0` turns the limit off; a negative or non-integer value aborts startup. +- 2026-10-08 a request past its deadline no longer waits for the database to + count it (closes #224). On a busy host, `TestService_Get_ReturnsByItsDeadline` + failed because its request, past its deadline, still waited for its miss count + to be written, a write with no deadline, which in pixad waits for the one + database connection every request shares. Each count write now keeps the + request's deadline but not its cancellation. A count not written by then is + kept in memory, where `Stats` sees it, and one goroutine of the cache writes + those counts, one write at a time, and once more at shutdown before the + database closes. The test phase of the `Dockerfile` also runs at most 4 tests + of a package at once (`go test -parallel 4`), where it ran as many as the host + has CPUs: on a busy host they then wait so long to be scheduled that a test + that times a request can fail. `-p`, how many packages are tested at once, is + unchanged, as capping it also slows compiling. - 2026-10-08 every output format is saved with settings pixa sets on purpose (closes #232): each format has its own govips export, as JPEG XL does, in place of govips' generic `Export`, which sent libvips a zero for some settings diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 0414026..6405d08 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -4,6 +4,7 @@ package handlers import ( "context" "encoding/json" + "errors" "log/slog" "net/http" "time" @@ -71,16 +72,21 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { } lc.Append(fx.Hook{ - //nolint:contextcheck // the eviction loop outlives OnStart; OnStop cancels it + //nolint:contextcheck // the cache's goroutines outlive OnStart; OnStop stops them OnStart: func(_ context.Context) error { return s.initImageService() }, + // The pending counts are written here, before the database's stop + // hook closes it. OnStop: func(ctx context.Context) error { if s.imgCache == nil { return nil } - return s.imgCache.StopEviction(ctx) + return errors.Join( + s.imgCache.StopEviction(ctx), + s.imgCache.StopPendingCountWrites(ctx), + ) }, }) @@ -122,6 +128,9 @@ func (s *Handlers) initImageService() error { // write-pressure passes. No-op when the disk cache is disabled. cache.StartEviction(imgcache.DefaultEvictionInterval) + // Writes the counts requests could not write by their deadline + cache.StartPendingCountWrites() + // Create the fetcher config fetcherCfg := httpfetcher.DefaultConfig() fetcherCfg.AllowHTTP = s.config.AllowHTTP diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 00589ab..ff27456 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -11,6 +11,8 @@ import ( "io" "log/slog" "path/filepath" + "sync" + "sync/atomic" "time" lru "github.com/hashicorp/golang-lru/v2" @@ -79,6 +81,27 @@ type Cache struct { evictionDone chan struct{} evictionCancel context.CancelFunc + // The pending counts: hits, misses, upstream fetches and transforms + // whose write to the database missed its request's deadline, kept + // here until they are written by the goroutine that + // StartPendingCountWrites starts. As for eviction, the channels are + // created in NewCache: pendingCountsAdded wakes that goroutine, and + // pendingCountsDone is closed when it returns. pendingCountsCancel, + // set by StartPendingCountWrites, cancels its context, which tells it + // to finish. + pendingHits atomic.Int64 + pendingMisses atomic.Int64 + pendingUpstreamFetches atomic.Int64 + pendingUpstreamFetchBytes atomic.Int64 + pendingTransforms atomic.Int64 + pendingCountsAdded chan struct{} + pendingCountsDone chan struct{} + pendingCountsCancel context.CancelFunc + + // Held through each write of the pending counts, and by + // readHitAndMissCounts to read them + pendingCountsWriteMutex sync.Mutex + // metaCache holds the content types of the variants most recently // stored or served, so a hit does not read the variant's .meta file. // It never stands in for the variant file, which is always opened. @@ -139,6 +162,9 @@ func newCache( metaCache: metaCache, contentLocks: newContentLock(), + pendingCountsAdded: make(chan struct{}, 1), + pendingCountsDone: make(chan struct{}), + reconciliationPageSize: defaultReconciliationPageSize, } @@ -474,12 +500,7 @@ func (c *Cache) CleanExpired(ctx context.Context) error { func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { var stats CacheStats - // Fetch hit/miss counts from the stats table - err := c.db.QueryRowContext(ctx, ` - SELECT hit_count, miss_count - FROM cache_stats WHERE id = 1 - `).Scan(&stats.HitCount, &stats.MissCount) - + err := c.readHitAndMissCounts(ctx, &stats) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("failed to get cache stats: %w", err) } @@ -511,19 +532,30 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { } // IncrementStats counts a cache hit or miss, and an upstream fetch that read -// fetchBytes bytes, as IncrementUpstreamFetch does. +// fetchBytes bytes, as IncrementUpstreamFetch does. Like the other Increment +// methods, it writes to the database with ctx's deadline but not its +// cancellation, so an ended request is still counted but does not wait for +// the database past its deadline; a count not written by then becomes a +// pending count (see StartPendingCountWrites). func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) { + countCtx, cancel := withoutCancelKeepingDeadline(ctx) + defer cancel() + var err error + pendingCount := &c.pendingMisses + if hit { - _, err = c.db.ExecContext(ctx, ` + pendingCount = &c.pendingHits + + _, err = c.db.ExecContext(countCtx, ` UPDATE cache_stats SET hit_count = hit_count + 1, last_updated_at = CURRENT_TIMESTAMP WHERE id = 1 `) } else { - _, err = c.db.ExecContext(ctx, ` + _, err = c.db.ExecContext(countCtx, ` UPDATE cache_stats SET miss_count = miss_count + 1, last_updated_at = CURRENT_TIMESTAMP @@ -531,7 +563,10 @@ func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) `) } - if err != nil { + switch { + case errors.Is(err, context.DeadlineExceeded): + c.addPendingCount(pendingCount, 1) + case err != nil: c.log.Warn("failed to count cache hit or miss", "hit", hit, "error", err) } @@ -545,14 +580,22 @@ func (c *Cache) IncrementUpstreamFetch(ctx context.Context, fetchBytes int64) { return } - _, err := c.db.ExecContext(ctx, ` + countCtx, cancel := withoutCancelKeepingDeadline(ctx) + defer cancel() + + _, err := c.db.ExecContext(countCtx, ` UPDATE cache_stats SET upstream_fetch_count = upstream_fetch_count + 1, upstream_fetch_bytes = upstream_fetch_bytes + ?, last_updated_at = CURRENT_TIMESTAMP WHERE id = 1 `, fetchBytes) - if err != nil { + + switch { + case errors.Is(err, context.DeadlineExceeded): + c.addPendingCount(&c.pendingUpstreamFetches, 1) + c.addPendingCount(&c.pendingUpstreamFetchBytes, fetchBytes) + case err != nil: c.log.Warn("failed to count upstream fetch", "fetch_bytes", fetchBytes, "error", err) } @@ -560,17 +603,43 @@ func (c *Cache) IncrementUpstreamFetch(ctx context.Context, fetchBytes int64) { // IncrementTransformCount counts one image transcoded by the image processor. func (c *Cache) IncrementTransformCount(ctx context.Context) { - _, err := c.db.ExecContext(ctx, ` + countCtx, cancel := withoutCancelKeepingDeadline(ctx) + defer cancel() + + _, err := c.db.ExecContext(countCtx, ` UPDATE cache_stats SET transform_count = transform_count + 1, last_updated_at = CURRENT_TIMESTAMP WHERE id = 1 `) - if err != nil { + + switch { + case errors.Is(err, context.DeadlineExceeded): + c.addPendingCount(&c.pendingTransforms, 1) + case err != nil: c.log.Warn("failed to count transform", "error", err) } } +// readHitAndMissCounts sets the hit and miss counts of stats to those of the +// cache_stats row plus the pending hits and misses (see +// StartPendingCountWrites). It holds pendingCountsWriteMutex, so that no +// write of the pending counts falls between the two reads. +func (c *Cache) readHitAndMissCounts(ctx context.Context, stats *CacheStats) error { + c.pendingCountsWriteMutex.Lock() + defer c.pendingCountsWriteMutex.Unlock() + + err := c.db.QueryRowContext(ctx, ` + SELECT hit_count, miss_count + FROM cache_stats WHERE id = 1 + `).Scan(&stats.HitCount, &stats.MissCount) + + stats.HitCount += c.pendingHits.Load() + stats.MissCount += c.pendingMisses.Load() + + return err +} + // loadVariantWithMeta is GetVariant for a variant metaCache does not // hold: it reads the content type from the variant's .meta file and // keeps it in metaCache, unless a StoreVariant has put one there diff --git a/internal/imgcache/pendingcounts.go b/internal/imgcache/pendingcounts.go new file mode 100644 index 0000000..c967b01 --- /dev/null +++ b/internal/imgcache/pendingcounts.go @@ -0,0 +1,149 @@ +package imgcache + +import ( + "context" + "fmt" + "sync/atomic" +) + +// StartPendingCountWrites starts the goroutine that writes the pending +// counts to the database whenever there are some, one UPDATE at a time: +// however many requests pass their deadline before their counts are +// written, at most this one write waits for the database. It is a no-op +// when already started. The goroutine outlives the caller, so it runs with +// its own context, which StopPendingCountWrites cancels to tell it to +// finish. +func (c *Cache) StartPendingCountWrites() { + if c.pendingCountsCancel != nil { + return + } + + ctx, cancel := context.WithCancel(context.Background()) + c.pendingCountsCancel = cancel + + go func() { + c.pendingCountWriteLoop(ctx) + }() +} + +// StopPendingCountWrites tells the goroutine StartPendingCountWrites +// started, if it did, to finish, and waits for its write under way to end +// and for it to return. It then writes the pending counts left, so that +// they are written before the database closes. It waits for all of this at +// most until ctx ends, then logs the counts not written and returns an +// error. +func (c *Cache) StopPendingCountWrites(ctx context.Context) error { + var err error + + if c.pendingCountsCancel != nil { + c.pendingCountsCancel() + + select { + case <-c.pendingCountsDone: + case <-ctx.Done(): + // The write given up on here may not reach the database: + // database/sql closes without waiting for a query, and a + // write still waiting for the connection fails when the + // database closes. + err = fmt.Errorf("pending counts still being written: %w", ctx.Err()) + } + } + + if err == nil { + err = c.writePendingCounts(ctx) + } + + if err != nil { + c.log.Error("counts not written at shutdown", + "hits", c.pendingHits.Load(), + "misses", c.pendingMisses.Load(), + "upstream_fetches", c.pendingUpstreamFetches.Load(), + "upstream_fetch_bytes", c.pendingUpstreamFetchBytes.Load(), + "transforms", c.pendingTransforms.Load(), + "error", err, + ) + + return err + } + + return nil +} + +// pendingCountWriteLoop is the body of the goroutine +// StartPendingCountWrites starts. Once ctx is cancelled it starts no write +// and returns, but does not cut off a write under way then. A write that +// fails leaves the counts pending, for the next write. +func (c *Cache) pendingCountWriteLoop(ctx context.Context) { + defer close(c.pendingCountsDone) + + for { + select { + case <-ctx.Done(): + return + case <-c.pendingCountsAdded: + } + + // select picks at random when ctx is cancelled and a wakeup + // waits too. + if ctx.Err() != nil { + return + } + + err := c.writePendingCounts(context.WithoutCancel(ctx)) + if err != nil { + c.log.Warn("failed to write pending counts", "error", err) + } + } +} + +// addPendingCount adds n to pendingCount, one of the pending counts, and +// wakes the goroutine that writes them. pendingCountsAdded has capacity one, +// so a wakeup already waiting is enough. +func (c *Cache) addPendingCount(pendingCount *atomic.Int64, n int64) { + pendingCount.Add(n) + + select { + case c.pendingCountsAdded <- struct{}{}: + default: + } +} + +// writePendingCounts adds the pending counts to the cache_stats row in one +// UPDATE, then takes what it wrote out of them, leaving any added +// meanwhile. When the UPDATE fails, they all stay pending. +func (c *Cache) writePendingCounts(ctx context.Context) error { + c.pendingCountsWriteMutex.Lock() + defer c.pendingCountsWriteMutex.Unlock() + + hits := c.pendingHits.Load() + misses := c.pendingMisses.Load() + upstreamFetches := c.pendingUpstreamFetches.Load() + upstreamFetchBytes := c.pendingUpstreamFetchBytes.Load() + transforms := c.pendingTransforms.Load() + + if hits+misses+upstreamFetches+upstreamFetchBytes+transforms == 0 { + return nil + } + + _, err := c.db.ExecContext(ctx, ` + UPDATE cache_stats + SET hit_count = hit_count + ?, + miss_count = miss_count + ?, + upstream_fetch_count = upstream_fetch_count + ?, + upstream_fetch_bytes = upstream_fetch_bytes + ?, + transform_count = transform_count + ?, + last_updated_at = CURRENT_TIMESTAMP + WHERE id = 1 + `, hits, misses, upstreamFetches, upstreamFetchBytes, transforms) + if err != nil { + return fmt.Errorf("failed to write pending counts: %w", err) + } + + c.pendingHits.Add(-hits) + c.pendingMisses.Add(-misses) + c.pendingUpstreamFetches.Add(-upstreamFetches) + c.pendingUpstreamFetchBytes.Add(-upstreamFetchBytes) + c.pendingTransforms.Add(-transforms) + + return nil +} diff --git a/internal/imgcache/pendingcounts_internal_test.go b/internal/imgcache/pendingcounts_internal_test.go new file mode 100644 index 0000000..77dd73e --- /dev/null +++ b/internal/imgcache/pendingcounts_internal_test.go @@ -0,0 +1,327 @@ +package imgcache + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" +) + +// pendingCountsWait is how long a test waits for the database's connection, +// for a count to reach the database, for a write to start waiting for the +// connection, for a request to return past its deadline, or for +// StopPendingCountWrites to return. +const pendingCountsWait = 5 * time.Second + +// otherWritesWait is how long a test gives other writes, once one write waits +// for the database's connection, to start waiting for it too. +const otherWritesWait = 100 * time.Millisecond + +// holdDatabase takes the one connection of cache's database, so that every +// other query waits for it, and returns the func that frees it. +func holdDatabase(t *testing.T, cache *Cache) func() { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), pendingCountsWait) + defer cancel() + + conn, err := cache.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to take the database connection: %v", err) + } + + return func() { _ = conn.Close() } +} + +// waitForConnectionWaits waits until db.Stats().WaitCount, the number of +// times a caller has waited for a connection, reaches waits. +func waitForConnectionWaits(t *testing.T, db *sql.DB, waits int64) { + t.Helper() + + deadline := time.Now().Add(pendingCountsWait) + + for db.Stats().WaitCount < waits { + if time.Now().After(deadline) { + t.Fatalf("callers waited for the database connection %d times, want %d", + db.Stats().WaitCount, waits) + } + + time.Sleep(10 * time.Millisecond) + } +} + +// waitForCounters waits until the cache_stats row holds want. +func waitForCounters(t *testing.T, cache *Cache, want cacheStatsCounters) { + t.Helper() + + deadline := time.Now().Add(pendingCountsWait) + + for { + got := readCacheStatsCounters(t, cache) + if got == want { + return + } + + if time.Now().After(deadline) { + t.Fatalf("counters = %+v, want %+v", got, want) + } + + time.Sleep(10 * time.Millisecond) + } +} + +// TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the +// database's one connection while a request whose fetch is held reaches its +// deadline. The request must return with the deadline's error while the +// connection is still held, as it waits for no database write past its +// deadline, and its miss must reach the database once the connection is free. +func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + svc.cache.StartPendingCountWrites() + defer func() { _ = svc.cache.StopPendingCountWrites(t.Context()) }() + + // Room for the request to reach its fetch on a busy host + const timeout = 2 * time.Second + + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + + deadline, _ := ctx.Deadline() + + results := startGet(ctx, svc, photoVariant(fixtures, 85, FitCover)) + + // The request has made its database reads by the time it fetches. + select { + case <-fetcher.started: + case <-time.After(timeout): + t.Fatal("request did not reach its fetch by its deadline") + } + + releaseDatabase := holdDatabase(t, svc.cache) + defer releaseDatabase() + + select { + case got := <-results: + t.Logf("Get() returned %v after its deadline, error = %v", + time.Since(deadline), got.err) + + if !errors.Is(got.err, context.DeadlineExceeded) { + t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded) + } + case <-time.After(time.Until(deadline) + pendingCountsWait): + t.Fatal("request did not return while the database was busy") + } + + releaseDatabase() + + waitForCounters(t, svc.cache, cacheStatsCounters{missCount: 1}) +} + +// TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting holds the +// database's one connection while many misses are counted past their +// deadline. At most one write may then wait for the connection, and every +// miss must reach the database once it is free. +func TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + cache.StartPendingCountWrites() + defer func() { _ = cache.StopPendingCountWrites(t.Context()) }() + + releaseDatabase := holdDatabase(t, cache) + defer releaseDatabase() + + waitsBefore := cache.db.Stats().WaitCount + + // Writes given a context past its deadline fail at once. + ended, cancel := context.WithDeadline(t.Context(), time.Now()) + defer cancel() + + const misses = 20 + + for range misses { + cache.IncrementStats(ended, false, 0) + } + + // Once one write waits for the connection, any others start waiting too. + waitForConnectionWaits(t, cache.db, waitsBefore+1) + time.Sleep(otherWritesWait) + + waits := cache.db.Stats().WaitCount - waitsBefore + t.Logf("%d writes waited for the database", waits) + + if waits > 1 { + t.Errorf("%d writes waited for the database, want at most 1", waits) + } + + releaseDatabase() + + waitForCounters(t, cache, cacheStatsCounters{missCount: misses}) +} + +// TestStats_IncludesPendingCounts counts hits and misses whose writes miss +// their deadline, and checks that Stats adds them to the database's counts +// before they are written. +func TestStats_IncludesPendingCounts(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + _, err := cache.db.ExecContext(t.Context(), + `UPDATE cache_stats SET hit_count = 75, miss_count = 25 WHERE id = 1`) + if err != nil { + t.Fatal(err) + } + + // Writes given a context past its deadline fail at once. + ended, cancel := context.WithDeadline(t.Context(), time.Now()) + defer cancel() + + cache.IncrementStats(ended, true, 0) + cache.IncrementStats(ended, true, 0) + cache.IncrementStats(ended, false, 0) + + stats, err := cache.Stats(t.Context()) + if err != nil { + t.Fatalf("Stats() error = %v", err) + } + + if stats.HitCount != 77 || stats.MissCount != 26 { + t.Errorf("HitCount = %d, MissCount = %d, want 77 and 26", + stats.HitCount, stats.MissCount) + } + + want := cacheStatsCounters{hitCount: 75, missCount: 25} + if got := readCacheStatsCounters(t, cache); got != want { + t.Errorf("counters in the database = %+v, want %+v", got, want) + } +} + +// TestStopPendingCountWrites_WritesPendingCounts holds the database's one +// connection while the goroutine that writes the pending counts waits for it, +// counts more, then stops that goroutine. The stop must leave the write under +// way to finish rather than cut it off, and once the connection is free it +// must write the counts that write left. +func TestStopPendingCountWrites_WritesPendingCounts(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + cache.StartPendingCountWrites() + defer func() { _ = cache.StopPendingCountWrites(t.Context()) }() + + releaseDatabase := holdDatabase(t, cache) + defer releaseDatabase() + + waitsBefore := cache.db.Stats().WaitCount + + // Writes given a context past its deadline fail at once. + ended, cancel := context.WithDeadline(t.Context(), time.Now()) + defer cancel() + + cache.IncrementStats(ended, true, 0) + + // The goroutine's write waits for the connection. + waitForConnectionWaits(t, cache.db, waitsBefore+1) + + // Counted after that write read the pending counts + cache.IncrementStats(ended, false, 0) + cache.IncrementUpstreamFetch(ended, 1024) + cache.IncrementTransformCount(ended) + + // The goroutine is told to finish here as well as by the stop, so that it + // is told before its write under way ends, however late the stop runs. + // The counts above then reach the database only through the stop's own + // write. + cache.pendingCountsCancel() + + stopped := make(chan error, 1) + + go func() { + stopped <- cache.StopPendingCountWrites(t.Context()) + }() + + // Had the stop cut off the write under way, its own write would wait for + // the connection too. + time.Sleep(otherWritesWait) + + waits := cache.db.Stats().WaitCount - waitsBefore + t.Logf("%d writes waited for the database", waits) + + if waits > 1 { + t.Errorf("%d writes waited for the database, want 1: "+ + "the stop cut off the write under way", waits) + } + + releaseDatabase() + + select { + case err := <-stopped: + if err != nil { + t.Fatalf("StopPendingCountWrites() error = %v", err) + } + case <-time.After(pendingCountsWait): + t.Fatal("StopPendingCountWrites() did not return once the database was free") + } + + want := cacheStatsCounters{1, 1, 1, 1024, 1} + if got := readCacheStatsCounters(t, cache); got != want { + t.Errorf("counters = %+v, want %+v", got, want) + } +} + +// TestStopPendingCountWrites_ReturnsWhenItsContextEnds holds the database's +// one connection while the goroutine that writes the pending counts waits for +// it, then stops that goroutine with a context that ends first. The stop must +// return the context's error, and the write under way must still reach the +// database once the connection is free. +func TestStopPendingCountWrites_ReturnsWhenItsContextEnds(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + cache.StartPendingCountWrites() + defer func() { _ = cache.StopPendingCountWrites(t.Context()) }() + + releaseDatabase := holdDatabase(t, cache) + defer releaseDatabase() + + waitsBefore := cache.db.Stats().WaitCount + + // Writes given a context past its deadline fail at once. + ended, cancel := context.WithDeadline(t.Context(), time.Now()) + defer cancel() + + cache.IncrementStats(ended, true, 0) + + // The goroutine's write waits for the connection. + waitForConnectionWaits(t, cache.db, waitsBefore+1) + + stopCtx, cancelStop := context.WithCancel(t.Context()) + cancelStop() + + stopped := make(chan error, 1) + + go func() { + stopped <- cache.StopPendingCountWrites(stopCtx) + }() + + select { + case err := <-stopped: + if !errors.Is(err, context.Canceled) { + t.Errorf("StopPendingCountWrites() error = %v, want %v", + err, context.Canceled) + } + case <-time.After(pendingCountsWait): + t.Fatal("StopPendingCountWrites() did not return once its context ended") + } + + releaseDatabase() + + waitForCounters(t, cache, cacheStatsCounters{hitCount: 1}) +} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index f196b26..52684a9 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -162,7 +162,7 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e // Fall through to re-process } else { // Counted also when the request context has ended meanwhile - s.cache.IncrementStats(context.WithoutCancel(ctx), true, 0) + s.cache.IncrementStats(ctx, true, 0) return &ImageResponse{ Content: reader, @@ -179,7 +179,7 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e // failed or the request context has ended meanwhile response, err := s.processOrWait(ctx, req) - s.cache.IncrementStats(context.WithoutCancel(ctx), false, 0) + s.cache.IncrementStats(ctx, false, 0) if err != nil { return nil, err @@ -300,14 +300,8 @@ func (s *Service) processOrWait( } }() - processingCtx := context.WithoutCancel(ctx) - - if deadline, ok := ctx.Deadline(); ok { - var cancel context.CancelFunc - - processingCtx, cancel = context.WithDeadline(processingCtx, deadline) - defer cancel() - } + processingCtx, cancel := withoutCancelKeepingDeadline(ctx) + defer cancel() return s.processFromSourceOrFetch(processingCtx, req, cacheKey) }) @@ -344,6 +338,20 @@ func (s *Service) processOrWait( }, nil } +// withoutCancelKeepingDeadline returns ctx without its cancellation but with +// its deadline, if it has one, and the func that releases the returned +// context. +func withoutCancelKeepingDeadline( + ctx context.Context, +) (context.Context, context.CancelFunc) { + deadline, hasDeadline := ctx.Deadline() + if !hasDeadline { + return context.WithoutCancel(ctx), func() {} + } + + return context.WithDeadline(context.WithoutCancel(ctx), deadline) +} + // loadCachedSource opens source content from cache, without reading it, and // returns it with its size; nil if the cached data is unavailable, empty or // exceeds maxResponseSize. @@ -453,7 +461,7 @@ func (s *Service) fetchAndProcess( fetchBytes := int64(len(sourceData)) // Counted also when the request context has ended meanwhile - s.cache.IncrementUpstreamFetch(context.WithoutCancel(ctx), fetchBytes) + s.cache.IncrementUpstreamFetch(ctx, fetchBytes) if err != nil { return nil, fmt.Errorf("failed to read upstream response: %w", err) @@ -535,7 +543,7 @@ func (s *Service) processAndStore( processDuration := time.Since(processStart) // Counted also when the request context has ended meanwhile - s.cache.IncrementTransformCount(context.WithoutCancel(ctx)) + s.cache.IncrementTransformCount(ctx) // Read processed content processedData, err := io.ReadAll(processResult.Content) From 52f69d3082af925e90632aa83ea27178ed149a01 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 07:13:17 +0200 Subject: [PATCH 060/131] Refuse SVG sources and never save an unknown input as JPEG (closes #68) The upstream fetch no longer accepts image/svg+xml, and both image routes answer 415, not 500, for any upstream Content-Type outside the source formats. Before decoding, the image processor names the input's format from govips' DetermineImageType and refuses, with 415, input whose first bytes libvips reads as anything but the six source formats, so an SVG passing the first-bytes check for AVIF never reaches librsvg. orig of such an input is refused instead of being saved as JPEG. README.md drops SVG from the source formats. SVG detection stays in internal/magic, unused on the request path, for https://git.eeqj.de/sneak/pixa/issues/168. An AVIF image sequence (brand avis) answers 415. Model: opus-5-5 --- README.md | 19 +++-- TODO.md | 16 +++++ internal/handlers/image.go | 8 +++ internal/handlers/imageenc.go | 4 ++ .../unsupported_source_internal_test.go | 54 ++++++++++++++ internal/httpfetcher/httpfetcher.go | 1 - internal/imageprocessor/imageprocessor.go | 48 ++++++++----- .../orig_format_internal_test.go | 39 ++++++++++ .../source_formats_internal_test.go | 46 ++++++++++++ internal/server/svg_as_avif_internal_test.go | 71 +++++++++++++++++++ internal/server/svg_internal_test.go | 41 +++++++++++ 11 files changed, 322 insertions(+), 25 deletions(-) create mode 100644 internal/handlers/unsupported_source_internal_test.go create mode 100644 internal/imageprocessor/orig_format_internal_test.go create mode 100644 internal/imageprocessor/source_formats_internal_test.go create mode 100644 internal/server/svg_as_avif_internal_test.go create mode 100644 internal/server/svg_internal_test.go diff --git a/README.md b/README.md index e8e8df4..49c74bb 100644 --- a/README.md +++ b/README.md @@ -190,9 +190,12 @@ path under `/v1/` answers 200, in maintenance mode too. the cache and the upstream fetch; 403 when the upstream host, or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an address in a blocked network (see `blocked_networks`); 429 over - `image_requests_per_minute` (below); 502 when the upstream answered with an - error status, and for 5 minutes after that for the same source URL; 503 when - pixa is busy or in maintenance mode; 500 for any other failure. + `image_requests_per_minute` (below); 415 when the upstream answers with a + `Content-Type` that is not one of the source formats (below), or with an image + whose first bytes libvips reads as another format; 502 when the upstream + answered with an error status, and for 5 minutes after that for the same + source URL; 503 when pixa is busy or in maintenance mode; 500 for any other + failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when `If-None-Match` matches the image's `ETag`; 400 for a token that does not @@ -200,7 +203,7 @@ path under `/v1/` answers 200, in maintenance mode too. expired; 504 when the upstream has not sent its response headers within `upstream_fetch_timeout`, but 500 when that time runs out while the image itself is still arriving; 400 for an `Accept` header that is not valid, and - 406, 403, 429, 502, 503 and 500, as for `/v1/image/`. + 406, 403, 415, 429, 502, 503 and 500, as for `/v1/image/`. - `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: nothing. Answers: 200. - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, @@ -260,9 +263,11 @@ once, is refused with 400. both optional (values under Signature Specification). Both are part of what is cached, so each value of either is a separate cached image. -The source image may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or SVG. The -upstream's `Content-Type` must name its format, and the image's first bytes must -match it. +The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's +`Content-Type` must name its format, and the image's first bytes must match it. +An upstream answering with any other `Content-Type`, SVG included, is refused +with 415, and so is an image whose first bytes libvips reads as another format, +such as an SVG whose first bytes pass the check for AVIF. With the format `auto`, pixa chooses the format for each request from its `Accept` header, in this order: diff --git a/TODO.md b/TODO.md index 66067f4..f239092 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,22 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-13 SVG sources are refused, and `orig` is never saved as JPEG in place + of the source's own format (closes #68): the upstream fetch no longer accepts + `image/svg+xml`, which libvips would render. Both image routes answer 415 for + an upstream `Content-Type` that is not one of the source formats, SVG + included, where they answered 500. Before decoding, the image processor + refuses with `ErrUnsupportedInputFormat`, also answered 415, any input that + govips' `DetermineImageType`, which picks the libvips loader from the first + bytes, does not name as a source format. That catches an SVG whose first bytes + pass the first-bytes check for another type, as an XML comment can make them + do for AVIF; an SVG whose first bytes do not pass that check answers 500, as + before, and so does a damaged image in a source format, whose first bytes + `DetermineImageType` names. `orig` is always the source's own format, where a + source the processor could not name was saved as JPEG. `README.md` no longer + lists SVG. The SVG detection in `internal/magic` stays: the request path does + not call it, and it goes with the unused code of + https://git.eeqj.de/sneak/pixa/issues/168. - 2026-10-12 per-client limit on the image routes (closes #236): the new setting `image_requests_per_minute` (`PIXA_IMAGE_REQUESTS_PER_MINUTE`), default 600, is how many `GET` and `HEAD` requests one client may send to `/v1/image/` and diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 848a252..9978565 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -247,6 +247,14 @@ func (s *Handlers) respondImageError( return } + if errors.Is(err, httpfetcher.ErrInvalidContentType) || + errors.Is(err, imageprocessor.ErrUnsupportedInputFormat) { + s.respondError(w, "unsupported source format", + http.StatusUnsupportedMediaType) + + return + } + s.respondError(w, "internal error", http.StatusInternalServerError) } diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index bf79054..816e83f 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -161,6 +161,10 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { errors.Is(err, imageprocessor.ErrTooManyImages): s.respondError(w, "server busy, try again later", http.StatusServiceUnavailable) + case errors.Is(err, httpfetcher.ErrInvalidContentType), + errors.Is(err, imageprocessor.ErrUnsupportedInputFormat): + s.respondError(w, "unsupported source format", + http.StatusUnsupportedMediaType) default: s.log.Error("image request failed", "error", err) s.respondError(w, "internal error", http.StatusInternalServerError) diff --git a/internal/handlers/unsupported_source_internal_test.go b/internal/handlers/unsupported_source_internal_test.go new file mode 100644 index 0000000..a10e37e --- /dev/null +++ b/internal/handlers/unsupported_source_internal_test.go @@ -0,0 +1,54 @@ +package handlers + +import ( + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestUnsupportedSourceFormatAnswers415 checks that both image routes answer +// 415 with a clear error when the upstream answers with a content type pixa +// does not accept, such as SVG, and when orig is asked of a source in a +// format the processor cannot name, wrapped as the service wraps them. +func TestUnsupportedSourceFormatAnswers415(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.svg"} + + for _, err := range []error{ + fmt.Errorf("upstream fetch failed: %w: image/svg+xml", + httpfetcher.ErrInvalidContentType), + fmt.Errorf("image processing failed: %w: unknown", + imageprocessor.ErrUnsupportedInputFormat), + } { + plain := httptest.NewRecorder() + h.respondImageError(plain, req, err) + + encrypted := httptest.NewRecorder() + h.handleImageError(encrypted, err) + + for route, rec := range map[string]*httptest.ResponseRecorder{ + "HandleImage": plain, "HandleImageEnc": encrypted, + } { + t.Logf("%s for %v: %d %s", route, err, rec.Code, rec.Body.String()) + + if rec.Code != http.StatusUnsupportedMediaType { + t.Errorf("%s for %v: status = %d, want %d", + route, err, rec.Code, http.StatusUnsupportedMediaType) + } + + if !strings.Contains(rec.Body.String(), "unsupported source format") { + t.Errorf("%s for %v: body = %q, want the unsupported source format error", + route, err, rec.Body.String()) + } + } + } +} diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index dd37489..9d09295 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -158,7 +158,6 @@ func DefaultConfig() *Config { contentTypeWebP, contentTypeAVIF, contentTypeJXL, - contentTypeSVG, }, AllowHTTP: false, MaxConnectionsPerHost: DefaultMaxConnectionsPerHost, diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 6d59b34..9f08387 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -137,6 +137,12 @@ var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size") // not supported. var ErrUnsupportedOutputFormat = errors.New("unsupported output format") +// ErrUnsupportedInputFormat is returned, before the input is decoded, when +// libvips reads the input's first bytes as another format than the source +// formats, or as none: for example an SVG whose first bytes pass the +// first-bytes check for AVIF. +var ErrUnsupportedInputFormat = errors.New("unsupported input format") + // ErrTooManyImages is returned when MaxConcurrentProcessing images are being // processed and none finishes within ProcessingWaitTimeout. var ErrTooManyImages = errors.New("too many images being processed at once") @@ -219,6 +225,17 @@ func (p *ImageProcessor) Process( return nil, ErrInputDataTooLarge } + // govips picks the libvips loader from the input's first bytes with + // DetermineImageType. Input whose first bytes it reads as no source + // format, such as SVG, which librsvg would render, is refused before + // libvips reads it. + inputFormat := p.detectFormat(vips.DetermineImageType(data)) + + sourceFormat, err := p.formatFromString(inputFormat) + if err != nil { + return nil, err + } + // Decode image img, err := vips.NewImageFromBuffer(data) if err != nil { @@ -237,9 +254,6 @@ func (p *ImageProcessor) Process( origWidth := img.Width() origHeight := img.Height() - // Detect input format - inputFormat := p.detectFormat(img) - // Validate input dimensions to prevent DoS via decompression bombs if origWidth > MaxInputDimension || origHeight > MaxInputDimension { return nil, ErrInputTooLarge @@ -259,7 +273,7 @@ func (p *ImageProcessor) Process( // orig is the source's own format; encode refuses an empty format outputFormat := req.Format if outputFormat == FormatOriginal { - outputFormat = p.formatFromString(inputFormat) + outputFormat = sourceFormat } // Encode to target format @@ -403,10 +417,9 @@ func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) { } } -// detectFormat returns the format string from a vips image. -func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string { - format := img.Format() - switch format { +// detectFormat returns the format string for a govips image type. +func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { + switch imageType { case vips.ImageTypeJPEG: return "jpeg" case vips.ImageTypePNG: @@ -687,22 +700,23 @@ func jxlDistance(quality int) float64 { return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0 } -// formatFromString converts a format string to Format. -func (p *ImageProcessor) formatFromString(format string) Format { +// formatFromString converts a format string from detectFormat to Format. A +// format detectFormat could not name is ErrUnsupportedInputFormat. +func (p *ImageProcessor) formatFromString(format string) (Format, error) { switch format { case "jpeg": - return FormatJPEG + return FormatJPEG, nil case "png": - return FormatPNG + return FormatPNG, nil case "gif": - return FormatGIF + return FormatGIF, nil case "webp": - return FormatWebP + return FormatWebP, nil case string(FormatAVIF): - return FormatAVIF + return FormatAVIF, nil case string(FormatJXL): - return FormatJXL + return FormatJXL, nil default: - return FormatJPEG + return "", fmt.Errorf("%w: %s", ErrUnsupportedInputFormat, format) } } diff --git a/internal/imageprocessor/orig_format_internal_test.go b/internal/imageprocessor/orig_format_internal_test.go new file mode 100644 index 0000000..453d176 --- /dev/null +++ b/internal/imageprocessor/orig_format_internal_test.go @@ -0,0 +1,39 @@ +package imageprocessor + +import ( + "bytes" + "errors" + "testing" + + "github.com/davidbyttow/govips/v2/vips" +) + +// TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF +// input, which is not a source format, is refused with +// ErrUnsupportedInputFormat before it is decoded, and not saved as JPEG. +func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) { + t.Parallel() + + img, err := vips.NewImageFromBuffer(createTestPNG(t, 20, 20)) + if err != nil { + t.Fatalf("failed to decode test PNG: %v", err) + } + + t.Cleanup(img.Close) + + tiff, _, err := img.ExportTiff(vips.NewTiffExportParams()) + if err != nil { + t.Fatalf("failed to save test TIFF: %v", err) + } + + result, err := New(Params{}).Process( + t.Context(), bytes.NewReader(tiff), &Request{Format: FormatOriginal}, + ) + if result != nil { + t.Logf("Process() returned %s", result.ContentType) + } + + if !errors.Is(err, ErrUnsupportedInputFormat) { + t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedInputFormat) + } +} diff --git a/internal/imageprocessor/source_formats_internal_test.go b/internal/imageprocessor/source_formats_internal_test.go new file mode 100644 index 0000000..5f1dbbd --- /dev/null +++ b/internal/imageprocessor/source_formats_internal_test.go @@ -0,0 +1,46 @@ +package imageprocessor + +import ( + "bytes" + "io" + "testing" +) + +// TestImageProcessor_LoadsEachSourceFormat verifies that an image in each of +// the six source formats, saved by the processor itself, is loaded and saved +// again in its own format (orig). +func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { + t.Parallel() + + proc := New(Params{}) + + for _, format := range []Format{ + FormatJPEG, FormatPNG, FormatGIF, FormatWebP, FormatAVIF, FormatJXL, + } { + saved, err := proc.Process(t.Context(), + bytes.NewReader(createTestPNG(t, 20, 20)), &Request{Format: format}) + if err != nil { + t.Fatalf("saving the test image as %s: %v", format, err) + } + + source, err := io.ReadAll(saved.Content) + if err != nil { + t.Fatalf("reading the %s image: %v", format, err) + } + + result, err := proc.Process(t.Context(), bytes.NewReader(source), + &Request{Format: FormatOriginal}) + if err != nil { + t.Errorf("orig of the %s image: %v", format, err) + + continue + } + + t.Logf("orig of the %s image: %s", format, result.ContentType) + + if result.ContentType != FormatToMIME(format) { + t.Errorf("orig of the %s image is %s, want %s", + format, result.ContentType, FormatToMIME(format)) + } + } +} diff --git a/internal/server/svg_as_avif_internal_test.go b/internal/server/svg_as_avif_internal_test.go new file mode 100644 index 0000000..cd05947 --- /dev/null +++ b/internal/server/svg_as_avif_internal_test.go @@ -0,0 +1,71 @@ +package server + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// svgServedAsAVIF is an SVG whose first bytes pass the first-bytes check for +// AVIF: its XML comment puts "ftypavis" at bytes 4 to 11, where an AVIF file +// has its ftyp box and brand. libvips would load it as SVG. +const svgServedAsAVIF = `` + +// TestImageProxyRefusesSVGServedAsAVIF requests svgServedAsAVIF, which the +// upstream answers as image/avif, through both image routes, resized as PNG +// and in its own format (orig). Each request is answered 415, as for an SVG +// source, never with an image. +func TestImageProxyRefusesSVGServedAsAVIF(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/avif") + _, _ = io.WriteString(w, svgServedAsAVIF) + })) + t.Cleanup(upstream.Close) + + s, _, _ := startImageProxy(t, upstream) + + generator, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator: %v", err) + } + + encryptedURL := func(format imgcache.ImageFormat) string { + token, err := generator.Generate(&encurl.Payload{ + SourceHost: upstreamHost, SourcePath: "/drawing.avif", + Width: 32, Height: 24, Format: format, + }) + if err != nil { + t.Fatalf("encurl Generate: %v", err) + } + + return "/v1/e/" + token + "/drawing." + string(format) + } + + for _, target := range []string{ + "/v1/image/" + upstreamHost + "/drawing.avif/32x24.png", + "/v1/image/" + upstreamHost + "/drawing.avif/64x48.orig", + encryptedURL(imgcache.FormatPNG), + encryptedURL(imgcache.FormatOriginal), + } { + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d, Content-Type %s, body %q", + target, rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) + + if rec.Code != http.StatusUnsupportedMediaType || + !strings.Contains(rec.Body.String(), "unsupported source format") { + t.Errorf("GET %s: %d %q, want %d unsupported source format", + target, rec.Code, rec.Body.String(), http.StatusUnsupportedMediaType) + } + } +} diff --git a/internal/server/svg_internal_test.go b/internal/server/svg_internal_test.go new file mode 100644 index 0000000..95b46fa --- /dev/null +++ b/internal/server/svg_internal_test.go @@ -0,0 +1,41 @@ +package server + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// TestImageProxyRefusesSVG requests an image the upstream answers as SVG, +// in its own format (orig) and resized as PNG, through pixa's router, +// upstream fetcher and image processor. pixa does not accept SVG as a +// source, so each request is answered 415, never with an image. +func TestImageProxyRefusesSVG(t *testing.T) { + t.Parallel() + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "image/svg+xml") + _, _ = io.WriteString(w, ``) + })) + t.Cleanup(upstream.Close) + + s, _, _ := startImageProxy(t, upstream) + + for _, sizeFormat := range []string{"64x48.orig", "32x24.png"} { + target := "/v1/image/" + upstreamHost + "/drawing.svg/" + sizeFormat + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d, Content-Type %s, body %s", + target, rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) + + if rec.Code != http.StatusUnsupportedMediaType { + t.Errorf("GET %s status = %d, want %d", + target, rec.Code, http.StatusUnsupportedMediaType) + } + } +} From ca512fdb4f9857ff9ff508c0f8554753ff649552 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 07:58:55 +0200 Subject: [PATCH 061/131] Remove the unused cache lifetime setting (closes #69) CacheConfig.CacheTTL was set to DefaultCacheTTL (24 hours) and never read, so cached images never expired. Remove the field, the constant and the line that set it. Cached source and transformed images stay until eviction removes them to keep the cache within cache_max_bytes. A new test caches a source and a transformed image, sets every time pixa stores for them two days back (database rows, metadata files and file times), and checks that both are still served from the cache without a new upstream fetch. Existing tests lose only the CacheTTL line of their cache config. Model: opus-5-5 --- TODO.md | 5 + internal/handlers/handlers.go | 1 - internal/handlers/handlers_internal_test.go | 1 - .../image_cache_control_internal_test.go | 1 - .../handlers/image_errors_internal_test.go | 1 - .../referer_blocklist_internal_test.go | 1 - internal/imgcache/cache.go | 1 - internal/imgcache/cache_internal_test.go | 2 - internal/imgcache/eviction_internal_test.go | 1 - internal/imgcache/module.go | 1 - .../imgcache/negative_cache_internal_test.go | 2 - internal/imgcache/service_internal_test.go | 168 ++++++++++++++++++ internal/imgcache/stats_internal_test.go | 5 - internal/imgcache/testutil_internal_test.go | 1 - 14 files changed, 173 insertions(+), 18 deletions(-) diff --git a/TODO.md b/TODO.md index f239092..5a5cf5d 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,11 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-13 the unused cache lifetime is removed (closes #69): + `CacheConfig.CacheTTL`, which pixad set to `DefaultCacheTTL` (24 hours), was + never read, so cached images never expired. The field and the constant are + removed. Cached source and transformed images stay until eviction removes them + to keep the cache within `cache_max_bytes`. - 2026-10-13 SVG sources are refused, and `orig` is never saved as JPEG in place of the source's own format (closes #68): the upstream fetch no longer accepts `image/svg+xml`, which libvips would render. Both image routes answer 415 for diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 6405d08..297ba2d 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -106,7 +106,6 @@ func (s *Handlers) WaitForProcessing(ctx context.Context) int { func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig { return imgcache.CacheConfig{ StateDir: cfg.StateDir, - CacheTTL: imgcache.DefaultCacheTTL, NegativeTTL: imgcache.DefaultNegativeTTL, MaxBytes: cfg.CacheMaxBytes, UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit, diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index 35c1522..f396a05 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -47,7 +47,6 @@ func setupTestHandler(t *testing.T) *testFixtures { cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ StateDir: tmpDir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index 7845df5..f4e65a5 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -31,7 +31,6 @@ func newSignedHostServer( cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index 0bb3fea..fbbcf6f 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -29,7 +29,6 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index 9235eb1..86f0c32 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -49,7 +49,6 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index ff27456..1f73019 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -35,7 +35,6 @@ const metaCacheSize = 10000 // CacheConfig holds cache configuration. type CacheConfig struct { StateDir string - CacheTTL time.Duration NegativeTTL time.Duration // MaxBytes is the disk cache size limit in bytes that eviction diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index b4026f5..6659b93 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -102,7 +102,6 @@ func setupTestCache(t *testing.T) *Cache { cache, err := NewCache(db, CacheConfig{ StateDir: tmpDir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { @@ -241,7 +240,6 @@ func TestCache_NegativeCacheExpiry(t *testing.T) { // Very short negative TTL for testing cache, err := NewCache(db, CacheConfig{ StateDir: tmpDir, - CacheTTL: time.Hour, NegativeTTL: 1 * time.Millisecond, }) if err != nil { diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index af91ac5..7d54df1 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -70,7 +70,6 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) { // backwards compatibility with existing fixtures. cache, err := NewCache(db, CacheConfig{ StateDir: tmpDir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, MaxBytes: maxBytes, DisableDiskCache: maxBytes == 0, diff --git a/internal/imgcache/module.go b/internal/imgcache/module.go index 266e398..81d329a 100644 --- a/internal/imgcache/module.go +++ b/internal/imgcache/module.go @@ -4,6 +4,5 @@ import "time" // CacheConfig defaults. const ( - DefaultCacheTTL = 24 * time.Hour DefaultNegativeTTL = 5 * time.Minute ) diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index 856f6ae..f2c2115 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -15,7 +15,6 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { @@ -63,7 +62,6 @@ func TestNegativeCache_Expired(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, - CacheTTL: time.Hour, NegativeTTL: 1 * time.Millisecond, // very short TTL }) if err != nil { diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index 5607853..85ad9dd 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -2,7 +2,9 @@ package imgcache import ( "context" + "encoding/json" "io" + "os" "testing" "time" @@ -430,6 +432,172 @@ func TestService_Get_Caching(t *testing.T) { } } +// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and +// a variant of it and sets back two days every time pixa stored for them. +// For the source: fetched_at in source_metadata, fetched_at and +// last_accessed_at in source_content, the fetched_at in its metadata file, +// and the access and modification times of its file and of its metadata +// file. For the variant: created_at and last_accessed_at in variant_content, +// the created_at in its .meta file, and the access and modification times of +// its file and of its .meta file. It then checks that the variant is still +// served from the cache and that a new variant is made from the cached source +// without fetching it again. +// +// Left at the present: the files' status change and birth times, which a +// program cannot set, and the times of the directories holding the files, +// which other entries share. The source's stored response headers are empty, +// as the mock fetcher sends none. expires_at, etag and last_modified in +// source_metadata and in the metadata file are never written, nor are the +// request_cache and output_content tables. +func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + variant := photoVariant(fixtures, 85, FitCover) + + resp, err := svc.Get(t.Context(), variant) + if err != nil { + t.Fatalf("Get() first request error = %v", err) + } + + _ = resp.Content.Close() + + twoDaysAgo := time.Now().AddDate(0, 0, -2) + backdateSource(t, svc.cache, variant, twoDaysAgo) + backdateVariant(t, svc.cache, variant, twoDaysAgo) + + resp, err = svc.Get(t.Context(), variant) + if err != nil { + t.Fatalf("Get() of the cached variant error = %v", err) + } + + _ = resp.Content.Close() + + if resp.CacheStatus != CacheHit { + t.Errorf("cached variant CacheStatus = %q, want %q", + resp.CacheStatus, CacheHit) + } + + // Another quality is a new variant, made from the cached source + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, FitCover)) + if err != nil { + t.Fatalf("Get() of a new variant error = %v", err) + } + + _ = resp.Content.Close() + + fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount + if fetches != 1 { + t.Errorf("upstream fetches = %d, want 1", fetches) + } +} + +// backdateSource sets every time pixa stored for req's cached source to when, +// as listed on TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo. +func backdateSource( + t *testing.T, cache *Cache, req *ImageRequest, when time.Time, +) { + t.Helper() + + pathHash := HashPath(req.SourcePath + "?" + req.SourceQuery) + + meta, err := cache.srcMetadata.Load(req.SourceHost, pathHash) + if err != nil { + t.Fatalf("reading the source's metadata file: %v", err) + } + + meta.FetchedAt = when.Unix() + + err = cache.srcMetadata.Store(req.SourceHost, pathHash, meta) + if err != nil { + t.Fatalf("writing the source's metadata file: %v", err) + } + + dbTime := when.UTC().Format(sqliteTimestampFormat) + + updateOneRow(t, cache, ` + UPDATE source_metadata SET fetched_at = ? + WHERE source_host = ? AND source_path = ? AND source_query = ? + `, dbTime, req.SourceHost, req.SourcePath, req.SourceQuery) + updateOneRow(t, cache, + `UPDATE source_content SET fetched_at = ? WHERE content_hash = ?`, + dbTime, meta.ContentHash) + setSourceLastAccessed(t, cache, ContentHash(meta.ContentHash), when) + + for _, path := range []string{ + cache.srcContent.hashToPath(ContentHash(meta.ContentHash)), + cache.srcMetadata.metaPath(req.SourceHost, pathHash), + } { + err = os.Chtimes(path, when, when) + if err != nil { + t.Fatalf("setting back the times of %s: %v", path, err) + } + } +} + +// backdateVariant sets every time pixa stored for the cached variant req asks +// for to when, as listed on +// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo. +func backdateVariant( + t *testing.T, cache *Cache, req *ImageRequest, when time.Time, +) { + t.Helper() + + key := CacheKey(req) + + content, size, contentType, err := cache.variants.LoadWithMeta(key) + if err != nil || contentType == "" { + t.Fatalf("reading the variant's .meta file: content type %q, error %v", + contentType, err) + } + + _ = content.Close() + + meta, err := json.Marshal(VariantMeta{ + ContentType: contentType, + Size: size, + CreatedAt: when.Unix(), + }) + if err != nil { + t.Fatalf("encoding the variant's .meta file: %v", err) + } + + path := cache.variants.keyToPath(key) + + err = os.WriteFile(path+variantMetaSuffix, meta, StorageFilePerm) + if err != nil { + t.Fatalf("writing the variant's .meta file: %v", err) + } + + updateOneRow(t, cache, + `UPDATE variant_content SET created_at = ? WHERE cache_key = ?`, + when.UTC().Format(sqliteTimestampFormat), string(key)) + setVariantLastAccessed(t, cache, key, when) + + for _, file := range []string{path, path + variantMetaSuffix} { + err = os.Chtimes(file, when, when) + if err != nil { + t.Fatalf("setting back the times of %s: %v", file, err) + } + } +} + +// updateOneRow runs the UPDATE statement query and fails the test unless it +// changed exactly one row. +func updateOneRow(t *testing.T, cache *Cache, query string, args ...any) { + t.Helper() + + result, err := cache.db.ExecContext(t.Context(), query, args...) + if err != nil { + t.Fatalf("%s: %v", query, err) + } + + rows, err := result.RowsAffected() + if err != nil || rows != 1 { + t.Fatalf("%s: rows = %d, error = %v, want 1 row", query, rows, err) + } +} + func TestService_Get_DifferentSizes(t *testing.T) { t.Parallel() diff --git a/internal/imgcache/stats_internal_test.go b/internal/imgcache/stats_internal_test.go index 94ee241..2fe9dbb 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imgcache/stats_internal_test.go @@ -43,7 +43,6 @@ func TestStats_HitRateIsRatio(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { @@ -91,7 +90,6 @@ func TestStats_ZeroCounts(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { @@ -120,7 +118,6 @@ func TestStats_LogsFailedCountQueries(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: t.TempDir(), - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, Logger: slog.New(slog.NewJSONHandler(&logBuf, nil)), }) @@ -162,7 +159,6 @@ func TestIncrementStats_LogsFailedUpdates(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: t.TempDir(), - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, Logger: slog.New(slog.NewJSONHandler(&logBuf, nil)), }) @@ -232,7 +228,6 @@ func TestStats_DisabledCacheReportsNoItems(t *testing.T) { disabled, err := NewCache(enabled.db, CacheConfig{ StateDir: t.TempDir(), - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, DisableDiskCache: true, }) diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index b29f750..326537f 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -185,7 +185,6 @@ func SetupTestService( cache, err := NewCache(db, CacheConfig{ StateDir: tmpDir, - CacheTTL: time.Hour, NegativeTTL: 5 * time.Minute, }) if err != nil { From d8760c40a1326f7c1ae4d9c7247915954ddf9d80 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 09:44:44 +0200 Subject: [PATCH 062/131] Refuse a size whose computed side exceeds the maximum (closes #241) A side given as 0 is computed from the source's proportions, and nothing held it to 8192: 0x8192 of a 3000x600 source was resized to 40960x8192 and encoded. Process now refuses such a size with ErrOutputTooLarge once it knows the source's dimensions, before the resize and the encode, and both image routes answer it 400 with the JSON error body. It is a processing error, not a failed fetch, so it is never negative-cached. imgcache.MaxDimension is now imageprocessor.MaxOutputDimension, so the request limit and the output limit are one value. README.md says it under Routes and Encrypted URLs. Model: opus-5-5 --- README.md | 7 +- TODO.md | 9 ++ internal/handlers/image.go | 8 + internal/handlers/imageenc.go | 4 + .../computed_side_internal_test.go | 52 ++++++ internal/imageprocessor/imageprocessor.go | 43 +++-- internal/imgcache/urlparser.go | 7 +- .../server/computed_side_internal_test.go | 153 ++++++++++++++++++ 8 files changed, 268 insertions(+), 15 deletions(-) create mode 100644 internal/imageprocessor/computed_side_internal_test.go create mode 100644 internal/server/computed_side_internal_test.go diff --git a/README.md b/README.md index 49c74bb..9525dcc 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,10 @@ once, is refused with 400. both optional (values under Signature Specification). Both are part of what is cached, so each value of either is a separate cached image. +A size with one side `0` has that side scaled to keep the source's proportions, +and is refused with 400 when that side would come out above 8192, the most +either side may be. + The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's `Content-Type` must name its format, and the image's first bytes must match it. An upstream answering with any other `Content-Type`, SVG included, is refused @@ -345,7 +349,8 @@ nor change what it asks for. (`POST /generate`). The format is JPEG XL unless another is chosen; a form sent with an empty format, or none, also makes a JPEG XL URL. Width and height both empty or `0` keep the original size; if only one of them is empty - or `0`, that side is scaled to keep the image's proportions. + or `0`, that side is scaled to keep the image's proportions. A URL whose + scaled side would come out above 8192 answers 400. 3. The page shows the URL, `https:///v1/e//img.`, and when it expires. `` is the host the page was opened on, and the URL starts with `http` instead while `debug` is on. The name after the token is ignored diff --git a/TODO.md b/TODO.md index 5a5cf5d..2ec0e84 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-13 a size with one side `0` no longer makes an output above 8192 on a + side (closes #241): that side is computed from the source's proportions, and + nothing held it to the 8192 a requested side is held to, so `0x8192` of a + 3000x600 source was resized to 40960x8192 and encoded. The image processor now + refuses such a size with `ErrOutputTooLarge` once it knows the source's + dimensions, before resizing, and both image routes answer it 400 with the JSON + error body. The refusal is not cached as a failed fetch, so the same source is + still served at other sizes. `imgcache.MaxDimension` is now the image + processor's `MaxOutputDimension`, so the two cannot differ. - 2026-10-13 the unused cache lifetime is removed (closes #69): `CacheConfig.CacheTTL`, which pixad set to `DefaultCacheTTL` (24 hours), was never read, so cached images never expired. The field and the constant are diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 9978565..aefde8e 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -255,6 +255,14 @@ func (s *Handlers) respondImageError( return } + if errors.Is(err, imageprocessor.ErrOutputTooLarge) { + s.respondError(w, fmt.Sprintf("size too large for the source: the side "+ + "computed from its proportions is above %d", + imageprocessor.MaxOutputDimension), http.StatusBadRequest) + + return + } + s.respondError(w, "internal error", http.StatusInternalServerError) } diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 816e83f..8da6d47 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -165,6 +165,10 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { errors.Is(err, imageprocessor.ErrUnsupportedInputFormat): s.respondError(w, "unsupported source format", http.StatusUnsupportedMediaType) + case errors.Is(err, imageprocessor.ErrOutputTooLarge): + s.respondError(w, fmt.Sprintf("size too large for the source: the side "+ + "computed from its proportions is above %d", + imageprocessor.MaxOutputDimension), http.StatusBadRequest) default: s.log.Error("image request failed", "error", err) s.respondError(w, "internal error", http.StatusInternalServerError) diff --git a/internal/imageprocessor/computed_side_internal_test.go b/internal/imageprocessor/computed_side_internal_test.go new file mode 100644 index 0000000..f5c3d5a --- /dev/null +++ b/internal/imageprocessor/computed_side_internal_test.go @@ -0,0 +1,52 @@ +package imageprocessor + +import ( + "bytes" + "errors" + "testing" +) + +// TestImageProcessor_RefusesComputedSideAboveMaximum verifies that a size +// with one side 0, which Process computes from the input's proportions, is +// served when that side comes out at MaxOutputDimension and refused with +// ErrOutputTooLarge when it would be above. A 4096x1 or 1x4096 input given +// 2 for its short side has 8192 for its long side. +func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputWidth, inputHeight int + size Size + wantErr error + }{ + {"width at the maximum", 4096, 1, Size{Height: 2}, nil}, + {"width above the maximum", 4096, 1, Size{Height: 3}, ErrOutputTooLarge}, + {"height at the maximum", 1, 4096, Size{Width: 2}, nil}, + {"height above the maximum", 1, 4096, Size{Width: 3}, ErrOutputTooLarge}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover}) + if result != nil { + t.Logf("Process() made %dx%d", result.Width, result.Height) + + if max(result.Width, result.Height) != MaxOutputDimension { + t.Errorf("Process() made %dx%d, want a side of %d", + result.Width, result.Height, MaxOutputDimension) + } + } + + if !errors.Is(err, tt.wantErr) { + t.Errorf("Process() error = %v, want %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 9f08387..4ad8007 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -133,6 +133,14 @@ var ErrInputTooLarge = errors.New("input image dimensions exceed maximum") // configured byte limit. var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size") +// MaxOutputDimension is the maximum width or height of an output image, and +// so of a requested size (imgcache.MaxDimension). +const MaxOutputDimension = 8192 + +// ErrOutputTooLarge is returned when a size with one side 0 would give that +// side, computed from the input's proportions, more than MaxOutputDimension. +var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum") + // ErrUnsupportedOutputFormat is returned when the requested output format is // not supported. var ErrUnsupportedOutputFormat = errors.New("unsupported output format") @@ -259,8 +267,11 @@ func (p *ImageProcessor) Process( return nil, ErrInputTooLarge } - // Determine target dimensions - targetWidth, targetHeight := targetDimensions(req.Size, origWidth, origHeight) + // Determine target dimensions; too large ones are refused before resizing + targetWidth, targetHeight, err := targetDimensions(req.Size, origWidth, origHeight) + if err != nil { + return nil, err + } // Resize if needed if targetWidth != origWidth || targetHeight != origHeight { @@ -296,21 +307,29 @@ func (p *ImageProcessor) Process( // targetDimensions calculates the output dimensions for a requested size, // scaling proportionally when only one dimension is given and keeping the -// original dimensions when both are zero. -func targetDimensions(size Size, origWidth, origHeight int) (int, int) { +// original dimensions when both are zero. A side scaled proportionally can +// come out above MaxOutputDimension, which is ErrOutputTooLarge. +func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) { + width, height := size.Width, size.Height + switch { - case size.Width == 0 && size.Height == 0: + case width == 0 && height == 0: // Both are 0: keep original size - return origWidth, origHeight - case size.Width == 0: + return origWidth, origHeight, nil + case width == 0: // Only height specified: calculate width proportionally - return origWidth * size.Height / origHeight, size.Height - case size.Height == 0: + width = origWidth * height / origHeight + case height == 0: // Only width specified: calculate height proportionally - return size.Width, origHeight * size.Width / origWidth - default: - return size.Width, size.Height + height = origHeight * width / origWidth } + + if width > MaxOutputDimension || height > MaxOutputDimension { + return 0, 0, fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge, + width, height, MaxOutputDimension) + } + + return width, height, nil } // MIME types for the supported image formats. diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index b11d80e..0d5e6c5 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -7,6 +7,8 @@ import ( "regexp" "strconv" "strings" + + "sneak.berlin/go/pixa/internal/imageprocessor" ) // URL parsing errors. @@ -20,8 +22,9 @@ var ( ErrPathTraversal = errors.New("path traversal detected") ) -// MaxDimension is the maximum allowed width or height. -const MaxDimension = 8192 +// MaxDimension is the maximum allowed width or height, the same as the image +// processor's maximum for an output image. +const MaxDimension = imageprocessor.MaxOutputDimension // ValidateDimension checks one requested width or height; name ("width" or // "height") appears in the error. 0 means "original size" and is valid. diff --git a/internal/server/computed_side_internal_test.go b/internal/server/computed_side_internal_test.go new file mode 100644 index 0000000..ded083f --- /dev/null +++ b/internal/server/computed_side_internal_test.go @@ -0,0 +1,153 @@ +package server + +import ( + "bytes" + "encoding/json" + "image/jpeg" + "net/http" + "net/http/httptest" + "testing" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestImageProxyRefusesComputedSideAboveMaximum requests sizes of a 3000x600 +// and a 600x3000 PNG through both image routes. A size with one side 0, whose +// side computed from the source's proportions would be above 8192, is refused +// with 400 and the JSON error body. The sizes after those refusals are served +// as before, which also shows that a refusal is not cached as a failed fetch of +// the source: orig, a size with both sides given, and a size with one side 0 +// whose computed side stays within 8192. +func TestImageProxyRefusesComputedSideAboveMaximum(t *testing.T) { + t.Parallel() + + // The upstream paths of the two sources + const ( + widePath = "/wide.png" + tallPath = "/tall.png" + ) + + wide := encodeTestPNG(t, 3000, 600) + tall := encodeTestPNG(t, 600, 3000) + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + + if r.URL.Path == tallPath { + _, _ = w.Write(tall) + + return + } + + _, _ = w.Write(wide) + })) + t.Cleanup(upstream.Close) + + s, _, _ := startImageProxy(t, upstream) + + generator, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator: %v", err) + } + + tests := []struct { + path string + // size is the size of the /v1/image/ URL, and width and height are + // those of the encrypted URL + size string + width, height int + // wantWidth and wantHeight are the served image's, 0 for a refusal + wantWidth, wantHeight int + }{ + {widePath, "0x8192", 0, 8192, 0, 0}, + {tallPath, "8192x0", 8192, 0, 0, 0}, + {widePath, "orig", 0, 0, 3000, 600}, + {widePath, "300x200", 300, 200, 300, 200}, + {widePath, "0x300", 0, 300, 1500, 300}, + {tallPath, "300x0", 300, 0, 300, 1500}, + } + + for _, tt := range tests { + token, err := generator.Generate(&encurl.Payload{ + SourceHost: upstreamHost, SourcePath: tt.path, + Width: tt.width, Height: tt.height, Format: imgcache.FormatJPEG, + }) + if err != nil { + t.Fatalf("encurl Generate: %v", err) + } + + for _, target := range []string{ + "/v1/image/" + upstreamHost + tt.path + "/" + tt.size + ".jpeg", + "/v1/e/" + token + "/img.jpg", + } { + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d, Content-Type %s", + target, rec.Code, rec.Header().Get("Content-Type")) + + if tt.wantWidth == 0 { + checkSizeRefused(t, rec) + } else { + checkServedSize(t, rec, tt.wantWidth, tt.wantHeight) + } + } + } +} + +// checkSizeRefused checks that rec answers 400 with the JSON error body that +// refuses a size whose side computed from the source's proportions is above +// 8192. +func checkSizeRefused(t *testing.T, rec *httptest.ResponseRecorder) { + t.Helper() + + const wantError = "size too large for the source: " + + "the side computed from its proportions is above 8192" + + var body struct { + Error string `json:"error"` + Status int `json:"status"` + } + + err := json.Unmarshal(rec.Body.Bytes(), &body) + if err != nil { + t.Errorf("status %d, body is not JSON: %v", rec.Code, err) + + return + } + + if rec.Code != http.StatusBadRequest || body.Status != http.StatusBadRequest || + body.Error != wantError { + t.Errorf("status %d, body %+v, want %d and the error %q", + rec.Code, body, http.StatusBadRequest, wantError) + } +} + +// checkServedSize checks that rec answers 200 with a JPEG of wantWidth by +// wantHeight. +func checkServedSize( + t *testing.T, rec *httptest.ResponseRecorder, wantWidth, wantHeight int, +) { + t.Helper() + + if rec.Code != http.StatusOK { + t.Errorf("status %d, want %d; body %s", + rec.Code, http.StatusOK, rec.Body.String()) + + return + } + + served, err := jpeg.DecodeConfig(bytes.NewReader(rec.Body.Bytes())) + if err != nil { + t.Errorf("decoding the served JPEG: %v", err) + + return + } + + if served.Width != wantWidth || served.Height != wantHeight { + t.Errorf("served %dx%d, want %dx%d", + served.Width, served.Height, wantWidth, wantHeight) + } +} From afb5dc5d0d66cab374362fcd882e0f6db8ae334c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 13 Oct 2026 10:44:45 +0200 Subject: [PATCH 063/131] Refuse a fit=outside size whose output exceeds the maximum (closes #257) fit=outside scales the source to cover the requested size, and the side scaled past it was held to no maximum: 1x8192?fit=outside of a 3000x600 source was resized to 40960x8192 and encoded. resize now refuses such an output with ErrOutputTooLarge before resizing, so both image routes answer it 400 with the error a size with one side 0 already gets. The other fit modes make the requested size or one within it. README.md and TODO.md say so. Model: opus-5-5 --- README.md | 4 +- TODO.md | 7 ++ .../fit_outside_internal_test.go | 62 +++++++++++++ internal/imageprocessor/imageprocessor.go | 16 +++- internal/server/fit_outside_internal_test.go | 91 +++++++++++++++++++ 5 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 internal/imageprocessor/fit_outside_internal_test.go create mode 100644 internal/server/fit_outside_internal_test.go diff --git a/README.md b/README.md index 9525dcc..a9d14e1 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,9 @@ once, is refused with 400. A size with one side `0` has that side scaled to keep the source's proportions, and is refused with 400 when that side would come out above 8192, the most -either side may be. +either side may be; so is a size with `fit=outside`, which scales the image to +cover the size and so past it on one side, when that side would come out +above 8192. The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's `Content-Type` must name its format, and the image's first bytes must match it. diff --git a/TODO.md b/TODO.md index 2ec0e84..190ecec 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,13 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side + (closes #257): it scales the image to cover the requested size, and the side + scaled past that size was held to no maximum, so `1x8192?fit=outside` of a + 3000x600 source was resized to 40960x8192 and encoded. The image processor now + refuses such a size with `ErrOutputTooLarge` before resizing, and both image + routes answer it 400, as for a size with one side `0`. The other fit modes + make the requested size or one within it, so none can go above 8192. - 2026-10-13 a size with one side `0` no longer makes an output above 8192 on a side (closes #241): that side is computed from the source's proportions, and nothing held it to the 8192 a requested side is held to, so `0x8192` of a diff --git a/internal/imageprocessor/fit_outside_internal_test.go b/internal/imageprocessor/fit_outside_internal_test.go new file mode 100644 index 0000000..ab3737f --- /dev/null +++ b/internal/imageprocessor/fit_outside_internal_test.go @@ -0,0 +1,62 @@ +package imageprocessor + +import ( + "bytes" + "errors" + "testing" +) + +// TestImageProcessor_RefusesFitOutsideAboveMaximum verifies that FitOutside, +// which scales the input to cover the requested size and so past it on one +// side, is refused with ErrOutputTooLarge when that side would come out above +// MaxOutputDimension, and otherwise makes the scaled size. A 1024x256 or +// 256x1024 input scaled by 8 has 8192 for its long side. +func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputWidth, inputHeight int + size Size + // wantWidth and wantHeight are the output's, 0 for a refusal + wantWidth, wantHeight int + }{ + {"width above the maximum", 3000, 600, Size{Width: 1, Height: 8192}, 0, 0}, + {"height above the maximum", 600, 3000, Size{Width: 8192, Height: 1}, 0, 0}, + {"width at the maximum", 1024, 256, Size{Width: 1, Height: 2048}, 8192, 2048}, + {"height at the maximum", 256, 1024, Size{Width: 2048, Height: 1}, 2048, 8192}, + {"within the maximum", 3000, 600, Size{Width: 1, Height: 1200}, 6000, 1200}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside}) + if result != nil { + t.Logf("Process() made %dx%d", result.Width, result.Height) + } + + if tt.wantWidth == 0 { + if !errors.Is(err, ErrOutputTooLarge) { + t.Errorf("Process() error = %v, want %v", err, ErrOutputTooLarge) + } + + return + } + + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + if result.Width != tt.wantWidth || result.Height != tt.wantHeight { + t.Errorf("Process() made %dx%d, want %dx%d", + result.Width, result.Height, tt.wantWidth, tt.wantHeight) + } + }) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 4ad8007..92cee8a 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -137,8 +137,10 @@ var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size") // so of a requested size (imgcache.MaxDimension). const MaxOutputDimension = 8192 -// ErrOutputTooLarge is returned when a size with one side 0 would give that -// side, computed from the input's proportions, more than MaxOutputDimension. +// ErrOutputTooLarge is returned, before resizing, when the output would have a +// side above MaxOutputDimension: the side of a size with one side 0, computed +// from the input's proportions, or with FitOutside, the side scaled past the +// requested size. var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum") // ErrUnsupportedOutputFormat is returned when the requested output format is @@ -460,7 +462,9 @@ func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { } } -// resize resizes the image according to the fit mode. +// resize resizes the image according to the fit mode. With FitOutside one +// side can come out larger than requested; when it would be above +// MaxOutputDimension, resize returns ErrOutputTooLarge without resizing. func (p *ImageProcessor) resize( img *vips.ImageRef, width, height int, fit FitMode, ) error { @@ -508,6 +512,12 @@ func (p *ImageProcessor) resize( newW := int(float64(imgW) * scale) newH := int(float64(imgH) * scale) + // The dimension that exceeds the target is held to the maximum too + if newW > MaxOutputDimension || newH > MaxOutputDimension { + return fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge, + newW, newH, MaxOutputDimension) + } + return img.Thumbnail(newW, newH, vips.InterestingNone) default: diff --git a/internal/server/fit_outside_internal_test.go b/internal/server/fit_outside_internal_test.go new file mode 100644 index 0000000..f0ada85 --- /dev/null +++ b/internal/server/fit_outside_internal_test.go @@ -0,0 +1,91 @@ +package server + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestImageProxyRefusesFitOutsideAboveMaximum requests sizes with fit=outside +// through both image routes. fit=outside scales the source to cover the size, +// and so past it on one side: a size for which that side would be above 8192 +// is refused with 400 and the JSON error body, whether it is the width or the +// height, and a size for which it stays within 8192 is served at the scaled +// size. +func TestImageProxyRefusesFitOutsideAboveMaximum(t *testing.T) { + t.Parallel() + + // The upstream paths of the sources + const ( + widePath = "/wide.png" + rowPath = "/row.png" + columnPath = "/column.png" + ) + + sources := map[string][]byte{ + widePath: encodeTestPNG(t, 3000, 600), + rowPath: encodeTestPNG(t, 8192, 1), + columnPath: encodeTestPNG(t, 1, 8192), + } + + upstream := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(sources[r.URL.Path]) + })) + t.Cleanup(upstream.Close) + + s, _, _ := startImageProxy(t, upstream) + + generator, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator: %v", err) + } + + tests := []struct { + path string + width, height int + // wantWidth and wantHeight are the served image's, 0 for a refusal + wantWidth, wantHeight int + }{ + {widePath, 1, 8192, 0, 0}, + {rowPath, 1, 8192, 0, 0}, + {columnPath, 8192, 1, 0, 0}, + {widePath, 1, 1200, 6000, 1200}, + } + + for _, tt := range tests { + token, err := generator.Generate(&encurl.Payload{ + SourceHost: upstreamHost, SourcePath: tt.path, + Width: tt.width, Height: tt.height, Format: imgcache.FormatJPEG, + FitMode: imgcache.FitOutside, + }) + if err != nil { + t.Fatalf("encurl Generate: %v", err) + } + + size := fmt.Sprintf("%dx%d", tt.width, tt.height) + + for _, target := range []string{ + "/v1/image/" + upstreamHost + tt.path + "/" + size + + ".jpeg?fit=outside", + "/v1/e/" + token + "/img.jpg", + } { + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d, Content-Type %s", + target, rec.Code, rec.Header().Get("Content-Type")) + + if tt.wantWidth == 0 { + checkSizeRefused(t, rec) + } else { + checkServedSize(t, rec, tt.wantWidth, tt.wantHeight) + } + } + } +} From 88e1b1cdfaa9e4d7b1d165d5d35898c19f2b2056 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 06:26:23 +0200 Subject: [PATCH 064/131] Allowlist no host in the example config (closes #249) The example config allowlisted github.com and user-images.githubusercontent.com, where anyone can put an image, so a deployment set up as Getting Started says served every image there to anyone without a signature. The example now allowlists nothing and shows one commented entry under example.com. README.md says in Getting Started and in "Running under upaas" that every image on an allowlisted host is served without a signature, so an allowlisted host must be one the operator controls. A new test loads the example config and fails if it allowlists any host. Model: opus-5-5 --- README.md | 9 ++++++++- TODO.md | 8 ++++++++ configs/config.example.yml | 11 +++++------ internal/config/config_internal_test.go | 26 +++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a9d14e1..250463e 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ make docker docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest ``` +The example config allowlists no host (`allowlist_hosts`), so pixa serves an +image only for a signed or encrypted URL. Every image on an allowlisted host is +served to anyone without a signature, at any size and in any format, so +allowlist only hosts you control, where nobody else can put images. + A container takes its settings from environment variables (see Configuration below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the container exits at startup naming the variable. Everything else has a built-in @@ -115,7 +120,9 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: - `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and login, 32+ characters, for example from `openssl rand -base64 32` - `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature, - comma-separated + comma-separated; default none. Every image on a listed host is served to + anyone without a signature, so list only hosts you control, where nobody + else can put images. - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it; default 75% of (free space + what the cache holds) - the rest are in the table under Configuration below diff --git a/TODO.md b/TODO.md index 190ecec..b85bf37 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the example config allowlists no host (closes #249): it allowlisted + `github.com` and `user-images.githubusercontent.com`, where anyone can put an + image, so a deployment set up as Getting Started says served those images to + anyone without a signature, at any size and in any format. + `configs/config.example.yml` now leaves `allowlist_hosts` out, with one + commented entry under `example.com`. Getting Started and "Running under upaas" + in `README.md` say that every image on an allowlisted host is served without a + signature, so an allowlisted host must be one the operator controls. - 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side (closes #257): it scales the image to cover the requested size, and the side scaled past that size was held to no maximum, so `1x8192?fit=outside` of a diff --git a/configs/config.example.yml b/configs/config.example.yml index 2c59549..cdf1022 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -45,15 +45,14 @@ state_dir: ./data signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32" # Hosts that don't require signatures (default: none) +# Every image on an allowlisted host is served to anyone without a +# signature, at any size and in any format, so list only hosts you +# control, where nobody else can put images. # Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com") # An entry that is neither a host name nor an IP address (IPv6 without # brackets), such as one with a port or a "*." wildcard, aborts startup. -allowlist_hosts: - - s3.sneak.cloud - - static.sneak.cloud - - sneak.berlin - - github.com - - user-images.githubusercontent.com +# allowlist_hosts: +# - images.example.com # Hosts whose pages may not show pixa's images, written as for # allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header diff --git a/internal/config/config_internal_test.go b/internal/config/config_internal_test.go index de33d40..b655b35 100644 --- a/internal/config/config_internal_test.go +++ b/internal/config/config_internal_test.go @@ -92,6 +92,32 @@ func TestGetStringSlice_Empty(t *testing.T) { } } +// TestExampleConfigAllowlistsNoHost loads configs/config.example.yml +// with a real signing key, as Getting Started in README.md has it copied +// and edited, and checks that it allowlists no host: every image on an +// allowlisted host is served to anyone without a signature. +func TestExampleConfigAllowlistsNoHost(t *testing.T) { + t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey) + + examplePath := filepath.Join("..", "..", "configs", "config.example.yml") + + sc, err := loadTestConfig(examplePath) + if err != nil { + t.Fatalf("failed to load %s: %v", examplePath, err) + } + + c, err := newFromSmartConfig(sc) + if err != nil { + t.Fatalf("%s is not a valid config: %v", examplePath, err) + } + + t.Logf("allowlist_hosts in %s: %v", examplePath, c.AllowlistHosts) + + if len(c.AllowlistHosts) != 0 { + t.Errorf("%s allowlists %v, want no host", examplePath, c.AllowlistHosts) + } +} + // loadTestConfig is a helper to load a config file for testing. func loadTestConfig(path string) (*smartconfig.Config, error) { return smartconfig.NewFromConfigPath(path) From 761ade2e369f8303a64c92fe46624012e23a358d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 07:44:09 +0200 Subject: [PATCH 065/131] Accept a chunked upstream body of exactly the maximum size (closes #248) Once upstream_max_response_size bytes were read, the fetcher refused the next read without looking for the end of the body. Go's HTTP client returns the end of a body sent with a Content-Length together with its last bytes, but the end of a chunked body can come in a later read, so a chunked body of exactly the limit was refused after it was downloaded whole and the image routes answered 500. Once the limit is used up, the fetcher now reads one more byte: if the body has ended it is accepted, and if there is more it is refused with ErrResponseTooLarge. Model: opus-5-5 --- TODO.md | 9 ++ internal/httpfetcher/fetch_internal_test.go | 134 ++++++++++++++++++++ internal/httpfetcher/httpfetcher.go | 23 +++- 3 files changed, 164 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index b85bf37..c046779 100644 --- a/TODO.md +++ b/TODO.md @@ -38,6 +38,15 @@ P2: security: per-origin rate limiting commented entry under `example.com`. Getting Started and "Running under upaas" in `README.md` say that every image on an allowlisted host is served without a signature, so an allowlisted host must be one the operator controls. +- 2026-10-15 a chunked upstream body of exactly `upstream_max_response_size` + bytes is accepted (closes #248): once the limit was used up, the fetcher's + next read refused the body without looking for its end. Go's HTTP client + returns the end of a body sent with a `Content-Length` together with its last + bytes, but the end of a chunked body can come in a later read, so such a body + was refused after it had been downloaded whole, and the image routes + answered 500. Once the limit is used up, the fetcher now reads one more byte: + if the body has ended it is accepted, and if there is more it is refused with + `ErrResponseTooLarge`, as before. - 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side (closes #257): it scales the image to cover the requested size, and the side scaled past that size was held to no maximum, so `1x8192?fit=outside` of a diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index 83c78c0..c443987 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "slices" + "strconv" "strings" "sync" "testing" @@ -297,6 +298,139 @@ func TestFetchMaxResponseSizeEnforced(t *testing.T) { } } +// fetchWithMaxResponseSize serves handler as the fake upstream and fetches +// its /image route with MaxResponseSize set to maxResponseSize. +func fetchWithMaxResponseSize( + t *testing.T, + maxResponseSize int64, + handler http.HandlerFunc, +) *FetchResult { + t.Helper() + + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + cfg := DefaultConfig() + cfg.MaxResponseSize = maxResponseSize + + f, _ := newServerFetcher(t, srv, cfg) + + res := fetchImage(t, f, "/image") + t.Cleanup(func() { _ = res.Content.Close() }) + + return res +} + +// TestFetchContentLengthBodyAtMaxResponseSize checks that a body sent with a +// Content-Length is read whole at exactly MaxResponseSize bytes and refused +// at one byte more. +func TestFetchContentLengthBodyAtMaxResponseSize(t *testing.T) { + t.Parallel() + + const maxResponseSize = 8 + + for _, tc := range []struct { + size int + wantErr error + }{ + {maxResponseSize, nil}, + {maxResponseSize + 1, ErrResponseTooLarge}, + } { + t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) { + t.Parallel() + + body := strings.Repeat("x", tc.size) + + res := fetchWithMaxResponseSize(t, maxResponseSize, + func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJPEG) + w.Header().Set("Content-Length", strconv.Itoa(tc.size)) + _, _ = io.WriteString(w, body) + }) + + if res.ContentLength != int64(tc.size) { + t.Fatalf("ContentLength = %d, want %d", res.ContentLength, tc.size) + } + + data, err := io.ReadAll(res.Content) + t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err) + + if !errors.Is(err, tc.wantErr) { + t.Fatalf("read error = %v, want %v", err, tc.wantErr) + } + + if err == nil && string(data) != body { + t.Errorf("read %q, want %q", data, body) + } + }) + } +} + +// TestFetchChunkedBodyAtMaxResponseSize checks that a chunked body is read +// whole at exactly MaxResponseSize bytes and refused at one byte more. The +// upstream sends the end of the body only once the test has read +// MaxResponseSize bytes, so the read after them comes before the end of the +// body has arrived, as it can from a real upstream. +func TestFetchChunkedBodyAtMaxResponseSize(t *testing.T) { + t.Parallel() + + const maxResponseSize = 8 + + for _, tc := range []struct { + size int + wantErr error + }{ + {maxResponseSize, nil}, + {maxResponseSize + 1, ErrResponseTooLarge}, + } { + t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) { + t.Parallel() + + body := strings.Repeat("x", tc.size) + endBody := make(chan struct{}) + + res := fetchWithMaxResponseSize(t, maxResponseSize, + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", contentTypeJPEG) + _, _ = io.WriteString(w, body) + // With no Content-Length, flushing sends the body + // chunked; its end is sent when the handler returns. + _ = http.NewResponseController(w).Flush() + + select { + case <-endBody: + case <-r.Context().Done(): + } + }) + + if res.ContentLength != -1 { + t.Fatalf("ContentLength = %d, want -1 (chunked)", res.ContentLength) + } + + head := make([]byte, maxResponseSize) + + _, err := io.ReadFull(res.Content, head) + if err != nil { + t.Fatalf("read the first %d bytes: %v", maxResponseSize, err) + } + + close(endBody) + + rest, err := io.ReadAll(res.Content) + data := string(head) + string(rest) + t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err) + + if !errors.Is(err, tc.wantErr) { + t.Fatalf("read error = %v, want %v", err, tc.wantErr) + } + + if err == nil && data != body { + t.Errorf("read %q, want %q", data, body) + } + }) + } +} + func TestFetchSemaphoreReleasedOnError(t *testing.T) { t.Parallel() diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 9d09295..1428040 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -660,14 +660,33 @@ func dialSSRFSafe( return nil, fmt.Errorf("%w to %s", errConnectFailed, host) } -// limitedReader wraps a reader and limits the number of bytes read. +// limitedReader wraps a reader and refuses a body longer than remaining +// bytes with ErrResponseTooLarge. remaining goes below 0 once the body is +// found to be longer. type limitedReader struct { reader io.Reader remaining int64 } +// Read reads at most the remaining bytes. Once they are read, it reads one +// more byte to tell the end of the body from more data: at the end it +// returns the wrapped reader's error, io.EOF; with more data it returns +// ErrResponseTooLarge, as does every later Read. func (r *limitedReader) Read(p []byte) (int, error) { - if r.remaining <= 0 { + if r.remaining < 0 { + return 0, ErrResponseTooLarge + } + + if r.remaining == 0 { + var extra [1]byte + + n, err := r.reader.Read(extra[:]) + r.remaining -= int64(n) + + if n == 0 { + return 0, err + } + return 0, ErrResponseTooLarge } From 5d1e3ea871faf536f0f8414bbe22e79fd5632dbd Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 09:37:01 +0200 Subject: [PATCH 066/131] Escape : and % in the signed and cached path and query (closes #247) The signed data and the cache key joined host, path and query with ":" and no escaping, so path /a:b with no query and path /a with query b: gave the same string: a URL signed for https://h/a:b verified for https://h/a?b:, and the two shared a cached image. Both now write each % in the path and the query as %25 and each : as %3A before joining them. The host never holds / and the path always starts with /, so the joined string reads back one way only. A path and query holding neither character are signed and keyed as before, so the existing known-answer vectors and the README example are unchanged; new vectors cover the escaped cases. Signature Specification in README.md states the escaping. Model: opus-5-5 --- README.md | 7 +- TODO.md | 8 ++ internal/imgcache/storage.go | 15 ++- internal/imgcache/storage_internal_test.go | 56 ++++++++++ internal/signature/escaping_test.go | 113 +++++++++++++++++++++ internal/signature/signature.go | 23 +++-- 6 files changed, 212 insertions(+), 10 deletions(-) create mode 100644 internal/signature/escaping_test.go diff --git a/README.md b/README.md index 250463e..5fb6a49 100644 --- a/README.md +++ b/README.md @@ -419,8 +419,11 @@ HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit" Where: - `host` — source origin hostname (e.g. `cdn.example.com`) -- `path` — source path (e.g. `/photos/cat.jpg`) -- `query` — source query string, empty string if none +- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25` + and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg` +- `query` — source query string, empty string if none, with `%` and `:` written + as in `path`; neither then holds a `:`, so path `/a:b` with no query and path + `/a` with query `b:` are signed differently - `width` — requested width in pixels, `0` for original - `height` — requested height in pixels, `0` for original - `format` — output format, one of those listed under Routes, with `original` diff --git a/TODO.md b/TODO.md index c046779..7793ef4 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a `:` in the source path or query no longer moves where the path + ends and the query begins (closes #247): the signed data and the cache key + joined host, path and query with `:` and no escaping, so a URL signed for + `https://h/a:b` verified for `https://h/a?b:`; the two also shared a cache + key, and whichever was fetched first was served for both. In both, the path + and the query now have each `%` written as `%25` and each `:` as `%3A`; a path + and query holding neither are signed and keyed as before. Signature + Specification in `README.md` says so. - 2026-10-15 the example config allowlists no host (closes #249): it allowlisted `github.com` and `user-images.githubusercontent.com`, where anyone can put an image, so a deployment set up as Getting Started says served those images to diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index 395178d..35ff2f1 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -10,6 +10,7 @@ import ( "log/slog" "os" "path/filepath" + "strings" "time" ) @@ -371,11 +372,13 @@ func HashPath(path string) PathHash { // CacheKey generates a unique key for a request variant. // Format: sha256(host:path:query:width:height:format:quality:fit_mode) +// The path and query go through escapePercentAndColon, so that a ":" in +// either cannot be taken for the one between them. func CacheKey(req *ImageRequest) VariantKey { data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s", req.SourceHost, - req.SourcePath, - req.SourceQuery, + escapePercentAndColon(req.SourcePath), + escapePercentAndColon(req.SourceQuery), req.Size.Width, req.Size.Height, req.Format, @@ -387,6 +390,14 @@ func CacheKey(req *ImageRequest) VariantKey { return VariantKey(hex.EncodeToString(h[:])) } +// escapePercentAndColon returns s with each "%" written as "%25" and each +// ":" as "%3A", so that the result holds no ":". +func escapePercentAndColon(s string) string { + s = strings.ReplaceAll(s, "%", "%25") + + return strings.ReplaceAll(s, ":", "%3A") +} + // VariantStorage handles key-based file storage for processed image variants. // Files are stored at: /// // Metadata is stored at: ///.meta diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go index c57e1ee..1d3996d 100644 --- a/internal/imgcache/storage_internal_test.go +++ b/internal/imgcache/storage_internal_test.go @@ -407,6 +407,62 @@ func TestCacheKey(t *testing.T) { } } +// TestCacheKey_ColonOrPercentInPathOrQuery checks that two different source +// URLs whose path or query holds ":" or "%" get different keys. +func TestCacheKey_ColonOrPercentInPathOrQuery(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + path1, query1 string + path2, query2 string + }{ + { + name: "colon moved from path to query", + path1: "/a:b", query1: "", + path2: "/a", query2: "b:", + }, + { + name: "colon in path and its percent-encoding", + path1: "/a:b", query1: "", + path2: "/a%3Ab", query2: "", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + key1 := CacheKey(&ImageRequest{ + SourceHost: testHostCDN, + SourcePath: tt.path1, + SourceQuery: tt.query1, + Size: Size{Width: 800, Height: 600}, + Format: FormatWebP, + Quality: 85, + FitMode: FitCover, + }) + key2 := CacheKey(&ImageRequest{ + SourceHost: testHostCDN, + SourcePath: tt.path2, + SourceQuery: tt.query2, + Size: Size{Width: 800, Height: 600}, + Format: FormatWebP, + Quality: 85, + FitMode: FitCover, + }) + + t.Logf("CacheKey() of path %q query %q = %s", tt.path1, tt.query1, key1) + t.Logf("CacheKey() of path %q query %q = %s", tt.path2, tt.query2, key2) + + if key1 == key2 { + t.Errorf("CacheKey() gave path %q query %q and path %q query %q "+ + "the same key", tt.path1, tt.query1, tt.path2, tt.query2) + } + }) + } +} + // TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write // that fails is logged at warn and the store still succeeds. func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { diff --git a/internal/signature/escaping_test.go b/internal/signature/escaping_test.go new file mode 100644 index 0000000..550805c --- /dev/null +++ b/internal/signature/escaping_test.go @@ -0,0 +1,113 @@ +package signature_test + +import ( + "errors" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/signature" +) + +// TestSigner_Verify_ColonCannotMovePathQueryBoundary checks that a URL +// signed for path "/a:b" with no query does not verify for path "/a" with +// query "b:", a different resource on the same host. +func TestSigner_Verify_ColonCannotMovePathQueryBoundary(t *testing.T) { + t.Parallel() + + signer := signature.New(goldenSigningKey) + + signed := &signature.Request{ + SourceHost: testHost, + SourcePath: "/a:b", + Width: 800, + Height: 600, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + Expires: time.Now().Add(1 * time.Hour), + } + signed.Signature = signer.Sign(signed) + + replayed := *signed + replayed.SourcePath = "/a" + replayed.SourceQuery = "b:" + + t.Logf("signature for path %q query %q: %s", + signed.SourcePath, signed.SourceQuery, signed.Signature) + t.Logf("signature for path %q query %q: %s", + replayed.SourcePath, replayed.SourceQuery, signer.Sign(&replayed)) + + err := signer.Verify(&replayed) + if !errors.Is(err, signature.ErrInvalid) { + t.Errorf("Verify() of path %q query %q with the signature for "+ + "path %q = %v, want %v", + replayed.SourcePath, replayed.SourceQuery, signed.SourcePath, + err, signature.ErrInvalid) + } +} + +// TestSigner_EscapedGoldenVectors pins the signature of requests whose path +// or query holds ":" or "%", which are signed with "%" written as "%25" and +// ":" as "%3A". The expected values were computed once, outside pixa, from +// the signed data in each comment. +func TestSigner_EscapedGoldenVectors(t *testing.T) { + t.Parallel() + + signer := signature.New(goldenSigningKey) + + vectors := []struct { + name string + path string + query string + wantSignature string + }{ + { + name: "colon in path", + path: "/photos/a:b.jpg", + // Signed data: + // "cdn.example.com:/photos/a%3Ab.jpg::800:600:webp:1704067200:85:cover" + wantSignature: "WnEpZ7bQp1Zvw8WDatlj2vu-4wYKWE3BFBizV8m__6k=", + }, + { + name: "colon in query", + path: testPath, + query: "t=1:2", + // Signed data: + // "cdn.example.com:/photos/cat.jpg:t=1%3A2:800:600:webp:1704067200:85:cover" + wantSignature: "85kRoN8hhr4cCUkEUjNvUsvVYKjYYfWdeffx5TjJjqA=", + }, + { + name: "percent in path and query", + path: "/photos/a%20b.jpg", + query: "q=50%", + // Signed data: + // "cdn.example.com:/photos/a%2520b.jpg:q=50%25:800:600:webp:1704067200:85:cover" + wantSignature: "MjDiiRdgXMrsjqPMsENtD4aG1hp1e-a6zy67vKTX4CQ=", + }, + } + + for _, tt := range vectors { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + req := &signature.Request{ + SourceHost: testHost, + SourcePath: tt.path, + SourceQuery: tt.query, + Width: 800, + Height: 600, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + Expires: time.Unix(goldenExpiresUnix, 0), + } + + gotSignature := signer.Sign(req) + t.Logf("Sign() of path %q query %q = %s", tt.path, tt.query, gotSignature) + + if gotSignature != tt.wantSignature { + t.Errorf("Sign() = %q, want %q", gotSignature, tt.wantSignature) + } + }) + } +} diff --git a/internal/signature/signature.go b/internal/signature/signature.go index 0b43032..a683e39 100644 --- a/internal/signature/signature.go +++ b/internal/signature/signature.go @@ -10,6 +10,7 @@ import ( "fmt" "net/url" "strconv" + "strings" "time" ) @@ -154,15 +155,17 @@ func (s *Signer) GenerateSignedURL( // buildSignatureData creates the string to be signed. // Format: "host:path:query:width:height:format:expiration:quality:fit" -// All components are used verbatim (exact match). No normalization, -// suffix matching, or wildcard expansion is performed. Quality and fit -// are the effective transform values, so replaying a signed URL with a -// different quality or fit mode fails verification. +// The path and query go through escapePercentAndColon, so that a ":" in +// either cannot be taken for the one between them; every other component +// is used verbatim. No normalization, suffix matching, or wildcard +// expansion is performed. Quality and fit are the effective transform +// values, so replaying a signed URL with a different quality or fit mode +// fails verification. func (s *Signer) buildSignatureData(req *Request) string { return fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%d:%s", req.SourceHost, - req.SourcePath, - req.SourceQuery, + escapePercentAndColon(req.SourcePath), + escapePercentAndColon(req.SourceQuery), req.Width, req.Height, req.Format, @@ -172,6 +175,14 @@ func (s *Signer) buildSignatureData(req *Request) string { ) } +// escapePercentAndColon returns s with each "%" written as "%25" and each +// ":" as "%3A", so that the result holds no ":". +func escapePercentAndColon(s string) string { + s = strings.ReplaceAll(s, "%", "%25") + + return strings.ReplaceAll(s, ":", "%3A") +} + // ParseParams extracts signature and expiration from query parameters. func ParseParams(sig, expStr string) (string, time.Time, error) { if expStr == "" { From c23585dd6daf86d0307ad656a16a8945a8bc7c3b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 12:31:14 +0200 Subject: [PATCH 067/131] Lower-case the image URL's host and require a signature for a port (closes #246) With `cdn.example.com` allowlisted, other letter cases and a host with a port also passed the allowlist; each spelling was fetched and cached on its own with its own per-host connection limit, and a port let an unsigned URL reach any port of the host. The image URL parser now lower-cases the host and refuses one holding `@` with 400. A host followed by `:`, with or without a port (read as Go's URL parser reads it, unbracketed IPv6 included), never counts as allowlisted, so it needs a signature. The host is signed lower-cased, as the URL writes it, so a signature made over an upper-case spelling is refused with 401. Model: opus-5-5 --- README.md | 10 ++- TODO.md | 11 +++ internal/imgcache/service.go | 12 ++- internal/imgcache/service_internal_test.go | 91 ++++++++++++++++++++ internal/imgcache/urlparser.go | 12 +++ internal/imgcache/urlparser_internal_test.go | 43 +++++++++ 6 files changed, 177 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5fb6a49..609ad9e 100644 --- a/README.md +++ b/README.md @@ -418,7 +418,9 @@ HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit" Where: -- `host` — source origin hostname (e.g. `cdn.example.com`) +- `host` — source host as the URL writes it up to the first `/`, in lower case, + keeping an IPv6 address's brackets and any `:` and port after it (e.g. + `cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`) - `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25` and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg` - `query` — source query string, empty string if none, with `%` and `:` written @@ -473,6 +475,12 @@ entry that is neither a host name (letters, digits, hyphens, underscores and dots, with at most one leading dot) nor an IP address, such as one with a port or a `*.` wildcard, aborts startup. +pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so +`CDN.Example.com` is matched, fetched, cached and counted against +`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has +a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature +even when the host is allowlisted. A host holding `@` is refused with 400. + ### Configuration Every setting can be given as an environment variable, in a YAML config file diff --git a/TODO.md b/TODO.md index 7793ef4..d1ccf51 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port + needs a signature (closes #246): with `cdn.example.com` allowlisted, + `CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and + each spelling was fetched and cached on its own and had connections of its own + under `upstream_connections_per_host`. The image URL parser now lower-cases + the host and refuses one holding `@`, which a URL reads as a user name and + password, with 400. A host with a port, or with a `:` and no port, needs a + signature even when it is allowlisted. That is checked in the image service, + not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a + `Referer` is blocked whatever its port. A URL is signed over its host as it + writes it, in lower case, with any IPv6 brackets, `:` and port kept. - 2026-10-15 a `:` in the source path or query no longer moves where the path ends and the query begins (closes #247): the signed data and the cache key joined host, path and query with `:` and no escaping, so a URL signed for diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 52684a9..01f0db3 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -9,6 +9,7 @@ import ( "log/slog" "net/url" "runtime/debug" + "strings" "time" "github.com/dustin/go-humanize" @@ -218,7 +219,16 @@ func (s *Service) ValidateRequest(req *ImageRequest) error { return fmt.Errorf("invalid source URL: %w", err) } - if s.allowlist.IsAllowed(parsedURL) { + // An allowlist entry names no port, so a host followed by ":", with or + // without a port, needs a signature. The port is read from the parsed + // URL, as the fetcher reads it: in an IPv6 address written without + // brackets, such as 2001:db8::1:8443, it is what follows the last ":". + // This is checked here and not in allowlist.IsAllowed, which also matches + // referer_blocklist, where a Referer must be blocked whatever its port. + hostHasPort := parsedURL.Port() != "" || + strings.HasSuffix(parsedURL.Host, ":") + + if !hostHasPort && s.allowlist.IsAllowed(parsedURL) { return nil } diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index 85ad9dd..e4bbb30 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -250,6 +250,97 @@ func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) { } } +// TestService_ValidateRequest_HostWithPortNeedsSignature verifies that a host +// followed by a port, or by a colon with no port, matches no allowlist entry, +// so that a request for it without a signature is refused, while the same +// host with no port is allowlisted. +func TestService_ValidateRequest_HostWithPortNeedsSignature(t *testing.T) { + t.Parallel() + + svc, _ := SetupTestService(t, + WithSigningKey(testSigningKey), + WithAllowlist(testHostCDN, "203.0.113.5", "2001:db8::1"), + ) + + tests := []struct { + host string + allowlisted bool + }{ + {testHostCDN, true}, + {"cdn.example.com:8443", false}, + {"cdn.example.com:443", false}, + {"cdn.example.com:", false}, + {"203.0.113.5", true}, + {"203.0.113.5:8080", false}, + {"[2001:db8::1]", true}, + {"[2001:db8::1]:8443", false}, + } + + for _, tt := range tests { + t.Run(tt.host, func(t *testing.T) { + t.Parallel() + + req := &ImageRequest{ + SourceHost: tt.host, + SourcePath: testPathCat, + Size: Size{Width: 50, Height: 50}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } + + err := svc.ValidateRequest(req) + t.Logf("ValidateRequest() for host %q with no signature: %v", + tt.host, err) + + if allowlisted := err == nil; allowlisted != tt.allowlisted { + t.Errorf("host %q served without a signature = %v, want %v", + tt.host, allowlisted, tt.allowlisted) + } + }) + } +} + +// TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature verifies that +// an allowlisted IPv6 address written without brackets and followed by a +// port, or by a colon with no port, needs a signature. Go's URL parser, which +// the fetcher uses, reads 2001:db8::1:8443 as the host 2001:db8::1 with the +// port 8443. +func TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature( + t *testing.T, +) { + t.Parallel() + + svc, _ := SetupTestService(t, + WithSigningKey(testSigningKey), + WithAllowlist("2001:db8::1"), + ) + + for _, host := range []string{"2001:db8::1:8443", "2001:db8::1:"} { + t.Run(host, func(t *testing.T) { + t.Parallel() + + req := &ImageRequest{ + SourceHost: host, + SourcePath: testPathCat, + Size: Size{Width: 50, Height: 50}, + Format: FormatJPEG, + Quality: 85, + FitMode: FitCover, + } + + err := svc.ValidateRequest(req) + t.Logf("ValidateRequest() for host %q with no signature: %v", + host, err) + + if err == nil { + t.Errorf("host %q served without a signature, want refused", + host) + } + }) + } +} + func TestService_Get_InvalidFile(t *testing.T) { t.Parallel() diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index 0d5e6c5..4247be6 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -15,6 +15,7 @@ import ( var ( ErrInvalidPath = errors.New("invalid image path") ErrMissingHost = errors.New("missing source host") + ErrInvalidHost = errors.New("invalid source host") ErrMissingSize = errors.New("missing size specification") ErrInvalidSize = errors.New("invalid size format") ErrInvalidFormat = errors.New("invalid or unsupported format") @@ -143,6 +144,17 @@ func parseImageComponents(remainder string) (*ParsedURL, error) { return nil, ErrMissingHost } + // A URL reads what comes before "@" as a user name and password, and the + // rest as the host + if strings.Contains(host, "@") { + return nil, fmt.Errorf("%w: %q holds @", ErrInvalidHost, host) + } + + // Host names do not depend on letter case, so the allowlist, the upstream + // connection limits, the cache and the signature see one spelling of each + // host + host = strings.ToLower(host) + // Extract query string if present in path if qIndex := strings.Index(path, "?"); qIndex != -1 { query = path[qIndex+1:] diff --git a/internal/imgcache/urlparser_internal_test.go b/internal/imgcache/urlparser_internal_test.go index 9f4e68d..518a04e 100644 --- a/internal/imgcache/urlparser_internal_test.go +++ b/internal/imgcache/urlparser_internal_test.go @@ -234,6 +234,49 @@ func TestParseImagePath(t *testing.T) { } } +// TestParseImagePath_LowerCasesHost verifies that the host is lower-cased, so +// that every spelling of its letter case is the same host to the allowlist, +// the upstream connection limits and the cache, and that the path keeps its +// case. +func TestParseImagePath_LowerCasesHost(t *testing.T) { + t.Parallel() + + got, err := ParseImagePath("CDN.Example.COM/Photos/Cat.jpg/800x600.webp") + if err != nil { + t.Fatalf("ParseImagePath() error = %v", err) + } + + t.Logf("parsed host %q, path %q", got.Host, got.Path) + + if got.Host != testHostCDN { + t.Errorf("Host = %q, want %q", got.Host, testHostCDN) + } + + if got.Path != "/Photos/Cat.jpg" { + t.Errorf("Path = %q, want %q", got.Path, "/Photos/Cat.jpg") + } +} + +// TestParseImagePath_RefusesUserinfo verifies that a host holding "@" is +// refused: a URL reads what comes before the "@" as a user name and password +// and the rest as the host. +func TestParseImagePath_RefusesUserinfo(t *testing.T) { + t.Parallel() + + for _, input := range []string{ + "user@cdn.example.com/photos/cat.jpg/800x600.webp", + "user:password@cdn.example.com/photos/cat.jpg/800x600.webp", + } { + got, err := ParseImagePath(input) + t.Logf("ParseImagePath(%q) = %+v, error = %v", input, got, err) + + if !errors.Is(err, ErrInvalidHost) { + t.Errorf("ParseImagePath(%q) error = %v, want %v", + input, err, ErrInvalidHost) + } + } +} + func TestParsedURL_ToImageRequest(t *testing.T) { t.Parallel() From 2790125cb274d1a90bb1def85110842bc42fb801 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 13:45:25 +0200 Subject: [PATCH 068/131] Serve a cached variant while its source URL is in the negative cache (closes #245) Get checked the negative cache before the variant cache, so after one failed fetch of a source every variant of it answered 502 for five minutes, also one on disk or one that could be made from the cached source. The check now sits at the start of fetchAndProcess, which runs only when no cached source can be used. A refusal from the negative cache now counts as a miss. Model: opus-5-5 --- README.md | 7 +- TODO.md | 10 +++ .../imgcache/negative_cache_internal_test.go | 65 +++++++++++++++++++ internal/imgcache/service.go | 36 +++++----- 4 files changed, 99 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 609ad9e..9dc720d 100644 --- a/README.md +++ b/README.md @@ -200,9 +200,10 @@ path under `/v1/` answers 200, in maintenance mode too. `image_requests_per_minute` (below); 415 when the upstream answers with a `Content-Type` that is not one of the source formats (below), or with an image whose first bytes libvips reads as another format; 502 when the upstream - answered with an error status, and for 5 minutes after that for the same - source URL; 503 when pixa is busy or in maintenance mode; 500 for any other - failure. + answered with an error status, and for 5 minutes after that for an image of + the same source URL that pixa can neither serve from its cache nor make from + the source image still in its cache; 503 when pixa is busy or in maintenance + mode; 500 for any other failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when `If-None-Match` matches the image's `ETag`; 400 for a token that does not diff --git a/TODO.md b/TODO.md index d1ccf51..712a369 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,16 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a cached variant is served while its source URL is in the negative + cache (closes #245): `Get` checked the negative cache before the variant + cache, so once a fetch of the source failed, as one does when another variant + misses after the source was evicted and the upstream is down, every variant of + that source answered 502 for 5 minutes, also one on disk or one that could be + made from the cached source. The negative cache is now checked only just + before a fetch from the upstream, so a cached variant is served and a variant + not in the cache is made from the cached source when there is one. `README.md` + says so under `/v1/image/`. A refusal from the negative cache now counts as a + miss, as any other request that misses the variant cache does. - 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port needs a signature (closes #246): with `cdn.example.com` allowlisted, `CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index f2c2115..ffb6079 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -94,6 +94,71 @@ func TestNegativeCache_Expired(t *testing.T) { } } +// TestService_Get_ServesCachedVariantOfNegativeCachedURL caches a variant of +// the test photo, then keeps a failed fetch of the photo's URL in the negative +// cache, as a failed fetch for another variant does once the source has been +// evicted. The cached variant is still served, and a variant not in the cache +// is made from the cached source. Once the source is evicted, a variant not in +// the cache needs a fetch and is refused. +func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + + resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover)) + if err != nil { + t.Fatalf("Get() first request error = %v", err) + } + + _ = resp.Content.Close() + + err = svc.cache.StoreNegative(t.Context(), photoVariant(fixtures, 85, FitCover), + httpStatusBadGateway, "upstream answered 503") + if err != nil { + t.Fatalf("StoreNegative() error = %v", err) + } + + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover)) + if err != nil { + t.Fatalf("Get() of the cached variant error = %v", err) + } + + _ = resp.Content.Close() + + if resp.CacheStatus != CacheHit { + t.Errorf("cached variant CacheStatus = %q, want %q", + resp.CacheStatus, CacheHit) + } + + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, FitCover)) + t.Logf("Get() of a variant made from the cached source: error = %v", err) + + if err != nil { + t.Fatalf("Get() of a variant made from the cached source error = %v", err) + } + + _ = resp.Content.Close() + + contentHash, _, err := svc.cache.LookupSource(t.Context(), + photoVariant(fixtures, 85, FitCover)) + if err != nil || contentHash == "" { + t.Fatalf("LookupSource() = %q, %v, want the cached source", contentHash, err) + } + + err = svc.cache.evictSourceBlob(t.Context(), contentHash) + if err != nil { + t.Fatalf("evictSourceBlob() error = %v", err) + } + + _, err = svc.Get(t.Context(), photoVariant(fixtures, 60, FitCover)) + t.Logf("Get() of a variant whose source was evicted: error = %v", err) + + if !errors.Is(err, ErrNegativeCached) { + t.Errorf("Get() of a variant whose source was evicted error = %v, want %v", + err, ErrNegativeCached) + } +} + func TestService_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) { t.Parallel() diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 01f0db3..57f4e82 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -130,25 +130,13 @@ func NewService(cfg *ServiceConfig) (*Service, error) { var ErrNegativeCached = errors.New("request is in negative cache (recently failed)") // Get retrieves a processed image, fetching and processing if necessary. +// While its source URL is in the negative cache, a cached variant is still +// served and a variant not in the cache is still made from a cached source; +// only a request that needs the source fetched is refused. func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) { // Propagate AllowHTTP setting to the request req.AllowHTTP = s.allowHTTP - // Check negative cache first - skip fetching for recently-failed URLs - negHit, err := s.cache.checkNegativeCache(ctx, req) - if err != nil { - s.log.Warn("negative cache check failed", "error", err) - } - - if negHit { - s.log.Debug("negative cache hit", - "host", req.SourceHost, - "path", req.SourcePath, - ) - - return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached) - } - // Check variant cache first (disk only, no DB) result, err := s.cache.Lookup(ctx, req) if err != nil { @@ -433,12 +421,28 @@ func (s *Service) processFromSourceOrFetch( // fetchAndProcess fetches from upstream, processes, and caches the result. // It counts the fetch with the bytes read from upstream, including when -// reading the response or a later step fails. +// reading the response or a later step fails. It fetches nothing while the +// source URL is in the negative cache. func (s *Service) fetchAndProcess( ctx context.Context, req *ImageRequest, cacheKey VariantKey, ) (*processedVariant, error) { + // Check negative cache - skip fetching for recently-failed URLs + negHit, err := s.cache.checkNegativeCache(ctx, req) + if err != nil { + s.log.Warn("negative cache check failed", "error", err) + } + + if negHit { + s.log.Debug("negative cache hit", + "host", req.SourceHost, + "path", req.SourcePath, + ) + + return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached) + } + // Fetch from upstream sourceURL := req.SourceURL() From 69f65d43028042d177d361c85e00e942b35bce98 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 15:02:27 +0200 Subject: [PATCH 069/131] Remove expired negative cache rows in the eviction loop (closes #244) A negative cache row was deleted only when its URL was requested again, so a client asking for a new missing path each time grew the database without limit. The eviction goroutine now removes expired rows at startup and on every tick, so a row stays about NegativeTTL plus one eviction interval. It deletes at most 1000 rows per statement, so a request on the one database connection waits behind one batch at most. The goroutine now also runs when the disk cache is disabled, since the negative cache is used either way. Model: opus-5-5 --- TODO.md | 11 + internal/handlers/handlers.go | 3 +- internal/imgcache/cache.go | 63 ++++- internal/imgcache/eviction.go | 32 ++- .../imgcache/negative_cache_internal_test.go | 237 ++++++++++++++++++ 5 files changed, 327 insertions(+), 19 deletions(-) diff --git a/TODO.md b/TODO.md index 712a369..8d09a38 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 expired negative cache rows are removed on a schedule (closes + #244): the row of a failed fetch, which makes pixa answer 502 for that source + URL for 5 minutes, was deleted only when the same URL was requested again + after that, so a client asking for a new missing path each time added rows + that stayed for good. The eviction goroutine now calls `CleanExpired` at + startup and on each periodic tick, every 5 minutes in pixad, so a row stays at + most about 10 minutes. `CleanExpired` deletes at most 1000 rows per statement, + so a request waits behind 1000 deletions at most, however many rows have + expired. The goroutine runs with the disk cache disabled + (`cache_max_bytes: 0`) too, where it does nothing else, as the negative cache + is used either way. - 2026-10-15 a cached variant is served while its source URL is in the negative cache (closes #245): `Get` checked the negative cache before the variant cache, so once a fetch of the source failed, as one does when another variant diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 297ba2d..da9572f 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -124,7 +124,8 @@ func (s *Handlers) initImageService() error { s.imgCache = cache // Background eviction: startup reconciliation, then periodic and - // write-pressure passes. No-op when the disk cache is disabled. + // write-pressure passes, and the removal of expired negative cache + // rows, which is all it does when the disk cache is disabled. cache.StartEviction(imgcache.DefaultEvictionInterval) // Writes the counts requests could not write by their deadline diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 1f73019..dc49692 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -32,6 +32,12 @@ const httpStatusOK = 200 // variant not among them is served as before, reading its .meta file. const metaCacheSize = 10000 +// cleanExpiredBatchSize is the most expired negative cache rows one +// statement of CleanExpired deletes. Each batch is a statement of its +// own, so a request waits behind one batch at most, however many rows +// have expired. +const cleanExpiredBatchSize = 1000 + // CacheConfig holds cache configuration. type CacheConfig struct { StateDir string @@ -50,8 +56,9 @@ type CacheConfig struct { // DisableDiskCache turns the disk cache off entirely: no cache // directories are created, lookups always miss, stores are - // no-ops, and no eviction machinery runs. The config layer sets - // this when the operator configures cache_max_bytes: 0. + // no-ops, and the eviction goroutine only removes the expired + // negative cache rows. The config layer sets this when the + // operator configures cache_max_bytes: 0. DisableDiskCache bool // Logger receives accounting and eviction log output. A nil @@ -482,17 +489,27 @@ func (c *Cache) GetSourceContent( return c.srcContent.LoadWithSize(contentHash) } -// CleanExpired removes expired entries from the cache. +// CleanExpired removes the negative cache rows past their expiry, one +// batch per statement, until none is left. Once ctx ends it starts no +// further batch and returns ctx's error. The eviction goroutine calls it +// at startup and on every periodic tick, so a row stays about +// NegativeTTL plus one eviction interval, whether or not its URL is +// requested again. func (c *Cache) CleanExpired(ctx context.Context) error { - // Clean expired negative cache entries - _, err := c.db.ExecContext(ctx, ` - DELETE FROM negative_cache WHERE expires_at < CURRENT_TIMESTAMP - `) - if err != nil { - return fmt.Errorf("failed to clean negative cache: %w", err) - } + for { + if ctx.Err() != nil { + return ctx.Err() + } - return nil + deleted, err := c.cleanExpiredBatch(ctx) + if err != nil { + return err + } + + if deleted < cleanExpiredBatchSize { + return nil + } + } } // Stats returns cache statistics. @@ -756,3 +773,27 @@ func (c *Cache) checkNegativeCache( return true, nil } + +// cleanExpiredBatch deletes up to cleanExpiredBatchSize negative cache +// rows past their expiry in one statement, and returns how many it +// deleted. DELETE ... LIMIT needs a SQLite build option, so the subquery +// picks the rows. +func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) { + result, err := c.db.ExecContext(ctx, ` + DELETE FROM negative_cache WHERE id IN ( + SELECT id FROM negative_cache + WHERE expires_at < CURRENT_TIMESTAMP + LIMIT ? + ) + `, cleanExpiredBatchSize) + if err != nil { + return 0, fmt.Errorf("failed to clean negative cache: %w", err) + } + + deleted, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to count cleaned negative cache rows: %w", err) + } + + return deleted, nil +} diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index a5aff55..71f1fb0 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -437,14 +437,16 @@ func (c *Cache) notifyWritePressure() { } // StartEviction launches the background eviction goroutine, which -// reconciles the database accounting with the cache directories at -// startup and again on every periodic tick thereafter, and evicts to -// the configured limit on the given periodic interval and on -// write-pressure notifications. It is a no-op on a disabled cache or -// when already started. The goroutine outlives the caller, so it runs -// with its own context, which StopEviction cancels. +// reconciles the database accounting with the cache directories and +// removes the expired negative cache rows at startup and again on every +// periodic tick thereafter, and evicts to the configured limit on the +// given periodic interval and on write-pressure notifications. On a +// disabled cache it only removes the expired negative cache rows, as +// the negative cache is used either way. It is a no-op when already +// started. The goroutine outlives the caller, so it runs with its own +// context, which StopEviction cancels. func (c *Cache) StartEviction(interval time.Duration) { - if c.disabled || c.evictionCancel != nil { + if c.evictionCancel != nil { return } @@ -480,6 +482,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { defer close(c.evictionDone) c.runReconciliationPass(ctx) + c.runCleanExpired(ctx) c.runEvictionPass(ctx) ticker := time.NewTicker(interval) @@ -500,6 +503,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { // unaccounted for to one interval, on a process that is // already running this loop regardless. c.runReconciliationPass(ctx) + c.runCleanExpired(ctx) case <-c.evictionPressure: } @@ -535,6 +539,20 @@ func (c *Cache) runReconciliationPass(ctx context.Context) { } } +// runCleanExpired runs CleanExpired, logging a failure instead of +// propagating it (the loop must keep running). It does nothing once ctx +// is cancelled. +func (c *Cache) runCleanExpired(ctx context.Context) { + if ctx.Err() != nil { + return + } + + err := c.CleanExpired(ctx) + if err != nil { + c.log.Warn("negative cache cleanup failed", "error", err) + } +} + // reconcileAccounting synchronizes the database size accounting with // the actual contents of the cache directories. It runs at startup and // again on every periodic eviction tick thereafter, off the request diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index ffb6079..dae2433 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -3,6 +3,7 @@ package imgcache import ( "context" "errors" + "strconv" "testing" "time" ) @@ -159,6 +160,242 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { } } +// TestEvictionLoopRemovesExpiredNegativeCacheRows stores a failed fetch +// that expires a second later and starts the eviction loop with a short +// interval: a later tick must remove its row although its URL is never +// requested again, and must keep a row that has not expired. It runs +// with the disk cache enabled and disabled, as the negative cache is +// used either way. +func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + disableDiskCache bool + }{ + {name: "disk cache enabled", disableDiskCache: false}, + {name: "disk cache disabled", disableDiskCache: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cache, err := NewCache(evictionTestDB(t), CacheConfig{ + StateDir: t.TempDir(), + NegativeTTL: time.Second, + DisableDiskCache: tt.disableDiskCache, + }) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + expired := &ImageRequest{ + SourceHost: testHostExample, SourcePath: "/gone.jpg", + } + + err = cache.StoreNegative(t.Context(), expired, 404, "not found") + if err != nil { + t.Fatalf("StoreNegative failed: %v", err) + } + + const notExpiredPath = "/later.jpg" + + _, err = cache.db.ExecContext(t.Context(), ` + INSERT INTO negative_cache + (source_host, source_path, status_code, expires_at) + VALUES (?, ?, 404, datetime('now', '+1 hour')) + `, testHostExample, notExpiredPath) + if err != nil { + t.Fatalf("failed to insert a row that has not expired: %v", err) + } + + started := time.Now() + + cache.StartEviction(100 * time.Millisecond) + defer func() { _ = cache.StopEviction(t.Context()) }() + + for negativeCacheRowCount(t, cache, expired.SourcePath) > 0 { + if time.Since(started) > 10*time.Second { + t.Fatal("the expired row is still in negative_cache") + } + + time.Sleep(50 * time.Millisecond) + } + + t.Logf("the expired row was removed %s after the loop started", + time.Since(started)) + + if negativeCacheRowCount(t, cache, notExpiredPath) != 1 { + t.Error("the row that has not expired was removed") + } + }) + } +} + +// negativeCacheRowCount returns how many negative_cache rows hold +// sourcePath. +func negativeCacheRowCount(tb testing.TB, cache *Cache, sourcePath string) int { + tb.Helper() + + var count int + + err := cache.db.QueryRowContext(tb.Context(), + `SELECT COUNT(*) FROM negative_cache WHERE source_path = ?`, + sourcePath, + ).Scan(&count) + if err != nil { + tb.Fatalf("failed to count negative_cache rows: %v", err) + } + + return count +} + +// TestCleanExpiredDeletesInBatches stores one expired negative cache row +// more than one batch holds, and one row that has not expired. One +// statement must delete one batch and no more, so a request waits behind +// one batch at most; CleanExpired must then remove the expired row left +// over and keep the row that has not expired. +func TestCleanExpiredDeletesInBatches(t *testing.T) { + t.Parallel() + + cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + ctx := t.Context() + + insertExpiredNegativeCacheRows(t, cache, cleanExpiredBatchSize+1) + + const notExpiredPath = "/later.jpg" + + _, err = cache.db.ExecContext(ctx, ` + INSERT INTO negative_cache + (source_host, source_path, status_code, expires_at) + VALUES (?, ?, 404, datetime('now', '+1 hour')) + `, testHostExample, notExpiredPath) + if err != nil { + t.Fatalf("failed to insert a row that has not expired: %v", err) + } + + deleted, err := cache.cleanExpiredBatch(ctx) + if err != nil { + t.Fatalf("cleanExpiredBatch failed: %v", err) + } + + t.Logf("one statement deleted %d of %d expired rows", + deleted, cleanExpiredBatchSize+1) + + if deleted != cleanExpiredBatchSize { + t.Errorf("one statement deleted %d rows, want %d", + deleted, cleanExpiredBatchSize) + } + + err = cache.CleanExpired(ctx) + if err != nil { + t.Fatalf("CleanExpired failed: %v", err) + } + + if count := negativeCacheTableRowCount(t, cache); count != 1 { + t.Errorf("negative_cache holds %d rows after CleanExpired, want 1", count) + } + + if negativeCacheRowCount(t, cache, notExpiredPath) != 1 { + t.Error("the row that has not expired was removed") + } +} + +// TestCleanExpiredRemovesMoreThanOneBatch stores one expired negative +// cache row more than one batch holds and calls CleanExpired once: it +// must remove them all, so it fails if CleanExpired stops after its +// first batch. +func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) { + t.Parallel() + + cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + insertExpiredNegativeCacheRows(t, cache, cleanExpiredBatchSize+1) + + before := negativeCacheTableRowCount(t, cache) + + err = cache.CleanExpired(t.Context()) + if err != nil { + t.Fatalf("CleanExpired failed: %v", err) + } + + after := negativeCacheTableRowCount(t, cache) + + t.Logf("CleanExpired removed %d of %d expired rows, %d per batch", + before-after, before, cleanExpiredBatchSize) + + if after != 0 { + t.Errorf("negative_cache holds %d rows after CleanExpired, want 0", after) + } +} + +// TestCleanExpiredStopsOnceContextEnds runs CleanExpired with a context +// that has already ended: it must return the context's error and delete +// no row. +func TestCleanExpiredStopsOnceContextEnds(t *testing.T) { + t.Parallel() + + cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + insertExpiredNegativeCacheRows(t, cache, 1) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + err = cache.CleanExpired(ctx) + if !errors.Is(err, context.Canceled) { + t.Errorf("CleanExpired() error = %v, want context.Canceled", err) + } + + if count := negativeCacheTableRowCount(t, cache); count != 1 { + t.Errorf("negative_cache holds %d rows, want the expired row kept", count) + } +} + +// insertExpiredNegativeCacheRows inserts count negative_cache rows that +// expired an hour ago, each for a different source path. +func insertExpiredNegativeCacheRows(t *testing.T, cache *Cache, count int) { + t.Helper() + + for i := range count { + _, err := cache.db.ExecContext(t.Context(), ` + INSERT INTO negative_cache + (source_host, source_path, status_code, expires_at) + VALUES (?, ?, 404, datetime('now', '-1 hour')) + `, testHostExample, "/expired/"+strconv.Itoa(i)+".jpg") + if err != nil { + t.Fatalf("failed to insert an expired row: %v", err) + } + } +} + +// negativeCacheTableRowCount returns how many rows negative_cache holds. +func negativeCacheTableRowCount(tb testing.TB, cache *Cache) int { + tb.Helper() + + var count int + + err := cache.db.QueryRowContext(tb.Context(), + `SELECT COUNT(*) FROM negative_cache`, + ).Scan(&count) + if err != nil { + tb.Fatalf("failed to count negative_cache rows: %v", err) + } + + return count +} + func TestService_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) { t.Parallel() From 7956da1f59d8499dbc5baab9649a115dffe67df8 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 16:37:20 +0200 Subject: [PATCH 070/131] Label request metrics by route, not by path (closes #254) The metrics middleware labelled each request with its path, so every new path added series to the registry that were never removed, and /metrics listed the token of every encrypted URL requested. The handler label is now the pattern of the chi route the request will match, looked up before routing, or "unmatched" when none does. /v1 and /v1/ ask the image subrouter, and an empty path is looked up as /, as chi routes it. TestMetricsRecordsServedRequest expects "unmatched" in place of "/test", as the owner approved. Model: opus-5-5 --- README.md | 5 +- TODO.md | 9 ++ internal/middleware/middleware.go | 61 +++++++++++- .../middleware/middleware_internal_test.go | 96 ++++++++++++++++++- 4 files changed, 166 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9dc720d..b39b3ff 100644 --- a/README.md +++ b/README.md @@ -640,7 +640,10 @@ See `configs/config.example.yml` for all options with defaults. - **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics (duration, response size, requests in flight) and the Go runtime and process metrics; requests are measured and `/metrics` is served only when - `metrics.username` and `metrics.password` are set + `metrics.username` and `metrics.password` are set. A request's `handler` label + is the pattern of the route it matches, such as `/v1/image/*`, never its path. + Requests that match no route share the label `unmatched`: those the router + answers 404 or 405, and CORS preflights - **Logging**: stdlib slog ## Entrypoints diff --git a/TODO.md b/TODO.md index 8d09a38..893924a 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the request metrics are labelled by route, not by path (closes + #254): their `handler` label was each request's path, so every new path, a 404 + or an encrypted URL, added series to the Prometheus registry that were never + removed, and `/metrics` listed every encrypted URL's token. The label is now + the pattern of the chi route the request matches, such as `/v1/image/*`, or + `unmatched` for a request that matches none: one the router answers 404 or + 405, or a CORS preflight. The metrics library takes the label before the + request is routed, so the metrics middleware asks the router which route + matches. - 2026-10-15 expired negative cache rows are removed on a schedule (closes #244): the row of a failed fetch, which makes pixa answer 502 for that source URL for 5 minutes, was deleted only when the same URL was requested again diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index f6eb83d..a35c23e 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -8,12 +8,15 @@ import ( "net/http" "net/netip" "regexp" + "strings" "time" basicauth "github.com/99designs/basicauth-go" + "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/cors" "github.com/go-chi/httprate" + "github.com/prometheus/client_golang/prometheus" metrics "github.com/slok/go-http-metrics/metrics/prometheus" ghmm "github.com/slok/go-http-metrics/middleware" "github.com/slok/go-http-metrics/middleware/std" @@ -63,6 +66,10 @@ type Middleware struct { log *slog.Logger config *config.Config clientIP *clientip.Resolver + + // metricsRegistry is the registry Metrics registers its metrics with. + // nil is Prometheus' default registry, the one /metrics serves. + metricsRegistry prometheus.Registerer } // New creates a new Middleware instance. @@ -209,17 +216,65 @@ func (s *Middleware) CORS() func(http.Handler) http.Handler { }) } -// Metrics returns a Prometheus metrics middleware. +// Metrics returns a Prometheus metrics middleware. Its handler label is the +// pattern of the chi route a request matches, such as /v1/image/*, or +// "unmatched" for a request that matches none, so the routes fix how many +// values the label has. It is never the path, with which any client could +// add series to the registry without limit. func (s *Middleware) Metrics() func(http.Handler) http.Handler { mdlw := ghmm.New(ghmm.Config{ - Recorder: metrics.NewRecorder(metrics.Config{}), + Recorder: metrics.NewRecorder(metrics.Config{ + Registry: s.metricsRegistry, + }), }) return func(next http.Handler) http.Handler { - return std.Handler("", mdlw, next) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + std.Handler(routePattern(r), mdlw, next).ServeHTTP(w, r) + }) } } +// routePattern returns the pattern of the chi route r matches, or +// "unmatched" when it matches none or is not served by a chi router. The +// metrics library takes the label before the request is routed, so the +// router is asked which route matches, with the path it routes on: +// r.URL.RawPath when set, else r.URL.Path, else /. +func routePattern(r *http.Request) string { + pattern := "" + + rctx := chi.RouteContext(r.Context()) + if rctx != nil { + path := r.URL.RawPath + if path == "" { + path = r.URL.Path + } + + if path == "" { + path = "/" + } + + pattern = rctx.Routes.Find(chi.NewRouteContext(), r.Method, path) + + // For the path a subrouter is mounted at, such as /v1 or /v1/, Find + // answers that path without asking the subrouter, which chi hands + // the request as /. + for _, route := range rctx.Routes.Routes() { + if route.SubRoutes != nil && + route.Pattern == strings.TrimSuffix(pattern, "/")+"/*" && + !route.SubRoutes.Match(chi.NewRouteContext(), r.Method, "/") { + pattern = "" + } + } + } + + if pattern == "" { + return "unmatched" + } + + return pattern +} + // MetricsAuth returns a basic auth middleware for the metrics endpoint. func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler { return basicauth.New( diff --git a/internal/middleware/middleware_internal_test.go b/internal/middleware/middleware_internal_test.go index 91ba3f3..0e17a05 100644 --- a/internal/middleware/middleware_internal_test.go +++ b/internal/middleware/middleware_internal_test.go @@ -3,12 +3,15 @@ package middleware import ( "bytes" "log/slog" + "maps" "net/http" "net/http/httptest" "net/url" "strings" "testing" + "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "sneak.berlin/go/pixa/internal/config" @@ -198,7 +201,7 @@ func TestMetricsRecordsServedRequest(t *testing.T) { // The line /metrics shows once one GET /test has been served. const want = `http_request_duration_seconds_count{` + - `code="200",handler="/test",method="GET",service=""} 1` + `code="200",handler="unmatched",method="GET",service=""} 1` mw := &Middleware{log: slog.Default(), config: &config.Config{}} @@ -220,6 +223,97 @@ func TestMetricsRecordsServedRequest(t *testing.T) { } } +// TestMetricsLabelsRequestsByRoute checks that the metrics middleware labels +// each request with the pattern of the route that answers it, never with its +// path: two paths one route answers share that route's pattern, and requests +// no route answers, a 404 or a 405, share "unmatched", among them requests for +// the path the subrouter is mounted at, /v1 and /v1/. A request with an empty +// path is answered by the / route, as chi routes it as /, and is labelled /. +// With a label per path, any client could add series to the registry without +// limit. The middleware registers with a registry of its own here, as another +// test in this package registers with Prometheus' default registry, which can +// be done only once. +func TestMetricsLabelsRequestsByRoute(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + mw := &Middleware{ + log: slog.Default(), + config: &config.Config{}, + metricsRegistry: registry, + } + + // The image route is in a subrouter, as in pixa's router. + router := chi.NewRouter() + router.Use(mw.Metrics()) + router.Get("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + router.Route("/v1", func(r chi.Router) { + r.Get("/image/*", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + }) + + for _, req := range []struct{ method, path string }{ + {http.MethodGet, "/v1/image/a.example.com/one.jpg/1x1.png"}, + {http.MethodGet, "/v1/image/b.example.com/two.jpg/1x1.png"}, + {http.MethodGet, "/no-such-route"}, + {http.MethodGet, "/v1/no-such-route"}, + {http.MethodGet, "/v1"}, + {http.MethodGet, "/v1/"}, + {http.MethodPost, "/v1/image/a.example.com/one.jpg/1x1.png"}, + // The request line GET http://pixa.example HTTP/1.1 leaves both + // r.URL.Path and r.URL.RawPath empty. + {http.MethodGet, "http://pixa.example"}, + } { + r := httptest.NewRequestWithContext( + t.Context(), req.method, req.path, nil) + t.Logf("%s %s: r.URL.Path %q, r.URL.RawPath %q", + req.method, req.path, r.URL.Path, r.URL.RawPath) + router.ServeHTTP(httptest.NewRecorder(), r) + } + + got := requestCountsByHandler(t, registry) + t.Logf("requests counted by handler label: %v", got) + + want := map[string]uint64{"/": 1, "/v1/image/*": 2, "unmatched": 5} + if !maps.Equal(got, want) { + t.Errorf("requests counted by handler label = %v, want %v", got, want) + } +} + +// requestCountsByHandler returns how many requests the request duration +// histogram in registry counted under each value of its handler label. +func requestCountsByHandler( + t *testing.T, registry *prometheus.Registry, +) map[string]uint64 { + t.Helper() + + families, err := registry.Gather() + if err != nil { + t.Fatalf("registry.Gather() error = %v", err) + } + + counts := map[string]uint64{} + + for _, family := range families { + if family.GetName() != "http_request_duration_seconds" { + continue + } + + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if label.GetName() == "handler" { + counts[label.GetValue()] += metric.GetHistogram().GetSampleCount() + } + } + } + } + + return counts +} + // TestLoggingLeavesOutSubmittedSigningKey checks that a login, a POST / // whose form carries the signing key, leaves no trace of the key in the // request's log line. From a3aa73f6fdb13d2c26be2e87c90797d118f5a5b7 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 18:16:17 +0200 Subject: [PATCH 071/131] Tell upaas operators to set the cache size limit (closes #251) "Running under upaas" in README.md listed PIXA_CACHE_MAX_BYTES as optional. Under upaas the volume is a directory on the host's filesystem, and the default limit is 75% of the sum of that filesystem's free space and what the cache holds, at least 500 MiB, so one pixa can take three quarters or more of the space every app has left, and all of it when that is 500 MiB or less. The section now tells the operator to set the variable and says what the default does there. The default is unchanged. Model: opus-5-5 --- README.md | 8 ++++++-- TODO.md | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b39b3ff..eb9685a 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,12 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: comma-separated; default none. Every image on a listed host is served to anyone without a signature, so list only hosts you control, where nobody else can put images. - - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it; - default 75% of (free space + what the cache holds) + - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it. Set + it: the volume is a directory on the host's filesystem, shared with the + host and every other app. The default is 75% of the sum of that + filesystem's free space and what the cache holds, and at least 500 MiB, so + the cache can grow to three quarters or more of the space they all have + left, and to all of it when that is 500 MiB or less. - the rest are in the table under Configuration below - **Health check:** the image's `HEALTHCHECK` requests `/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds diff --git a/TODO.md b/TODO.md index 893924a..97ccf43 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,13 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 "Running under upaas" in `README.md` tells the operator to set + `PIXA_CACHE_MAX_BYTES` (closes #251): it listed the variable as optional, but + under upaas the volume is a directory on the host's filesystem. The default, + 75% of the sum of that filesystem's free space and what the cache holds, and + at least 500 MiB, let one pixa take three quarters or more of the space the + host and every other app had left, and all of it when that was 500 MiB or + less. The section now says so. The default is unchanged. - 2026-10-15 the request metrics are labelled by route, not by path (closes #254): their `handler` label was each request's path, so every new path, a 404 or an encrypted URL, added series to the Prometheus registry that were never From 3ba9ae9c3992f7fcf6fb7e15f54cf07be432a5c3 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 18:44:11 +0200 Subject: [PATCH 072/131] Stop a transcode whose request has ended before it encodes (closes #243) Process looked at its context only while waiting for a processing slot, so a transcode kept its slot and worked on after its request's downstream_timeout. acquireSlot now gives the slot back when the context has ended by the time it has one, and encode returns the context's error without beginning. govips gives pixa no way to stop libvips once an encode has begun, so that encode keeps its slot. A request waiting on a shared transcode stopped at the first request's deadline is answered 503, not 500. README.md "Routes" says both. Model: opus-5-5 --- README.md | 23 +++- TODO.md | 28 ++++- internal/handlers/image.go | 3 +- internal/handlers/imageenc.go | 3 +- .../handlers/server_busy_internal_test.go | 38 +++++++ .../imageprocessor/deadline_internal_test.go | 106 ++++++++++++++++++ internal/imageprocessor/imageprocessor.go | 54 +++++---- .../concurrent_misses_internal_test.go | 42 +++++++ internal/imgcache/service.go | 12 ++ 9 files changed, 279 insertions(+), 30 deletions(-) create mode 100644 internal/imageprocessor/deadline_internal_test.go diff --git a/README.md b/README.md index eb9685a..78b5b56 100644 --- a/README.md +++ b/README.md @@ -318,9 +318,26 @@ cache at once, they share one upstream fetch (or one read of the cached source) and one transcode: the first request does the work, and the others wait for its image or its error, holding no upstream connection or processing slot of their own. A waiting request stops waiting when its own client goes away. The work -goes on for the others even if the first request's client goes away, until that -request's `downstream_timeout` ends. The shared fetch sends the first request's -ID upstream, and the lines logged for the fetch and the transcode carry that ID. +goes on for the others even if the first request's client goes away. The shared +fetch sends the first request's ID upstream, and the lines logged for the fetch +and the transcode carry that ID. + +A transcode whose request's `downstream_timeout` has ended (for a shared one, +the first request's) stops waiting for a processing slot, or, once it has one, +gives it back without beginning to encode. A request waiting for a shared +transcode that stops this way is answered 503 with the error +`server busy, try again later`, though its own `downstream_timeout` may not have +ended. govips gives pixa no way to stop libvips once an encode has begun, so an +encode keeps its slot until it finishes, which can be after +`downstream_timeout`. The limit of 8192 on each side of the source and of the +output bounds how long that takes. With one libvips thread, as pixad runs it, +encoding 8192x8192 random pixels takes up to about 1 second as JPEG, 15 as PNG, +16 as GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the slowest of 8 or 16 +bits per sample, with or without an alpha channel. WebP, AVIF and JPEG XL are +that slow only with an alpha channel: 8-bit RGB takes about 44, 53 and 28 +seconds. Decoding such a source adds up to about 11 seconds. A WebP, AVIF or +JPEG XL encode can therefore hold its slot past the default `downstream_timeout` +of 60 seconds. The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is diff --git a/TODO.md b/TODO.md index 97ccf43..ec06a81 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,24 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a transcode stops once its request's `downstream_timeout` has + ended, unless it has begun encoding (closes #243): the image processor looked + at its context only while it waited for a processing slot, so an image was + decoded and encoded, holding the slot, however long after its request's + deadline that took. `Process` now returns the context's error, and gives the + slot back, when the context has ended by the time it has the slot or before it + encodes. A request that waited for a shared transcode stopped at the first + request's deadline is answered 503 `server busy, try again later`, not 500, + though its own `downstream_timeout` may not have ended. govips gives pixa no + way to stop libvips once an encode has begun, so that encode keeps its slot + until it finishes; the limit of 8192 on each side of the source and of the + output bounds how long that takes. With one libvips thread, 8192x8192 random + pixels take up to about 1 second to encode as JPEG, 15 as PNG, 16 as GIF, 63 + as WebP, 110 as AVIF and 96 as JPEG XL, the last three with an alpha channel, + plus up to about 11 seconds to decode the source; `README.md` gives these + times and no longer says the work stops when `downstream_timeout` ends. The + settings of the WebP, AVIF and JPEG XL encoders, which can hold a slot past + the default `downstream_timeout` of 60 seconds, are unchanged. - 2026-10-15 "Running under upaas" in `README.md` tells the operator to set `PIXA_CACHE_MAX_BYTES` (closes #251): it listed the variable as optional, but under upaas the volume is a directory on the host's filesystem. The default, @@ -549,11 +567,11 @@ P2: security: per-origin rate limiting cancellation but keeps its deadline, and the others wait for its image or error holding no upstream connection or processing slot, and stop waiting when their own context ends; the request doing the processing waits for it even - then, up to its deadline; a request whose context has already ended starts - nothing; each request counts one miss, and the processing counts its fetch and - transcode once; a panic while processing is reported to Sentry when - `sentry_dsn` is set and becomes an error for every waiting request instead of - stopping pixad; documented in `README.md`. + then, until it ends, which can be after its deadline; a request whose context + has already ended starts nothing; each request counts one miss, and the + processing counts its fetch and transcode once; a panic while processing is + reported to Sentry when `sentry_dsn` is set and becomes an error for every + waiting request instead of stopping pixad; documented in `README.md`. - 2026-09-29 only the image routes send CORS headers (closes #98): the CORS middleware, with the `access_control_allow_origin` origin, moved from the router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it diff --git a/internal/handlers/image.go b/internal/handlers/image.go index aefde8e..9b28da1 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -240,7 +240,8 @@ func (s *Handlers) respondImageError( } if errors.Is(err, httpfetcher.ErrTooManyConnections) || - errors.Is(err, imageprocessor.ErrTooManyImages) { + errors.Is(err, imageprocessor.ErrTooManyImages) || + errors.Is(err, imgcache.ErrProcessingTimedOut) { s.respondError(w, "server busy, try again later", http.StatusServiceUnavailable) diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 8da6d47..4da08eb 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -158,7 +158,8 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { case errors.Is(err, httpfetcher.ErrUpstreamTimeout): s.respondError(w, "upstream timeout", http.StatusGatewayTimeout) case errors.Is(err, httpfetcher.ErrTooManyConnections), - errors.Is(err, imageprocessor.ErrTooManyImages): + errors.Is(err, imageprocessor.ErrTooManyImages), + errors.Is(err, imgcache.ErrProcessingTimedOut): s.respondError(w, "server busy, try again later", http.StatusServiceUnavailable) case errors.Is(err, httpfetcher.ErrInvalidContentType), diff --git a/internal/handlers/server_busy_internal_test.go b/internal/handlers/server_busy_internal_test.go index 2f3471c..58bae9b 100644 --- a/internal/handlers/server_busy_internal_test.go +++ b/internal/handlers/server_busy_internal_test.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "fmt" "log/slog" "net/http" @@ -47,3 +48,40 @@ func TestServerBusyAnswers503(t *testing.T) { } } } + +// TestProcessingTimedOutAnswers503 checks that respondImageError and +// handleImageError, which answer the errors of the two image routes, answer +// 503 with the server busy error when the image service reports that +// processing stopped at the deadline of the request doing it, wrapped as the +// service wraps it. A request that waited for that processing gets this error +// while its own deadline may have time left. The request's source only goes +// into a log line, so it is left empty. +func TestProcessingTimedOutAnswers503(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + err := fmt.Errorf("%w: failed to encode: %w", + imgcache.ErrProcessingTimedOut, context.DeadlineExceeded) + + plain := httptest.NewRecorder() + h.respondImageError(plain, &imgcache.ImageRequest{}, err) + + encrypted := httptest.NewRecorder() + h.handleImageError(encrypted, err) + + for name, rec := range map[string]*httptest.ResponseRecorder{ + "respondImageError": plain, "handleImageError": encrypted, + } { + t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String()) + + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("%s: status = %d, want %d", + name, rec.Code, http.StatusServiceUnavailable) + } + + if !strings.Contains(rec.Body.String(), "server busy, try again later") { + t.Errorf("%s: body = %q, want the server busy error", + name, rec.Body.String()) + } + } +} diff --git a/internal/imageprocessor/deadline_internal_test.go b/internal/imageprocessor/deadline_internal_test.go new file mode 100644 index 0000000..1bde965 --- /dev/null +++ b/internal/imageprocessor/deadline_internal_test.go @@ -0,0 +1,106 @@ +package imageprocessor + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" +) + +// readerAfterDeadline is a Process input whose first Read waits until done +// is closed, so the deadline passes after Process has taken its slot and +// before it encodes. +type readerAfterDeadline struct { + data *bytes.Reader + done <-chan struct{} + waited bool +} + +func (r *readerAfterDeadline) Read(p []byte) (int, error) { + if !r.waited { + r.waited = true + + <-r.done + } + + return r.data.Read(p) +} + +// inputNotToBeRead is a Process input that fails the test when it is read. +type inputNotToBeRead struct { + t *testing.T +} + +func (r inputNotToBeRead) Read([]byte) (int, error) { + r.t.Error("Process read its input although its deadline had passed") + + return 0, io.EOF +} + +// TestProcessStopsWhenDeadlinePassed checks that Process returns +// context.DeadlineExceeded, and holds no slot afterwards, when its context's +// deadline has passed before it starts, then without reading its input, or +// while it reads its input. govips gives pixa no way to stop libvips once an +// encode has begun, so Process must not begin one for a request that has +// ended. +func TestProcessStopsWhenDeadlinePassed(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, 50, 50) + + t.Run("before Process starts", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithDeadline(t.Context(), + time.Now().Add(-time.Second)) + defer cancel() + + // With a slot free, the select in acquireSlot takes the slot or sees + // the ended context at random, so Process is called many times + for range 100 { + checkProcessStopsAtDeadline(ctx, t, inputNotToBeRead{t: t}) + } + }) + + t.Run("while Process reads its input", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + + checkProcessStopsAtDeadline(ctx, t, &readerAfterDeadline{ + data: bytes.NewReader(input), done: ctx.Done(), + }) + }) +} + +// checkProcessStopsAtDeadline runs Process on input with ctx, whose deadline +// has passed by the time Process could encode, and fails the test unless +// Process returns context.DeadlineExceeded and gives its slot back. +func checkProcessStopsAtDeadline( + ctx context.Context, t *testing.T, input io.Reader, +) { + t.Helper() + + proc := New(Params{MaxConcurrentProcessing: 1}) + + result, err := proc.Process(ctx, input, smallJPEGRequest()) + if result != nil { + t.Logf("Process() made a %dx%d image after its deadline passed", + result.Width, result.Height) + + _ = result.Content.Close() + } + + t.Logf("Process() error = %v", err) + + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Process() error = %v, want context.DeadlineExceeded", err) + } + + if held := len(proc.processingSemaphore); held != 0 { + t.Errorf("%d slots still held after Process() returned", held) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 92cee8a..ca033e7 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -208,7 +208,10 @@ func New(params Params) *ImageProcessor { // Process transforms an image according to the request. When // MaxConcurrentProcessing images are already being processed, it waits up // to ProcessingWaitTimeout for one to finish, then fails with -// ErrTooManyImages. +// ErrTooManyImages. When ctx has ended once Process has a slot, or before it +// encodes, it returns ctx's error and gives the slot back. govips gives pixa +// no way to stop libvips once an encode has begun, so an encode keeps its +// slot until it finishes. func (p *ImageProcessor) Process( ctx context.Context, input io.Reader, @@ -290,7 +293,7 @@ func (p *ImageProcessor) Process( } // Encode to target format - output, err := p.encode(img, outputFormat, req.Quality) + output, err := p.encode(ctx, img, outputFormat, req.Quality) if err != nil { return nil, fmt.Errorf("failed to encode: %w", err) } @@ -417,25 +420,28 @@ func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int { // acquireSlot takes a slot in processingSemaphore, waiting at most // processingWaitTimeout for one to free up, and returns the func that gives -// it back. A free slot is taken even when ctx has ended; only the wait for -// one stops when ctx ends, as the rest of Process does not check ctx. +// it back. When ctx has ended by the time it has a slot, it gives the slot +// back and returns ctx's error. func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) { - release := func() { <-p.processingSemaphore } - select { case p.processingSemaphore <- struct{}{}: - return release, nil - default: - } - - select { - case p.processingSemaphore <- struct{}{}: - return release, nil case <-time.After(p.processingWaitTimeout): return nil, ErrTooManyImages case <-ctx.Done(): return nil, ctx.Err() } + + release := func() { <-p.processingSemaphore } + + // select takes the slot even when ctx has ended at the same moment + err := ctx.Err() + if err != nil { + release() + + return nil, err + } + + return release, nil } // detectFormat returns the format string for a govips image type. @@ -527,10 +533,17 @@ func (p *ImageProcessor) resize( const defaultQuality = 85 -// encode encodes an image to the specified format. +// encode encodes an image to the specified format. govips gives pixa no way +// to stop libvips once an encode has begun, so when ctx has ended encode +// returns ctx's error without beginning one. func (p *ImageProcessor) encode( - img *vips.ImageRef, format Format, quality int, + ctx context.Context, img *vips.ImageRef, format Format, quality int, ) ([]byte, error) { + err := ctx.Err() + if err != nil { + return nil, err + } + if quality <= 0 { quality = defaultQuality } @@ -540,7 +553,7 @@ func (p *ImageProcessor) encode( // libvips' built-in profile; govips' own sRGB path variable is set on // first use but read without a lock, so concurrent requests race on it. if img.HasICCProfile() { - err := img.TransformICCProfileWithFallback("srgb", "srgb") + err = img.TransformICCProfileWithFallback("srgb", "srgb") if err != nil { return nil, fmt.Errorf("failed to convert to sRGB: %w", err) } @@ -632,10 +645,11 @@ func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) { // avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest // govips can set. With one thread, as pixad runs libvips, 1 takes about 51 -// seconds to save an 8192x8192 image of random pixels, the worst case, -// against the default downstream_timeout of 60 seconds. On an image of -// milder noise, which 1 saves in about 12 seconds, 2 takes nearly a minute -// and libvips' default, 4, takes minutes. +// seconds to save an 8192x8192 RGB image of random pixels, and about 110 +// with an alpha channel, the worst case, against the default +// downstream_timeout of 60 seconds. On an image of milder noise, which 1 +// saves in about 12 seconds, 2 takes nearly a minute and libvips' default, +// 4, takes minutes. const avifEffort = 1 // avifBitdepth is the AVIF bit depth, 8 bits per sample for every image. diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index ad0e1b2..b81d63a 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -15,6 +15,7 @@ import ( "github.com/getsentry/sentry-go" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/magic" ) @@ -435,6 +436,47 @@ func TestService_Get_ReturnsByItsDeadline(t *testing.T) { } } +// TestService_Get_WaiterGetsProcessingTimedOut has a request with no deadline +// join the held fetch of a first request with a short one. With the only +// processing slot held, the first request's transcode waits for it until the +// first request's deadline ends, and stops then. The waiting request, whose +// own context has not ended, must get ErrProcessingTimedOut, which the image +// routes answer 503 as when pixa is busy. +func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + svc.processor = imageprocessor.New( + imageprocessor.Params{MaxConcurrentProcessing: 1}, + ) + + holdProcessingSlot(t, svc.processor) + + const firstTimeout = 500 * time.Millisecond + + firstCtx, cancelFirst := context.WithTimeout(t.Context(), firstTimeout) + defer cancelFirst() + + first := startGet(firstCtx, svc, photoVariant(fixtures, 85, FitCover)) + + <-fetcher.started + + waiter := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + + time.Sleep(arrivalWait) + fetcher.releaseFetches() + + t.Logf("first request: Get() error = %v", (<-first).err) + + got := <-waiter + t.Logf("waiting request: Get() error = %v", got.err) + + if !errors.Is(got.err, ErrProcessingTimedOut) { + t.Errorf("waiting request: Get() error = %v, want %v", + got.err, ErrProcessingTimedOut) + } +} + // panickingFetcher panics on every fetch. type panickingFetcher struct{} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 57f4e82..1806a42 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -129,6 +129,12 @@ func NewService(cfg *ServiceConfig) (*Service, error) { // ErrNegativeCached is returned when a URL is in the negative cache (recently failed). var ErrNegativeCached = errors.New("request is in negative cache (recently failed)") +// ErrProcessingTimedOut is returned when processing an image stopped because +// the downstream_timeout of the request doing it ended, to that request and to +// every request that waited for its image. +var ErrProcessingTimedOut = errors.New( + "image processing stopped at the deadline of the request doing it") + // Get retrieves a processed image, fetching and processing if necessary. // While its source URL is in the negative cache, a cached variant is still // served and a variant not in the cache is still made from a cached source; @@ -549,7 +555,13 @@ func (s *Service) processAndStore( FitMode: imageprocessor.FitMode(req.FitMode), } + // ctx has the deadline of the request doing the processing, which can + // end while a request waiting for its image has time left processResult, err := s.processor.Process(ctx, source, processReq) + if errors.Is(err, context.DeadlineExceeded) { + return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err) + } + if err != nil { return nil, fmt.Errorf("image processing failed: %w", err) } From 2ab96155ba651632495706092843d80ee7ad04d5 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 19:27:33 +0200 Subject: [PATCH 073/131] Sync cached files and refuse ones left empty or partial (closes #252) A crash could leave a cached source or transformed image empty or cut short, and pixa then served it, or fetched it again forever. Source and transformed image files, and the .meta file beside each transformed image, are now written under a temporary name, synced, renamed into place and their directory synced; a failed directory sync removes the renamed file. The reconciliation pass compares each cached file's size with the size its row records and removes a file that is empty or of another size, with its rows and a warning. Requests compare no sizes. Not verified by a test: the syncs, and the removal after a failed sync. Model: opus-5-5 --- README.md | 8 + TODO.md | 12 + internal/imgcache/cache.go | 8 +- internal/imgcache/eviction.go | 129 ++++--- .../partial_cache_files_internal_test.go | 359 ++++++++++++++++++ internal/imgcache/storage.go | 126 +++--- 6 files changed, 535 insertions(+), 107 deletions(-) create mode 100644 internal/imgcache/partial_cache_files_internal_test.go diff --git a/README.md b/README.md index 78b5b56..61fc6c9 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,14 @@ name. Multiple source paths may reference the same content blob; the database tracks references rather than using filesystem refcounting. +Source and transformed image files, and the `.meta` file beside each transformed +image, are written under a temporary name, synced to disk, and only then renamed +into place, so a crash leaves no file under its own name with part of its data. +When pixa starts, and every five minutes after, it removes each cached source or +transformed image whose file is empty, or not the size the database records for +it, with a warning naming the file; the next request for it fetches the source +or converts the image again. + pixa's target is 1-5k r/s, which has not been measured at that rate (see Load Test). Toward it, pixa keeps in memory the content types of the 10,000 transformed images most recently cached or served, so a cache hit on one of them diff --git a/TODO.md b/TODO.md index ec06a81..a9f3843 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,18 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a cached file left empty or partial is no longer served, or fetched + again on every request (closes #252). Source and transformed image files, and + each transformed image's `.meta` file, are synced to disk before they are + renamed into place, and their directory after; the `.meta` file is written + before the image, and a failed directory sync removes the renamed file. The + reconciliation pass, at startup and every five minutes, removes a cached + source or transformed image whose file is empty, or not the size the database + records for it, with its rows and a warning naming the file, so the next + request fetches the source or converts the image again; requests compare no + sizes. Storing a source over a file of another size replaces that file. A + source that fails to decode is not removed for that alone: a damaged upstream + image would then be fetched on every request. - 2026-10-15 a transcode stops once its request's `downstream_timeout` has ended, unless it has begun encoding (closes #243): the image processor looked at its context only while it waited for a processing slot, so an image was diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index dc49692..4a78953 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -660,10 +660,10 @@ func (c *Cache) readHitAndMissCounts(ctx context.Context, stats *CacheStats) err // hold: it reads the content type from the variant's .meta file and // keeps it in metaCache, unless a StoreVariant has put one there // meanwhile, as the store's is newer. A read that finds no .meta file, -// as one can between a store's writing of the variant file and of its -// .meta file, serves application/octet-stream and keeps nothing, so -// metaCache only ever holds a type read from a .meta file or passed to -// StoreVariant. +// as one can when the store's .meta write failed or an eviction removed +// the file after the read opened the variant, serves +// application/octet-stream and keeps nothing, so metaCache only ever +// holds a type read from a .meta file or passed to StoreVariant. func (c *Cache) loadVariantWithMeta( cacheKey VariantKey, ) (io.ReadCloser, int64, string, error) { diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 71f1fb0..2fe34fb 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "io/fs" "os" @@ -560,10 +561,11 @@ func (c *Cache) runCleanExpired(ctx context.Context) { // (or whose accounting insert failed, e.g. StoreVariant's best-effort // insert under transient DB contention), drops accounting rows whose // files are missing, removes source blob files the database does not -// know (and rows whose files are gone), sweeps stale temp files left -// behind by crashed writes, and last checks the total cache usage -// against the tables. It reads the tables a page at a time. Running it -// periodically, not just once, bounds how long such drift can +// know (and rows whose files are gone), removes source and variant files +// that are empty or not the size their rows record, sweeps stale temp +// files left behind by crashed writes, and last checks the total cache +// usage against the tables. It reads the tables a page at a time. +// Running it periodically, not just once, bounds how long such drift can // accumulate unaccounted for on a long-running process to one eviction // interval. Once ctx is cancelled, it stops at the next file or row and // returns ctx's error. @@ -595,8 +597,9 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error { return c.reconcileUsageTotal(ctx) } -// reconcileVariantFiles walks the variant storage directory, adopting -// files without accounting rows and sweeping stale temp files. +// reconcileVariantFiles walks the variant storage directory, checking +// each variant file against its accounting row (reconcileVariantFile) +// and sweeping stale temp files. func (c *Cache) reconcileVariantFiles(ctx context.Context) error { return filepath.WalkDir( c.variants.baseDir, @@ -621,38 +624,55 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error { return nil } - return c.adoptVariantFile(ctx, path, entry, VariantKey(name)) + return c.reconcileVariantFile(ctx, path, entry, VariantKey(name)) }, ) } -// adoptVariantFile inserts an accounting row for a variant file that -// has none, using the file's size and modification time. -func (c *Cache) adoptVariantFile( +// reconcileVariantFile checks one variant file against its accounting +// row. A file without a row is adopted into the accounting, unless it is +// empty. A file that is empty, or not the size its row records, as a +// crash or a restored snapshot can leave it, is removed with its row and +// a warning naming it, so the variant is converted again. +func (c *Cache) reconcileVariantFile( ctx context.Context, path string, entry fs.DirEntry, cacheKey VariantKey, ) error { - var rowExists int - - err := c.db.QueryRowContext(ctx, - `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, string(cacheKey), - ).Scan(&rowExists) - if err != nil { - return fmt.Errorf("failed to check variant accounting row: %w", err) - } - - if rowExists > 0 { - return nil - } - info, err := entry.Info() if err != nil { return fmt.Errorf("failed to stat variant file: %w", err) } + var recordedSize int64 + + err = c.db.QueryRowContext(ctx, + `SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey), + ).Scan(&recordedSize) + + switch { + case errors.Is(err, sql.ErrNoRows) && info.Size() > 0: + return c.adoptVariantFile(ctx, path, info, cacheKey) + case err != nil && !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("failed to check variant accounting row: %w", err) + case info.Size() > 0 && info.Size() == recordedSize: + return nil + } + + // The file is empty, or not the size its row records + c.log.Warn("removing damaged cached variant file", "path", path, + "file_bytes", info.Size(), "recorded_bytes", recordedSize) + + return c.evictVariant(ctx, cacheKey) +} + +// adoptVariantFile inserts an accounting row for a variant file that +// has none, using the file's size and modification time. +func (c *Cache) adoptVariantFile( + ctx context.Context, path string, info fs.FileInfo, cacheKey VariantKey, +) error { modTime := info.ModTime().UTC().Format(sqliteTimestampLayout) contentType := c.variantContentTypeFromSidecar(path) - _, err = c.db.ExecContext(ctx, ` + _, err := c.db.ExecContext(ctx, ` INSERT INTO variant_content (cache_key, size_bytes, content_type, created_at, last_accessed_at) VALUES (?, ?, ?, ?, ?) @@ -771,10 +791,9 @@ func queryStringColumn[T ~string]( return values, nil } -// reconcileSourceFiles walks the source content directory, removing -// blob files the database does not track (they are unreachable: source -// lookups always go through source_metadata) and sweeping stale temp -// files. +// reconcileSourceFiles walks the source content directory, checking +// each blob file against its source_content row (reconcileSourceFile) +// and sweeping stale temp files. func (c *Cache) reconcileSourceFiles(ctx context.Context) error { return filepath.WalkDir( c.srcContent.baseDir, @@ -795,31 +814,55 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error { return nil } - return c.removeUntrackedSourceFile(ctx, path, ContentHash(name)) + return c.reconcileSourceFile(ctx, path, entry, ContentHash(name)) }, ) } +// reconcileSourceFile checks one source blob file against its +// source_content row. A file without a row is removed: it is +// unreachable, as source lookups always go through source_metadata. A +// file that is empty, or not the size its row records, as a crash or a +// restored snapshot can leave it, is removed with its rows and a warning +// naming it, so the source is fetched again. +func (c *Cache) reconcileSourceFile( + ctx context.Context, path string, entry fs.DirEntry, contentHash ContentHash, +) error { + info, err := entry.Info() + if err != nil { + return fmt.Errorf("failed to stat source content file: %w", err) + } + + var recordedSize int64 + + err = c.db.QueryRowContext(ctx, + `SELECT size_bytes FROM source_content WHERE content_hash = ?`, + string(contentHash), + ).Scan(&recordedSize) + + switch { + case errors.Is(err, sql.ErrNoRows): + return c.removeUntrackedSourceFile(ctx, path, contentHash) + case err != nil: + return fmt.Errorf("failed to check source content row: %w", err) + case info.Size() > 0 && info.Size() == recordedSize: + return nil + } + + // The file is empty, or not the size its row records + c.log.Warn("removing damaged cached source file", "path", path, + "file_bytes", info.Size(), "recorded_bytes", recordedSize) + + return c.evictSourceBlob(ctx, contentHash) +} + // removeUntrackedSourceFile deletes a source blob file that has no // source_content row. Any source_metadata rows referencing the hash // are removed first so no row ever points at a deleted file. func (c *Cache) removeUntrackedSourceFile( ctx context.Context, path string, contentHash ContentHash, ) error { - var rowExists int - - err := c.db.QueryRowContext(ctx, - `SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(contentHash), - ).Scan(&rowExists) - if err != nil { - return fmt.Errorf("failed to check source content row: %w", err) - } - - if rowExists > 0 { - return nil - } - - _, err = c.db.ExecContext(ctx, + _, err := c.db.ExecContext(ctx, `DELETE FROM source_metadata WHERE content_hash = ?`, string(contentHash)) if err != nil { return fmt.Errorf("failed to delete metadata rows for untracked blob: %w", err) diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go new file mode 100644 index 0000000..283ac13 --- /dev/null +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -0,0 +1,359 @@ +package imgcache + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" +) + +// getImage calls Get for req and returns the image it answered with and +// its cache status. +func getImage(t *testing.T, svc *Service, req *ImageRequest) ([]byte, CacheStatus) { + t.Helper() + + resp, err := svc.Get(t.Context(), req) + if err != nil { + t.Fatalf("Get() of quality %d error = %v", req.Quality, err) + } + + defer func() { _ = resp.Content.Close() }() + + body, err := io.ReadAll(resp.Content) + if err != nil { + t.Fatalf("reading the image of quality %d: %v", req.Quality, err) + } + + return body, resp.CacheStatus +} + +// fileSize returns the size of the file at path. +func fileSize(t *testing.T, path string) int64 { + t.Helper() + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + + return info.Size() +} + +// truncateFile cuts the file at path down to size bytes. +func truncateFile(t *testing.T, path string, size int64) { + t.Helper() + + t.Logf("cutting %s from %d to %d bytes", + filepath.Base(path), fileSize(t, path), size) + + err := os.Truncate(path, size) + if err != nil { + t.Fatalf("truncating %s: %v", path, err) + } +} + +// cachedSourcePath returns the path of the cached source file req uses. +func cachedSourcePath(t *testing.T, svc *Service, req *ImageRequest) string { + t.Helper() + + contentHash, _, err := svc.cache.LookupSource(t.Context(), req) + if err != nil || contentHash == "" { + t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err) + } + + return svc.cache.srcContent.hashToPath(contentHash) +} + +// reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does +// when it starts, and returns what the pass logged. +func reconcileAsAtStartup(t *testing.T, cache *Cache) string { + t.Helper() + + var logBuf bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil)) + + err := cache.reconcileAccounting(t.Context()) + if err != nil { + t.Fatalf("reconcileAccounting() error = %v", err) + } + + t.Logf("reconciliation log: %s", logBuf.String()) + + return logBuf.String() +} + +// assertWarningNames checks that logs hold a warning naming the file at +// path. +func assertWarningNames(t *testing.T, logs, path string) { + t.Helper() + + for line := range strings.Lines(logs) { + if strings.Contains(line, `"level":"WARN"`) && + strings.Contains(line, `"path":"`+path+`"`) { + return + } + } + + t.Errorf("reconciliation logged no warning naming %s", path) +} + +// TestService_Get_FetchesAgainForEmptyCachedSource empties the cached source +// file, as a crash or a restored snapshot could, and runs the reconciliation +// pass, which must remove it with a warning naming it. It then asks for two +// new sizes of the source: the first must fetch it again and cache it whole, +// so that the second is made from the cache without a third fetch. +func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + + getImage(t, svc, photoVariant(fixtures, 85, FitCover)) + + path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover)) + wholeSize := fileSize(t, path) + + truncateFile(t, path, 0) + assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) + + getImage(t, svc, photoVariant(fixtures, 40, FitCover)) + getImage(t, svc, photoVariant(fixtures, 30, FitCover)) + + fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount + t.Logf("upstream fetches = %d", fetches) + + if fetches != 2 { + t.Errorf("upstream fetches = %d, want 2: the empty cached source "+ + "was kept and fetched again on every new size", fetches) + } + + size := fileSize(t, path) + if size != wholeSize { + t.Errorf("cached source is %d bytes, want %d", size, wholeSize) + } +} + +// TestService_Get_FetchesAgainForPartialCachedSource cuts the cached source +// file to half its size, as a crash or a restored snapshot could, and runs +// the reconciliation pass, which must remove it with a warning naming it. +// It then asks for two new sizes of the source: the first must fetch it +// again rather than fail to decode the partial file, and the second is +// made from the cache. +func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + + getImage(t, svc, photoVariant(fixtures, 85, FitCover)) + + path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover)) + wholeSize := fileSize(t, path) + + truncateFile(t, path, wholeSize/2) + assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) + + getImage(t, svc, photoVariant(fixtures, 40, FitCover)) + getImage(t, svc, photoVariant(fixtures, 30, FitCover)) + + fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount + t.Logf("upstream fetches = %d", fetches) + + if fetches != 2 { + t.Errorf("upstream fetches = %d, want 2", fetches) + } + + size := fileSize(t, path) + if size != wholeSize { + t.Errorf("cached source is %d bytes, want %d", size, wholeSize) + } +} + +// assertConvertedAgain caches a variant of the photo, cuts its file to size +// bytes and runs the reconciliation pass, which must remove it with a +// warning naming it. It then checks that Get converts the image again +// instead of serving the file, and that the next Get serves the whole +// variant from the cache. +func assertConvertedAgain(t *testing.T, size int64) { + t.Helper() + + svc, fixtures := SetupTestService(t) + req := photoVariant(fixtures, 85, FitCover) + + want, _ := getImage(t, svc, req) + + path := svc.cache.variants.keyToPath(CacheKey(req)) + + truncateFile(t, path, size) + assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) + + got, status := getImage(t, svc, req) + t.Logf("after the cut: %d bytes, %s", len(got), status) + + if status != CacheMiss || !bytes.Equal(got, want) { + t.Errorf("after the cut: Get() = %d bytes, %s; want the %d bytes "+ + "converted again, %s", len(got), status, len(want), CacheMiss) + } + + got, status = getImage(t, svc, req) + t.Logf("next request: %d bytes, %s", len(got), status) + + if status != CacheHit || !bytes.Equal(got, want) { + t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s", + len(got), status, len(want), CacheHit) + } +} + +// TestService_Get_ConvertsAgainForEmptyCachedVariant empties a cached +// variant file, as a crash or a restored snapshot could, and checks that it +// is converted again rather than served empty. +func TestService_Get_ConvertsAgainForEmptyCachedVariant(t *testing.T) { + t.Parallel() + + assertConvertedAgain(t, 0) +} + +// TestService_Get_ConvertsAgainForPartialCachedVariant cuts a cached +// variant file to ten bytes, as a crash or a restored snapshot could, and +// checks that it is converted again rather than served cut short. +func TestService_Get_ConvertsAgainForPartialCachedVariant(t *testing.T) { + t.Parallel() + + assertConvertedAgain(t, 10) +} + +// TestReconciliationRemovesEmptyVariantFileWithoutRow checks that the +// reconciliation pass removes an empty variant file that has no +// accounting row, with a warning naming it, instead of adding a row for +// it that would have it served empty. +func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + _, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil), + testContentTypeWebP) + if err != nil { + t.Fatalf("failed to store an empty variant file: %v", err) + } + + logs := reconcileAsAtStartup(t, cache) + + assertWarningNames(t, logs, cache.variants.keyToPath(testVariantKeyOne)) + + if cache.variants.Exists(testVariantKeyOne) { + t.Error("the empty variant file is still on disk") + } + + rows := countRows(t, cache, + `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, + string(testVariantKeyOne)) + if rows != 0 { + t.Errorf("the empty variant file got %d accounting rows, want 0", rows) + } +} + +// TestVariantStorage_StoreReplacesMetaFile stores a variant over one +// already stored and checks that Store puts a new .meta file in place of +// the old one, as it does the image, rather than writing into the old +// file, which a crash could leave empty. The old .meta file is given a +// second name before the store, so that it can be told apart from the new +// one after it. +func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) { + t.Parallel() + + storage, key := storeTestVariant(t) + metaPath := storage.keyToPath(key) + variantMetaSuffix + oldMetaPath := metaPath + ".old" + + err := os.Link(metaPath, oldMetaPath) + if err != nil { + t.Fatalf("giving the .meta file a second name: %v", err) + } + + _, err = storage.Store(key, bytes.NewReader([]byte("new variant data")), + testContentTypeJPEG) + if err != nil { + t.Fatalf("Store() error = %v", err) + } + + oldMeta, err := os.Stat(oldMetaPath) + if err != nil { + t.Fatalf("stat of the old .meta file: %v", err) + } + + newMeta, err := os.Stat(metaPath) + if err != nil { + t.Fatalf("stat of the new .meta file: %v", err) + } + + if os.SameFile(oldMeta, newMeta) { + t.Error("Store() wrote into the old .meta file instead of putting " + + "a new one in its place") + } + + reader, _, contentType, err := storage.LoadWithMeta(key) + if err != nil { + t.Fatalf("LoadWithMeta() error = %v", err) + } + + _ = reader.Close() + + t.Logf("content type after the store: %s", contentType) + + if contentType != testContentTypeJPEG { + t.Errorf("content type = %q, want %q", contentType, testContentTypeJPEG) + } +} + +// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a +// store over a file of another size at the content's path, such as a +// partial file a crash left, replaces it with the content. +func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) { + t.Parallel() + + storage, err := NewContentStorage(t.TempDir()) + if err != nil { + t.Fatalf("NewContentStorage() error = %v", err) + } + + content := []byte("the whole source image") + sum := sha256.Sum256(content) + hash := ContentHash(hex.EncodeToString(sum[:])) + path := storage.hashToPath(hash) + + err = os.MkdirAll(filepath.Dir(path), StorageDirPerm) + if err != nil { + t.Fatalf("creating the content's directory: %v", err) + } + + err = os.WriteFile(path, content[:5], StorageFilePerm) + if err != nil { + t.Fatalf("writing a partial file: %v", err) + } + + _, err = storage.StoreHashed(hash, content) + if err != nil { + t.Fatalf("StoreHashed() error = %v", err) + } + + stored, err := storage.Load(hash) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + + defer func() { _ = stored.Close() }() + + got, err := io.ReadAll(stored) + if err != nil { + t.Fatalf("reading the stored content: %v", err) + } + + if !bytes.Equal(got, content) { + t.Errorf("stored file = %q, want %q", got, content) + } +} diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index 35ff2f1..de55df4 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -80,8 +80,8 @@ func (s *ContentStorage) Store(r io.Reader) (ContentHash, int64, error) { // from hash, without recomputing it. Callers that already know the // hash before writing (e.g. because they must hold a hash-keyed lock // across the whole store operation) use this instead of Store. Like -// Store, it is idempotent: content already on disk at that path is -// left untouched. +// Store, it is idempotent: a file of the content's size already at that +// path is left untouched, and a file of another size is replaced. func (s *ContentStorage) StoreHashed( hash ContentHash, data []byte, ) (int64, error) { @@ -152,28 +152,35 @@ func (s *ContentStorage) Exists(hash ContentHash) bool { return err == nil } -// writeIfAbsent writes data to the path derived from hash, unless -// content already exists there, via a temp-file-plus-rename so -// concurrent readers never observe a partial file. +// writeIfAbsent writes data with writeFileSynced to the path derived +// from hash, unless a file of the data's size is already there. func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error { // Build path: /// path := s.hashToPath(hash) - // Check if already exists - _, statErr := os.Stat(path) - if statErr == nil { + // A file of another size there is damaged, and is replaced + info, statErr := os.Stat(path) + if statErr == nil && info.Size() == int64(len(data)) { return nil } - // Create directory structure - dir := filepath.Dir(path) - - err := os.MkdirAll(dir, StorageDirPerm) + err := os.MkdirAll(filepath.Dir(path), StorageDirPerm) if err != nil { return fmt.Errorf("failed to create directory: %w", err) } - // Write to temp file first, then rename for atomicity + return writeFileSynced(path, data) +} + +// writeFileSynced writes data to path through a temp file in the same +// directory: the temp file is synced to disk, renamed to path, and the +// directory synced after, so that neither a concurrent reader nor a +// crash finds a partial file under path. When the directory sync fails, +// the file is removed from path, so that a failed write leaves no file +// there. +func writeFileSynced(path string, data []byte) error { + dir := filepath.Dir(path) + tmpFile, err := os.CreateTemp(dir, ".tmp-*") if err != nil { return fmt.Errorf("failed to create temp file: %w", err) @@ -181,18 +188,20 @@ func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error { tmpPath := tmpFile.Name() - // Each failure path below unlinks the temp file explicitly. This - // replaces a deferred cleanup that read a named result, which the - // canonical config does not permit; the set of paths that remove - // tmpPath, and the order relative to Close, is unchanged. This - // mirrors how MetadataStorage.Store and VariantStorage.Store below - // already express the same cleanup. _, err = tmpFile.Write(data) if err != nil { _ = tmpFile.Close() _ = os.Remove(tmpPath) - return fmt.Errorf("failed to write content: %w", err) + return fmt.Errorf("failed to write temp file: %w", err) + } + + err = tmpFile.Sync() + if err != nil { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + + return fmt.Errorf("failed to sync temp file: %w", err) } err = tmpFile.Close() @@ -202,7 +211,6 @@ func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error { return fmt.Errorf("failed to close temp file: %w", err) } - // Atomic rename err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path)) if err != nil { _ = os.Remove(tmpPath) @@ -210,6 +218,31 @@ func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error { return fmt.Errorf("failed to rename temp file: %w", err) } + err = syncDirectory(dir) + if err != nil { + _ = os.Remove(path) + + return err + } + + return nil +} + +// syncDirectory syncs the directory dir to disk, so that a file just +// renamed into it is still there after a crash. +func syncDirectory(dir string) error { + d, err := os.Open(filepath.Clean(dir)) + if err != nil { + return fmt.Errorf("failed to open directory: %w", err) + } + + err = d.Sync() + _ = d.Close() + + if err != nil { + return fmt.Errorf("failed to sync directory: %w", err) + } + return nil } @@ -427,7 +460,9 @@ func NewVariantStorage(baseDir string, log *slog.Logger) (*VariantStorage, error return &VariantStorage{baseDir: baseDir, log: log}, nil } -// Store writes content and metadata to storage at the given key. +// Store writes content and metadata to storage at the given key, each +// with writeFileSynced. The .meta file is written first, so that a crash +// cannot leave the content in place without it. func (s *VariantStorage) Store( key VariantKey, r io.Reader, contentType string, ) (int64, error) { @@ -440,46 +475,11 @@ func (s *VariantStorage) Store( path := s.keyToPath(key) metaPath := path + ".meta" - // Create directory structure - dir := filepath.Dir(path) - - err = os.MkdirAll(dir, StorageDirPerm) + err = os.MkdirAll(filepath.Dir(path), StorageDirPerm) if err != nil { return 0, fmt.Errorf("failed to create directory: %w", err) } - // Write content to temp file first, then rename for atomicity - tmpFile, err := os.CreateTemp(dir, ".tmp-*") - if err != nil { - return 0, fmt.Errorf("failed to create temp file: %w", err) - } - - tmpPath := tmpFile.Name() - - _, err = tmpFile.Write(data) - if err != nil { - _ = tmpFile.Close() - _ = os.Remove(tmpPath) - - return 0, fmt.Errorf("failed to write content: %w", err) - } - - err = tmpFile.Close() - if err != nil { - _ = os.Remove(tmpPath) - - return 0, fmt.Errorf("failed to close temp file: %w", err) - } - - // Atomic rename content - err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path)) - if err != nil { - _ = os.Remove(tmpPath) - - return 0, fmt.Errorf("failed to rename temp file: %w", err) - } - - // Write metadata meta := VariantMeta{ ContentType: contentType, Size: size, @@ -491,13 +491,19 @@ func (s *VariantStorage) Store( return 0, fmt.Errorf("failed to marshal metadata: %w", err) } - // Metadata write failure is non-fatal; content is already stored. - err = os.WriteFile(metaPath, metaData, StorageFilePerm) + // Metadata write failure is non-fatal: the content is then served as + // application/octet-stream. + err = writeFileSynced(metaPath, metaData) if err != nil { s.log.Warn("failed to write variant metadata sidecar", "path", metaPath, "error", err) } + err = writeFileSynced(path, data) + if err != nil { + return 0, err + } + return size, nil } From 742db9ceb13532a293d7a711d7bbff68d527af52 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 20:44:44 +0200 Subject: [PATCH 074/131] Logging out ends every login session (closes #256) A login session was a cookie with nothing kept on the server, so logging out removed it from one browser and a copy kept working for 30 days. The session manager now keeps a random value in the database, made when it has none and replaced by each logout; a session carries the value current at its login and is refused once it has changed. The logout check compares no time, so the host's clock cannot block a login, and losing the state directory ends every login session. Logging out is now a POST with the CSRF token, sent by a form on the generator page. Model: opus-5-5 --- README.md | 21 +- TODO.md | 17 ++ internal/db/migrations/001_schema.sql | 8 + internal/handlers/auth.go | 12 +- .../handlers/auth_session_internal_test.go | 92 ++++++ internal/handlers/handlers.go | 5 +- internal/server/logout_internal_test.go | 269 ++++++++++++++++++ internal/server/routes.go | 3 +- internal/session/session.go | 118 +++++++- internal/session/session_logout_test.go | 148 ++++++++++ internal/static/style.css | 12 +- internal/templates/generator.html | 9 +- 12 files changed, 698 insertions(+), 16 deletions(-) create mode 100644 internal/server/logout_internal_test.go create mode 100644 internal/session/session_logout_test.go diff --git a/README.md b/README.md index 61fc6c9..87d3838 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the database and the disk cache: - It needs a persistent volume: without one, every restart starts with an empty - cache. In the container, the startup script gives the directory to the user + cache. Losing the directory also ends every login session (see Encrypted + URLs). In the container, the startup script gives the directory to the user pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user must be able to write the directory. - `cache_max_bytes` limits the source and transformed images together. The @@ -196,7 +197,11 @@ path under `/v1/` answers 200, in maintenance mode too. session it answers 303 to `/`. Answers: 200 with the page showing the URL; 400 with the page naming a field that is not valid; 500 when the URL cannot be made. -- `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`. +- `POST /logout` — log out with the generator page's Logout button: end every + login session, in every browser (see Encrypted URLs), and remove the session + cookie. Needs: a login session and the generator page's form (below); without + a login session it only removes the cookie. Answers: 303 to `/`; 500 when the + logout cannot be saved in the database. - `GET` or `HEAD` `/v1/image///.`, or `/v1/image///` with no format — an image, fetched, resized and converted (below). Needs: a signature, unless the host is allowlisted (see @@ -244,7 +249,7 @@ the request carries the same ID as `request_id`, and so do the lines it logs when it fetches, converts and serves an image; the fetch sends it to the upstream host as `X-Request-ID`. -Both `POST` routes accept only a form that pixa's own page served: the page puts +The `POST` routes accept only a form that pixa's own page served: the page puts a token in the form and sets a cookie to match, and a request without both is refused with 403, so another site cannot submit the form from a visitor's browser. The login and generator pages are meant to be opened over HTTPS: while @@ -382,7 +387,15 @@ nor change what it asks for. 1. Open `/` in a browser over HTTPS (or over plain HTTP while `debug` is on, see Routes) and log in with the signing key (`signing_key`). The login session - lasts 30 days, or until `/logout`. + lasts 30 days, or until someone logs out with the generator page's Logout + button. Every login is made with the one signing key, so logging out ends + every login session, in every browser, a copy of the session cookie included. + pixa keeps a random value in its database, which it makes when the database + has none and replaces at each logout. A login session carries the value + current at its login and is refused once the value has changed, also after a + restart (see Deployment). The check for a logout compares no time, so a login + after a logout works whatever the host's clock says; the 30 days are still + measured by the host's clock. 2. On the generator page, give the source image's URL, the width and height, the format, quality and fit, and how long the URL lasts, then submit the form (`POST /generate`). The format is JPEG XL unless another is chosen; a form diff --git a/TODO.md b/TODO.md index a9f3843..724a404 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,23 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 logging out ends every login session (closes #256): a login session + was a cookie with nothing kept on the server, so `/logout` removed it from one + browser and a copy of it kept working for its 30 days. Every login is made + with the one signing key, so logging out now ends them all: the session + manager keeps a random value, made with `crypto/rand` when the new `logout` + table has none and replaced by each logout, saves it there so that it outlasts + a restart, puts the value at login in each session, and refuses a session + carrying another value. The check for a logout compares no time, so a login + after a logout works whatever the host's clock does; a session's 30 days are + still checked against the clock. Losing the state directory ends every login + session, and no logout can make an ended one work again. Logging out is now + `POST /logout` from a form on the generator page, behind the CSRF check the + other forms have, so a link or image elsewhere on the same site cannot end the + sessions. A logout without a valid login session ends none and only removes + the cookie, as anyone can take a CSRF token from the login page. A logout that + cannot be saved answers 500. `session.NewManager`, which the tests use, keeps + the value in memory only; pixad uses `session.NewManagerWithDatabase`. - 2026-10-15 a cached file left empty or partial is no longer served, or fetched again on every request (closes #252). Source and transformed image files, and each transformed image's `.meta` file, are synced to disk before they are diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index 68f3a32..234a141 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -176,3 +176,11 @@ CREATE TABLE IF NOT EXISTS cache_stats ( last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ); INSERT OR IGNORE INTO cache_stats (id) VALUES (1); + +-- A random value that a login session carries from its login. pixa makes it +-- when the table has no row, and each logout replaces it; a login session +-- carrying another value is refused, so logging out ends every login session. +CREATE TABLE IF NOT EXISTS logout ( + id INTEGER PRIMARY KEY CHECK (id = 1), + logout_value TEXT NOT NULL +); diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index 052ccd2..dff2763 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -89,10 +89,18 @@ func (s *Handlers) handleLoginPost(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/", http.StatusSeeOther) } -// HandleLogout clears the session and redirects to login. +// HandleLogout logs out, which ends every login session (see +// session.Manager.Logout), and redirects to login. func (s *Handlers) HandleLogout() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - s.sessMgr.ClearSession(w) + err := s.sessMgr.Logout(w, r) + if err != nil { + s.log.Error("failed to log out", "error", err) + http.Error(w, "Failed to log out", http.StatusInternalServerError) + + return + } + http.Redirect(w, r, "/", http.StatusSeeOther) } } diff --git a/internal/handlers/auth_session_internal_test.go b/internal/handlers/auth_session_internal_test.go index 883e434..250d627 100644 --- a/internal/handlers/auth_session_internal_test.go +++ b/internal/handlers/auth_session_internal_test.go @@ -185,6 +185,98 @@ func TestHandleLogout_ClearsSessionCookie(t *testing.T) { } } +// TestHandleLogout_EndsCopiedSession verifies that after POST /logout a copy +// of the session cookie taken before it shows the login page, not the +// generator page. +func TestHandleLogout_EndsCopiedSession(t *testing.T) { + t.Parallel() + + h, srv := newCSRFTestRouter(t) + + copied := newSessionCookie(t, h) + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodPost, "/logout", nil) + req.AddCookie(copied) + + rec := httptest.NewRecorder() + h.HandleLogout().ServeHTTP(rec, req) + + if rec.Code != http.StatusSeeOther { + t.Fatalf("POST /logout status = %d, want %d", + rec.Code, http.StatusSeeOther) + } + + req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + req.AddCookie(copied) + + rec = httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + body := rec.Body.String() + t.Logf("GET / with the copied cookie: status %d, login page %v", + rec.Code, strings.Contains(body, loginForm)) + + if !strings.Contains(body, loginForm) || strings.Contains(body, generatorForm) { + t.Error("a cookie copied before /logout still opens the generator page") + } +} + +// TestHandleLogout_SaveFails_Answers500 verifies that when the logout cannot +// be saved in the database, POST /logout answers 500, leaves the session +// cookie in place and ends no session. +func TestHandleLogout_SaveFails_Answers500(t *testing.T) { + t.Parallel() + + h, srv := newCSRFTestRouter(t) + + db := setupTestDB(t) + + sessMgr, err := session.NewManagerWithDatabase( + t.Context(), testSigningKey, db) + if err != nil { + t.Fatalf("session.NewManagerWithDatabase() error = %v", err) + } + + h.sessMgr = sessMgr + cookie := newSessionCookie(t, h) + + // Saving the logout fails on a closed database. + err = db.Close() + if err != nil { + t.Fatalf("closing the database: %v", err) + } + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodPost, "/logout", nil) + req.AddCookie(cookie) + + rec := httptest.NewRecorder() + h.HandleLogout().ServeHTTP(rec, req) + + t.Logf("POST /logout: status %d, Set-Cookie %q", + rec.Code, rec.Header().Get("Set-Cookie")) + + if rec.Code != http.StatusInternalServerError { + t.Errorf("POST /logout status = %d, want %d", + rec.Code, http.StatusInternalServerError) + } + + if c := findSessionCookie(rec); c != nil { + t.Errorf("a logout that failed set the session cookie: %s", c) + } + + req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + req.AddCookie(cookie) + + rec = httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if !strings.Contains(rec.Body.String(), generatorForm) { + t.Error("a logout that failed ended the session") + } +} + // TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate // with a valid CSRF token but no login session answers 303 to / and makes no // URL. diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index da9572f..9cfc3bb 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -164,7 +164,10 @@ func (s *Handlers) initImageService() error { // Initialize session manager (signing key is validated at config load // time). Session cookies are always Secure/HttpOnly/SameSite=Strict. - sessMgr, err := session.NewManager(s.config.SigningKey) + // The random value that each logout replaces is saved in the database, + // so it outlasts a restart. + sessMgr, err := session.NewManagerWithDatabase( + context.Background(), s.config.SigningKey, s.db.DB()) if err != nil { return err } diff --git a/internal/server/logout_internal_test.go b/internal/server/logout_internal_test.go new file mode 100644 index 0000000..2d46d76 --- /dev/null +++ b/internal/server/logout_internal_test.go @@ -0,0 +1,269 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "regexp" + "strings" + "testing" + + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/handlers" + "sneak.berlin/go/pixa/internal/healthcheck" + "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/middleware" + "sneak.berlin/go/pixa/internal/session" +) + +// logoutFormPattern extracts the CSRF token rendered into the generator +// page's logout form. +var logoutFormPattern = regexp.MustCompile( + `(?s)action="/logout".*?name="gorilla\.csrf\.Token" value="([^"]+)"`) + +// startPixa starts the components pixad's fx app builds, from cfg, and +// returns the server with its routes, as newTestServer does. +func startPixa(t *testing.T, cfg *config.Config) *Server { + t.Helper() + + var ( + h *handlers.Handlers + mw *middleware.Middleware + ) + + app := fxtest.New(t, + fx.Supply(cfg), + fx.Provide( + globals.New, + logger.New, + database.New, + healthcheck.New, + handlers.New, + middleware.New, + ), + fx.Populate(&h, &mw), + ) + app.RequireStart() + t.Cleanup(app.RequireStop) + + s := &Server{config: cfg, mw: mw, h: h} + s.SetupRoutes() + + return s +} + +// logIn logs in to s with the signing key, as postLogin does, and returns +// the session cookie it sets. +func logIn(t *testing.T, s *Server) *http.Cookie { + t.Helper() + + rec := postLogin(t, s, firstClient, "", testSigningKey) + + for _, c := range rec.Result().Cookies() { + if c.Name == session.CookieName { + return c + } + } + + t.Fatal("logging in set no session cookie") + + return nil +} + +// opensGeneratorPage reports whether GET / with cookie shows s's generator +// page, which only a valid login session opens. +func opensGeneratorPage(t *testing.T, s *Server, cookie *http.Cookie) bool { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + req.AddCookie(cookie) + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + return rec.Code == http.StatusOK && + strings.Contains(rec.Body.String(), `action="/generate"`) +} + +// postLogout loads the generator page with GET / and sessionCookie and +// submits its logout form with POST /logout, as a browser does. +func postLogout( + t *testing.T, s *Server, sessionCookie *http.Cookie, +) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + req.AddCookie(sessionCookie) + + page := httptest.NewRecorder() + s.ServeHTTP(page, req) + + match := logoutFormPattern.FindStringSubmatch(page.Body.String()) + if match == nil { + t.Fatal("the generator page has no logout form with a CSRF token") + } + + form := url.Values{"gorilla.csrf.Token": {match[1]}} + req = httptest.NewRequestWithContext(t.Context(), http.MethodPost, + "/logout", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.AddCookie(sessionCookie) + + for _, c := range page.Result().Cookies() { + req.AddCookie(c) + } + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + return rec +} + +// TestLogoutWithoutCSRFTokenEndsNoSession verifies that a request to /logout +// carrying the session cookie but not the logout form's CSRF token ends no +// session: GET /logout, as an image on another page of the same site makes a +// browser send, answers 405, and POST /logout without the token answers 403. +func TestLogoutWithoutCSRFTokenEndsNoSession(t *testing.T) { + t.Parallel() + + tests := []struct { + method string + want int + }{ + {http.MethodGet, http.StatusMethodNotAllowed}, + {http.MethodPost, http.StatusForbidden}, + } + + for _, tc := range tests { + t.Run(tc.method, func(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + cookie := logIn(t, s) + + req := httptest.NewRequestWithContext( + t.Context(), tc.method, "/logout", nil) + req.AddCookie(cookie) + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + t.Logf("%s /logout without the CSRF token: status %d", + tc.method, rec.Code) + + if rec.Code != tc.want { + t.Errorf("%s /logout without the CSRF token status = %d, "+ + "want %d", tc.method, rec.Code, tc.want) + } + + if !opensGeneratorPage(t, s, cookie) { + t.Errorf("%s /logout without the CSRF token ended the "+ + "login session", tc.method) + } + }) + } +} + +// TestLoginAfterLogoutOpensGeneratorPage verifies that after a logout from +// the generator page a copy of the session cookie taken before it is +// refused, and that a login after it opens the generator page. +func TestLoginAfterLogoutOpensGeneratorPage(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + + copied := logIn(t, s) + if !opensGeneratorPage(t, s, copied) { + t.Fatal("logging in did not open the generator page") + } + + logout := postLogout(t, s, copied) + if logout.Code != http.StatusSeeOther { + t.Fatalf("POST /logout status = %d, want %d", + logout.Code, http.StatusSeeOther) + } + + if opensGeneratorPage(t, s, copied) { + t.Error("a cookie copied before the logout still opens the " + + "generator page") + } + + opens := opensGeneratorPage(t, s, logIn(t, s)) + t.Logf("a login after the logout opens the generator page: %v", opens) + + if !opens { + t.Error("a login after the logout does not open the generator page") + } +} + +// TestLogoutOutlastsRestart verifies that a copy of the session cookie taken +// before a logout is still refused by a pixa started again on the same state +// directory, as pixad is after a restart. +func TestLogoutOutlastsRestart(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + + copied := logIn(t, s) + if !opensGeneratorPage(t, s, copied) { + t.Fatal("logging in did not open the generator page") + } + + logout := postLogout(t, s, copied) + if logout.Code != http.StatusSeeOther { + t.Fatalf("POST /logout status = %d, want %d", + logout.Code, http.StatusSeeOther) + } + + restarted := startPixa(t, s.config) + + opens := opensGeneratorPage(t, restarted, copied) + t.Logf("after the restart, the copied cookie opens the generator page: %v", + opens) + + if opens { + t.Error("after a restart, a cookie copied before the logout " + + "still opens the generator page") + } +} + +// TestLostStateDirectoryEndsEveryLoginSession verifies that a pixa started on +// an empty state directory, as pixad is after a restart that lost its state +// directory, refuses a session cookie from before, and that a login on it +// opens the generator page. +func TestLostStateDirectoryEndsEveryLoginSession(t *testing.T) { + t.Parallel() + + s := newTestServer(t) + + before := logIn(t, s) + if !opensGeneratorPage(t, s, before) { + t.Fatal("logging in did not open the generator page") + } + + emptyStateDir := *s.config + emptyStateDir.StateDir = t.TempDir() + emptyStateDir.DBURL = "file:" + + filepath.Join(emptyStateDir.StateDir, "state.sqlite3") + + restarted := startPixa(t, &emptyStateDir) + + opens := opensGeneratorPage(t, restarted, before) + t.Logf("on an empty state directory, the cookie from before opens the "+ + "generator page: %v", opens) + + if opens { + t.Error("on an empty state directory, a cookie from before still " + + "opens the generator page") + } + + if !opensGeneratorPage(t, restarted, logIn(t, restarted)) { + t.Error("a login on an empty state directory does not open the " + + "generator page") + } +} diff --git a/internal/server/routes.go b/internal/server/routes.go index 104f2e6..b7ed1a2 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -69,10 +69,9 @@ func (s *Server) SetupRoutes() { r.With(s.mw.RateLimit(LoginAttemptsPerMinute, time.Minute, nil)). Post("/", s.h.HandleRoot()) r.Post("/generate", s.h.HandleGenerateURL()) + r.Post("/logout", s.h.HandleLogout()) }) - s.router.Get("/logout", s.h.HandleLogout()) - // Image routes, the only ones that send CORS headers, as pages on other // sites read them. They are a subrouter rather than a group: a group's // middleware runs only for a request that matches one of its routes, diff --git a/internal/session/session.go b/internal/session/session.go index 7233d9e..788c114 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -1,9 +1,16 @@ -// Package session provides encrypted session cookie management. +// Package session provides encrypted session cookie management, and keeps in +// the database a random value that each logout replaces, so that a logout +// ends every session made before it. package session import ( + "context" + "crypto/rand" + "database/sql" "errors" + "fmt" "net/http" + "sync" "time" "github.com/gorilla/securecookie" @@ -32,14 +39,36 @@ type Data struct { Authenticated bool `json:"auth"` CreatedAt time.Time `json:"created"` ExpiresAt time.Time `json:"expires"` + + // LogoutValue is the Manager's random value, which each logout + // replaces, when the session was made. + LogoutValue string `json:"logoutValue"` } // Manager handles session creation and validation using encrypted cookies. +// +// Every login is made with the one signing key, so a logout ends every +// session made before it, in every browser. The Manager keeps a random +// value, which each logout replaces: a session carries the value current at +// its login, and a session carrying another value is refused. The check for +// a logout compares no time, so a login after a logout works whatever the +// host's clock does; the session's expiry is still checked against the +// clock. The Manager keeps the value in memory, so checking a session reads +// no database; a Manager from NewManagerWithDatabase also saves it in the +// database, so that it outlasts a restart. type Manager struct { sc *securecookie.SecureCookie + db *sql.DB + + // mu guards logoutValue, and is held while a logout is saved, so that + // the database and logoutValue agree. + mu sync.Mutex + logoutValue string } // NewManager creates a session manager with keys derived from the signing key. +// It makes a new random value and keeps it in memory only, so a restart ends +// every session; NewManagerWithDatabase keeps the value in the database. // // Session cookies always carry the Secure, HttpOnly, and SameSite=Strict // attributes; this cannot be configured. Browsers treat http://localhost as a @@ -63,10 +92,44 @@ func NewManager(signingKey string) (*Manager, error) { sc.MaxAge(int(SessionTTL.Seconds())) return &Manager{ - sc: sc, + sc: sc, + logoutValue: rand.Text(), }, nil } +// NewManagerWithDatabase creates a session manager like NewManager that keeps +// its random value in db, which must have pixa's schema: it saves the value +// NewManager made when db has none, and otherwise starts from the one saved +// there. +func NewManagerWithDatabase( + ctx context.Context, signingKey string, db *sql.DB, +) (*Manager, error) { + m, err := NewManager(signingKey) + if err != nil { + return nil, err + } + + _, err = db.ExecContext(ctx, + "INSERT OR IGNORE INTO logout (id, logout_value) VALUES (1, ?)", + m.logoutValue) + if err != nil { + return nil, fmt.Errorf( + "failed to save the random value in the logout table: %w", err) + } + + err = db.QueryRowContext(ctx, + "SELECT logout_value FROM logout WHERE id = 1", + ).Scan(&m.logoutValue) + if err != nil { + return nil, fmt.Errorf( + "failed to read the random value in the logout table: %w", err) + } + + m.db = db + + return m, nil +} + // CreateSession creates a new authenticated session and sets the cookie. func (m *Manager) CreateSession(w http.ResponseWriter) error { now := time.Now() @@ -74,6 +137,7 @@ func (m *Manager) CreateSession(w http.ResponseWriter) error { Authenticated: true, CreatedAt: now, ExpiresAt: now.Add(SessionTTL), + LogoutValue: m.currentLogoutValue(), } encoded, err := m.sc.Encode(CookieName, data) @@ -122,9 +186,30 @@ func (m *Manager) ValidateSession(r *http.Request) (*Data, error) { return nil, ErrInvalidSession } + // A logout since the session was made has ended it. + if data.LogoutValue != m.currentLogoutValue() { + return nil, ErrInvalidSession + } + return &data, nil } +// Logout removes the session cookie and, when r has a valid session, ends +// every session made until now. A request without one ends no session, so +// someone who has not logged in cannot end the sessions of those who have. +func (m *Manager) Logout(w http.ResponseWriter, r *http.Request) error { + if m.IsAuthenticated(r) { + err := m.saveLogout(r.Context()) + if err != nil { + return err + } + } + + m.ClearSession(w) + + return nil +} + // ClearSession removes the session cookie. func (m *Manager) ClearSession(w http.ResponseWriter) { http.SetCookie(w, &http.Cookie{ @@ -145,3 +230,32 @@ func (m *Manager) IsAuthenticated(r *http.Request) bool { return err == nil && data != nil && data.Authenticated } + +// saveLogout replaces the random value with a new one, saving it in the +// database first when the Manager has one. +func (m *Manager) saveLogout(ctx context.Context) error { + value := rand.Text() + + m.mu.Lock() + defer m.mu.Unlock() + + if m.db != nil { + _, err := m.db.ExecContext(ctx, + "UPDATE logout SET logout_value = ? WHERE id = 1", value) + if err != nil { + return fmt.Errorf("failed to save the logout: %w", err) + } + } + + m.logoutValue = value + + return nil +} + +// currentLogoutValue returns the Manager's random value. +func (m *Manager) currentLogoutValue() string { + m.mu.Lock() + defer m.mu.Unlock() + + return m.logoutValue +} diff --git a/internal/session/session_logout_test.go b/internal/session/session_logout_test.go new file mode 100644 index 0000000..b86dbb7 --- /dev/null +++ b/internal/session/session_logout_test.go @@ -0,0 +1,148 @@ +package session_test + +import ( + "database/sql" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/session" +) + +const logoutTestSigningKey = "test-signing-key-12345" + +// newTestDatabase opens an empty in-memory database with pixa's schema. It +// has one connection, as pixad's has: each connection to ":memory:" would +// open a database of its own. +func newTestDatabase(t *testing.T) *sql.DB { + t.Helper() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("sql.Open() error = %v", err) + } + + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + + err = database.ApplyMigrations(t.Context(), db, nil) + if err != nil { + t.Fatalf("ApplyMigrations() error = %v", err) + } + + return db +} + +// createSessionCookie logs in with mgr and returns the session cookie it +// sets. +func createSessionCookie(t *testing.T, mgr *session.Manager) *http.Cookie { + t.Helper() + + rec := httptest.NewRecorder() + + err := mgr.CreateSession(rec) + if err != nil { + t.Fatalf("CreateSession() error = %v", err) + } + + for _, c := range rec.Result().Cookies() { + if c.Name == session.CookieName { + return c + } + } + + t.Fatalf("CreateSession() did not set a %q cookie", session.CookieName) + + return nil +} + +// requestWithCookie returns a GET request carrying cookie, or no cookie when +// cookie is nil. +func requestWithCookie(t *testing.T, cookie *http.Cookie) *http.Request { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) + if cookie != nil { + req.AddCookie(cookie) + } + + return req +} + +// TestLogout_EndsEverySessionMadeBeforeIt verifies that after a logout a +// copy of the session cookie taken before it is refused and a login after it +// works, and that both still hold for a new Manager on the same database, as +// after a restart. +func TestLogout_EndsEverySessionMadeBeforeIt(t *testing.T) { + t.Parallel() + + db := newTestDatabase(t) + + mgr, err := session.NewManagerWithDatabase( + t.Context(), logoutTestSigningKey, db) + if err != nil { + t.Fatalf("NewManagerWithDatabase() error = %v", err) + } + + copied := createSessionCookie(t, mgr) + + err = mgr.Logout(httptest.NewRecorder(), requestWithCookie(t, copied)) + if err != nil { + t.Fatalf("Logout() error = %v", err) + } + + _, err = mgr.ValidateSession(requestWithCookie(t, copied)) + t.Logf("cookie copied before the logout: ValidateSession() error = %v", err) + + if !errors.Is(err, session.ErrInvalidSession) { + t.Errorf("cookie copied before the logout: error = %v, want %v", + err, session.ErrInvalidSession) + } + + loginAfter := createSessionCookie(t, mgr) + + _, err = mgr.ValidateSession(requestWithCookie(t, loginAfter)) + if err != nil { + t.Errorf("login after the logout: ValidateSession() error = %v", err) + } + + restarted, err := session.NewManagerWithDatabase( + t.Context(), logoutTestSigningKey, db) + if err != nil { + t.Fatalf("NewManagerWithDatabase() after restart error = %v", err) + } + + if restarted.IsAuthenticated(requestWithCookie(t, copied)) { + t.Error("after a restart, a cookie copied before the logout is accepted") + } + + if !restarted.IsAuthenticated(requestWithCookie(t, loginAfter)) { + t.Error("after a restart, a login made after the logout is refused") + } +} + +// TestLogout_WithoutSession_EndsNoSession verifies that a logout from a +// request without a session cookie ends no session, so someone who has not +// logged in cannot end the sessions of those who have. +func TestLogout_WithoutSession_EndsNoSession(t *testing.T) { + t.Parallel() + + mgr, err := session.NewManagerWithDatabase( + t.Context(), logoutTestSigningKey, newTestDatabase(t)) + if err != nil { + t.Fatalf("NewManagerWithDatabase() error = %v", err) + } + + cookie := createSessionCookie(t, mgr) + + err = mgr.Logout(httptest.NewRecorder(), requestWithCookie(t, nil)) + if err != nil { + t.Fatalf("Logout() error = %v", err) + } + + if !mgr.IsAuthenticated(requestWithCookie(t, cookie)) { + t.Error("a logout without a session cookie ended a session") + } +} diff --git a/internal/static/style.css b/internal/static/style.css index da22318..407c46e 100644 --- a/internal/static/style.css +++ b/internal/static/style.css @@ -122,12 +122,20 @@ header { margin-bottom: 2rem; } -header a { +/* The logout button, drawn as a link. */ + +header button { + width: auto; + margin: 0; + padding: 0; + background: none; font-size: 0.875rem; color: #4b5563; + text-decoration: underline; } -header a:hover { +header button:hover { + background: none; color: #1f2937; } diff --git a/internal/templates/generator.html b/internal/templates/generator.html index 2caaaac..6d679d2 100644 --- a/internal/templates/generator.html +++ b/internal/templates/generator.html @@ -10,9 +10,12 @@

Pixa URL Generator

- - Logout - +
+ {{ .CSRFField }} + +
{{if .GeneratedURL}} From 16f9d1dfd12c68c97955e22091ec0cf9b52a5843 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 21:47:11 +0200 Subject: [PATCH 075/131] Label request metrics with standard HTTP methods only (closes #274) The metrics middleware labelled each request with its method as sent, and Go's HTTP server accepts any method name that is a valid token, so each made-up method a client sent added series that were never removed. The method label is now the request's method only for the nine standard methods, and "other" for any other. The middleware hands the metrics library a copy of the request carrying the label and serves the next handler the original, so handlers still see the method the client sent. Model: opus-5-5 --- README.md | 4 +- TODO.md | 9 ++++ internal/middleware/middleware.go | 33 +++++++++++- .../middleware/middleware_internal_test.go | 53 +++++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 87d3838..13a6a3b 100644 --- a/README.md +++ b/README.md @@ -685,7 +685,9 @@ See `configs/config.example.yml` for all options with defaults. `metrics.username` and `metrics.password` are set. A request's `handler` label is the pattern of the route it matches, such as `/v1/image/*`, never its path. Requests that match no route share the label `unmatched`: those the router - answers 404 or 405, and CORS preflights + answers 404 or 405, and CORS preflights. A request's `method` label is its + method when that is `GET`, `HEAD`, `POST`, `PUT`, `PATCH`, `DELETE`, + `OPTIONS`, `CONNECT` or `TRACE`, and `other` for any other method - **Logging**: stdlib slog ## Entrypoints diff --git a/TODO.md b/TODO.md index 724a404..3c33482 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the request metrics record any method that is not a standard one as + `other` (closes #274): their `method` label was each request's method as sent, + and Go's HTTP server accepts any method name, so a client sending made-up + methods added series to the Prometheus registry that were never removed. The + label is now the request's method only for `GET`, `HEAD`, `POST`, `PUT`, + `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` and `TRACE`. The metrics library reads + the method from the request it is handed, so the metrics middleware hands it a + copy of the request carrying the label; the handlers still see the method the + client sent. - 2026-10-15 logging out ends every login session (closes #256): a login session was a cookie with nothing kept on the server, so `/logout` removed it from one browser and a copy of it kept working for its 30 days. Every login is made diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index a35c23e..4053a0a 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -220,7 +220,10 @@ func (s *Middleware) CORS() func(http.Handler) http.Handler { // pattern of the chi route a request matches, such as /v1/image/*, or // "unmatched" for a request that matches none, so the routes fix how many // values the label has. It is never the path, with which any client could -// add series to the registry without limit. +// add series to the registry without limit. Its method label is the +// request's method when that is a standard one, such as GET, and "other" +// for any other: Go's HTTP server accepts any method name, so a label per +// method would let a client add series the same way. func (s *Middleware) Metrics() func(http.Handler) http.Handler { mdlw := ghmm.New(ghmm.Config{ Recorder: metrics.NewRecorder(metrics.Config{ @@ -230,7 +233,19 @@ func (s *Middleware) Metrics() func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - std.Handler(routePattern(r), mdlw, next).ServeHTTP(w, r) + // The metrics library takes the method label from the request + // it is handed, so it is handed a shallow copy of r whose + // method is the label. next is served r itself, through the + // response writer the library wraps to see the status and size. + labelled := r.WithContext(r.Context()) + labelled.Method = methodLabel(r.Method) + + serveNext := http.HandlerFunc( + func(wrapped http.ResponseWriter, _ *http.Request) { + next.ServeHTTP(wrapped, r) + }) + + std.Handler(routePattern(r), mdlw, serveNext).ServeHTTP(w, labelled) }) } } @@ -275,6 +290,20 @@ func routePattern(r *http.Request) string { return pattern } +// methodLabel returns method when it is one of the standard methods net/http +// names: GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS, CONNECT and TRACE. +// For any other method it returns "other". +func methodLabel(method string) string { + switch method { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, + http.MethodPatch, http.MethodDelete, http.MethodOptions, + http.MethodConnect, http.MethodTrace: + return method + default: + return "other" + } +} + // MetricsAuth returns a basic auth middleware for the metrics endpoint. func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler { return basicauth.New( diff --git a/internal/middleware/middleware_internal_test.go b/internal/middleware/middleware_internal_test.go index 0e17a05..21d18b4 100644 --- a/internal/middleware/middleware_internal_test.go +++ b/internal/middleware/middleware_internal_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "slices" "strings" "testing" @@ -314,6 +315,58 @@ func requestCountsByHandler( return counts } +// TestMetricsGivesMadeUpMethodsOneLabel checks that the metrics middleware +// gives requests with methods that are not standard ones a single method +// label, "other", and still serves each request with the method it was sent +// with. With a label per method, any client could add series to the +// registry without limit by making up a new method for each request. +func TestMetricsGivesMadeUpMethodsOneLabel(t *testing.T) { + t.Parallel() + + // The line /metrics shows once both made-up methods have been served. + const want = `http_request_duration_seconds_count{` + + `code="200",handler="unmatched",method="other",service=""} 2` + + registry := prometheus.NewRegistry() + mw := &Middleware{ + log: slog.Default(), + config: &config.Config{}, + metricsRegistry: registry, + } + + var served []string + + handler := mw.Metrics()(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + served = append(served, r.Method) + + w.WriteHeader(http.StatusOK) + })) + + methods := []string{"FROBNICATE", "WIBBLE"} + for _, method := range methods { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext( + t.Context(), method, "/", nil)) + } + + t.Logf("methods the handler was served: %v", served) + + if !slices.Equal(served, methods) { + t.Errorf("methods the handler was served = %v, want %v", + served, methods) + } + + rec := httptest.NewRecorder() + promhttp.HandlerFor(registry, promhttp.HandlerOpts{}).ServeHTTP(rec, + httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/metrics", nil)) + + if !strings.Contains(rec.Body.String(), want) { + t.Errorf("/metrics does not count both made-up methods as one; "+ + "want the line %q in:\n%s", want, rec.Body.String()) + } +} + // TestLoggingLeavesOutSubmittedSigningKey checks that a login, a POST / // whose form carries the signing key, leaves no trace of the key in the // request's log line. From 4e30dc746716d50d3df0bbac3272fdc340fccb75 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 22:40:38 +0200 Subject: [PATCH 076/131] Re-vendor the canonical files from sneak/prompts at e3f79c4 (closes #221) The dotfiles, REPO_POLICIES.md and the standard scripts are fetched whole from sneak/prompts e3f79c4, with pixa's own entries kept after them. Check builds pass a new CACHEBUST build argument instead of --no-cache, and every Dockerfile stage declares it first, so every RUN step still runs while Go's build cache is kept. Lint runs golangci-lint v2.14.0, and the test phase runs on the Debian Go image. Of the two new lint findings, one is fixed and one (gosec G703 on the fixed config file locations) is suppressed. The agent guidance moves unchanged from CLAUDE.md to AGENTS.md. Deviation: the workflow keeps its docker-smoke step while issue 203 waits on the owner. Model: opus-5-5 --- .dockerignore | 27 ++- .editorconfig | 3 + .gitea/workflows/check.yml | 12 +- .gitignore | 43 +++- .golangci.yml | 3 + .prettierignore | 5 - CLAUDE.md => AGENTS.md | 0 Dockerfile | 78 ++++--- README.md | 19 +- REPO_POLICIES.md | 363 +++++++++++++++++++++++---------- TODO.md | 22 +- internal/config/config.go | 1 + internal/imgcache/cachesize.go | 4 - script/bootstrap | 45 ++-- script/cibuild | 23 ++- script/docker | 22 +- script/fmt | 22 +- script/fmt-check | 22 +- script/lint | 13 +- script/test | 12 +- 20 files changed, 517 insertions(+), 222 deletions(-) rename CLAUDE.md => AGENTS.md (100%) diff --git a/.dockerignore b/.dockerignore index c2dfed1..d711d2c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,12 +13,21 @@ # `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and # deletes the package directory from the context. -# Unlike the standard file, which leaves out all of .git, pixa sends -# .git without its config. Without a VERSION build argument the stage -# that compiles runs `git describe --tags --always` on .git, which does -# not need .git/config; that file can hold a credential, such as a +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a # password in a remote URL or the token the CI checkout step stores there. -.git/config +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config # Agent scratch: one full checkout of the repo per in-flight agent. # Anchored because it occurs once where agents run at the repo root. @@ -42,7 +51,9 @@ **/[iI][dD]_[rR][sS][aA] **/[iI][dD]_[dD][sS][aA] **/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] **/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] # Dependencies: restored inside the image, never copied in. **/node_modules @@ -60,10 +71,8 @@ **/.vscode **/*.sublime-* -# pixa's own entries. Nothing in the build reads .gitignore. On the -# host, `make build` writes bin/pixad, and the example config keeps its -# state directory in data/. -.gitignore +# pixa's own entries. On the host, `make build` writes bin/pixad, and the +# example config keeps its state directory in data/. /bin /data diff --git a/.editorconfig b/.editorconfig index 92ec261..de84fae 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,3 +13,6 @@ indent_style = tab [*.go] indent_style = tab + +# This repository's own sections, such as one for another language it +# uses, go below this comment, and a re-vendor keeps them. diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index bf68d24..3556efe 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -1,15 +1,21 @@ name: check on: [push] +# Free the shared runner: a new push cancels only the same branch's older run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: runs-on: ubuntu-latest + # Free the shared runner from a hung build. + timeout-minutes: 20 steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - # The default clone is shallow and has no tags, so the - # version the build takes from `git describe` would be a - # bare commit; this fetches the whole history with its tags. + # script/cibuild needs no token, so none is left in .git/config. with: + persist-credentials: false + # All history and tags, so git describe finds the version tag. fetch-depth: 0 - run: script/cibuild - run: script/docker-smoke diff --git a/.gitignore b/.gitignore index 8549307..6c4d819 100644 --- a/.gitignore +++ b/.gitignore @@ -17,15 +17,46 @@ Thumbs.db # entry and must not be given a `**/` prefix on the way into one. .claude/ -# Environment / secrets -.env -.env.* -*.pem -*.key +# Node +node_modules/ + +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation at the end of this file, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] + +# This repository's own entries, such as its build outputs, go below +# this comment, and a re-vendor keeps them. Anchor a binary built at the +# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`. + +# Go +*.log +*.out +*.test # Dependencies vendor/ -node_modules/ # Build output /bin/ diff --git a/.golangci.yml b/.golangci.yml index a7a74c2..32db9fe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -17,6 +17,7 @@ linters: disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) - godot # Requires comments to end with periods - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go @@ -24,6 +25,8 @@ linters: # silenced by disabling that name, not by enabling the successor. - wsl # Deprecated, replaced by wsl_v5 - gomodguard # Deprecated, replaced by gomodguard_v2 + # Misses findings at random in v2.14.0; back once a pinned release fixes it + - canonicalheader settings: lll: line-length: 88 diff --git a/.prettierignore b/.prettierignore index 888b018..23d67fc 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,7 +1,2 @@ node_modules/ yarn.lock - -# A byte-for-byte copy of the one in sneak/prompts. -REPO_POLICIES.md - -vendor/ diff --git a/CLAUDE.md b/AGENTS.md similarity index 100% rename from CLAUDE.md rename to AGENTS.md diff --git a/Dockerfile b/Dockerfile index e82d2d0..b2d7c80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,12 @@ +# Every stage declares ARG CACHEBUST first: script/ passes a new value +# on every build, which reruns every RUN step after it. The steps that +# compile mount Go's build cache, which survives from build to build. + # Lint phase. script/lint builds it alone. The linter is run directly: # `make lint` and script/lint are themselves a docker build. -# golangci/golangci-lint:v2.12.2, 2026-10-04 -FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint +# golangci/golangci-lint:v2.14.0, 2026-10-15 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint +ARG CACHEBUST # The linter compiles every package, and govips needs the libvips # headers for that. REPO_POLICIES.md has the lint phase install them @@ -14,17 +19,20 @@ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN golangci-lint run --config .golangci.yml ./... +RUN --mount=type=cache,target=/root/.cache/go-build \ + golangci-lint run --config .golangci.yml ./... -# Test phase. script/test builds it alone. -# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too -FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test +# Test phase. script/test builds it alone. -race needs cgo and so a C +# compiler, which the Debian Go image ships and the alpine one does not. +# golang:1.25.4-trixie, 2026-10-15 +FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test +ARG CACHEBUST WORKDIR /src -# script/bootstrap --cgo installs the build dependencies (a C compiler, -# the libvips and libheif headers, and libvips' JPEG XL support, which -# the tests need) and downloads the Go modules. +# script/bootstrap --cgo installs the build dependencies (the libvips +# and libheif headers, and libvips' JPEG XL support, which the tests +# need) and downloads the Go modules. COPY script/ ./script/ COPY go.mod go.sum ./ RUN script/bootstrap --cgo @@ -32,11 +40,14 @@ RUN script/bootstrap --cgo COPY . . # Without -v first; on a failure, again with -v for the details, and -# the step fails even if the second run passes. -parallel 4: by default -# a package runs as many of its tests at once as the host has CPUs, and -# on a busy host they then wait so long to be scheduled that a test -# that times a request can see it return a second late. -RUN go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \ +# the step fails even if the second run passes. -count=1 because the +# mounted cache also holds test results, which go test would otherwise +# replay. -parallel 4: by default a package runs as many of its tests at +# once as the host has CPUs, and on a busy host they then wait so long +# to be scheduled that a test that times a request can see it return a +# second late. +RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; } @@ -45,38 +56,44 @@ RUN go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \ # test passed. # golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder +ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null WORKDIR /src -# Build dependencies and Go modules, as in the test phase +# Build dependencies, git among them, and Go modules, as in the test +# phase COPY script/ ./script/ COPY go.mod go.sum ./ RUN script/bootstrap --cgo +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src # Copy source code COPY . . -# VERSION is declared here, not earlier: a new value reruns only the -# build, not script/bootstrap. Given none, the version is -# `git describe --tags --always` of the .git in the build context (git -# comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH -# after one, the short commit when no tag is reachable. A context that -# carries .git and still yields no version fails the build; one without -# .git, as from a source tarball, stamps an empty version. CGO stays -# enabled for govips; -trimpath keeps build paths out of the binary, and -# -s -w leave out the symbol table and debug information. +# The VERSION build arg when one is given, otherwise +# `git describe --tags --always` on the .git in the build context: the +# tag on a tagged commit, tag-N-gHASH after one, the short commit when +# no tag is reachable. With .git present, a version that is still empty, +# dev or unknown fails the build: git is missing or could not read the +# checkout. A context without .git, as from a source tarball, stamps an +# empty version. CGO stays enabled for govips; -trimpath keeps build +# paths out of the binary, and -s -w leave out the symbol table and +# debug information. ARG VERSION -RUN version="${VERSION:-$(git describe --tags --always)}"; \ - if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \ - [ "$version" = unknown ]; }; then \ - echo "the build context carries .git but yields no version" >&2; \ - exit 1; \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ fi; \ CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \ - -ldflags "-s -w -X main.Version=${version}" \ + -ldflags="-s -w -X main.Version=${VERSION}" \ -o /pixad ./cmd/pixad # Runtime stage, and the last one: a plain `docker build .` builds this @@ -85,6 +102,7 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \ # the libvips and musl it was built with. # alpine:3.22, 2026-10-08 FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 +ARG CACHEBUST # Install runtime dependencies only. vips-jxl is libvips' JPEG XL # support, without which pixad does not start. diff --git a/README.md b/README.md index 13a6a3b..6c46fe4 100644 --- a/README.md +++ b/README.md @@ -706,9 +706,9 @@ provide: install-precommit) - `script/projectname` — output the project name ("pixa") - `script/test` — run the test suite: build the `test` phase of the - `Dockerfile`, tagged `pixa-test` + `Dockerfile`, writing no image - `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`, - tagged `pixa-lint`; the linter never runs on the host + writing no image; the linter never runs on the host - `script/fmt` — format the Go code with gofmt and the markdown with prettier (writes) - `script/fmt-check` — check the same formatting (read-only), on the host @@ -726,15 +726,20 @@ provide: - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` -Every `docker build` in these scripts passes `--no-cache`, so the lint and test -phases run on every build instead of coming from the build cache. +Every `docker build` in these scripts passes a new value of the `CACHEBUST` +build argument, which every stage of the `Dockerfile` declares first, so every +`RUN` step, the lint and test phases included, runs on every build instead of +coming from the build cache. Go's build cache is kept from build to build: the +steps that compile mount it, so a package whose inputs are unchanged is not +compiled again, and both `go test` runs pass `-count=1` so that no stored test +result is replayed. The `test` phase runs on the Debian Go image, with Debian's +libvips and libheif; the image pixa ships is built and runs on Alpine. `script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`, `script/setup` and `script/install-precommit` are the standard copies from `sneak/prompts`, kept identical to them. `script/fmt` and `script/fmt-check` are the standard copies with pixa's `gofmt` step kept before prettier. prettier -formats the markdown only: not the HTML templates, as it cannot parse a Go -template action inside a tag, and not `REPO_POLICIES.md` (see -`.prettierignore`), a copy of the one in `sneak/prompts`. +formats the markdown only, not the HTML templates, as it cannot parse a Go +template action inside a tag. ## Load Test diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 2256291..5b24098 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-09-08 +last_modified: 2026-10-15 --- This document covers repository structure, tooling, and workflow standards. Code @@ -54,34 +54,39 @@ style conventions are in separate documents: `cibuild`. `script/bootstrap` installs all dependencies idempotently and assumes nothing is present: base tools come from nix, apt, brew, or apk (detected in that order; apt runs noninteractive). For node it uses the - installed node if present; otherwise it installs a PINNED node version via - nvm, first installing nvm itself if missing — from a hash-verified GitHub - release archive (never `curl | sh`), with bash installed as an explicit - prerequisite since nvm requires bash. yarn is then pinned via - `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; - always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root, runs `script/bootstrap`, runs `script/check`, and builds the image - with the version; the Gitea workflow calls it. **`script/cibuild` runs + installed node only when its major version is the pinned one; otherwise (node + missing, or another major, such as the node 24 the CI runner image ships) it + installs the PINNED node version via nvm and installs yarn and the packages + under it. nvm itself is installed first if missing, from a hash-verified + GitHub release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. Only the major version is compared + because the `Dockerfile` stages start from a node image whose exact version is + not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned + via `corepack prepare yarn@ --activate`. Never install "latest" or + "lts"; always exact versions. `script/cibuild` runs the CI build: it changes + to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the + image with the version; the Gitea workflow calls it. **`script/cibuild` runs `script/bootstrap` first**, because the workflow checks out the repo and runs nothing else, while `script/fmt-check` runs the formatter on the host: on a pristine checkout with nothing installed the run dies there, after the - containerised gates have passed. **The bootstrap alone is not enough**: - `script/bootstrap` installs node and yarn under nvm and leaves neither on the - `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host - entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore - source nvm for the pinned node version before invoking it, exactly as - `script/bootstrap`'s own install step does. A runner carrying nothing but - docker and git then gets through `script/check`. Four further scripts are our - own extensions to the standard: `script/check` runs `script/test`, - `script/lint` and `script/fmt-check`; `script/precommit` is what the git - pre-commit hook runs, and it calls `script/check`; `script/install-precommit` - installs the git pre-commit hook (the `make hooks` target shims to it); and - `script/projectname` (literally that filename) simply outputs the project's - name. Scripts that need the name call `script/projectname` — e.g. - `script/docker` assembles its image tag from it — so those scripts stay - byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. - `go mod tidy` verification in Go repos) belong in `script/precommit`, not in - the hook itself. Model scripts are at + containerised gates have passed. **The bootstrap alone is not enough**: when + `script/bootstrap` installs node and yarn under nvm it leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs + under another node. The host entrypoints that need yarn — `script/fmt` and + `script/fmt-check` — therefore use the same test as `script/bootstrap`: when + the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, + and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the + pinned node. A runner carrying nothing but docker and git then gets through + `script/check`. Four further scripts are our own extensions to the standard: + `script/check` runs `script/test`, `script/lint` and `script/fmt-check`; + `script/precommit` is what the git pre-commit hook runs, and it calls + `script/check`; `script/install-precommit` installs the git pre-commit hook + (the `make hooks` target shims to it); and `script/projectname` (literally + that filename) simply outputs the project's name. Scripts that need the name + call `script/projectname` — e.g. `script/docker` assembles its image tag from + it — so those scripts stay byte-identical across all repos. Repo-type-specific + pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in + `script/precommit`, not in the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -104,18 +109,24 @@ style conventions are in separate documents: `lint` phase and a `test` phase, with the final stage depending on both so the image cannot be built unless they pass. For non-server repos the final stage brings up a development environment; for server repos it is the runtime image. - Dockerfiles install development prerequisites by running `script/bootstrap` - rather than duplicating installs inline; COPY `script/` and the dependency - manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before - running it. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. - **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is no separate lint file. `script/lint` and `script/test` each build one phase and nothing else: ```sh - docker build --no-cache --target lint -t "$(script/projectname)-lint" . - docker build --no-cache --target test -t "$(script/projectname)-test" . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target lint --output type=cacheonly . + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ + --target test --output type=cacheonly . ``` **A stage that is not the last one in the file is built only when the final @@ -125,10 +136,15 @@ style conventions are in separate documents: plain `docker build .` builds the last stage alone and exits 0 having linted and tested nothing. - **Every `docker build` in `script/` is tagged**, here and in - `script/cibuild` and `script/docker`. An untagged build leaves a dangling - image behind on every invocation, on every developer host and every CI - runner; a tagged one replaces the previous image. + **The gate builds write no image.** With `--output type=cacheonly` the phase + runs and a failing step fails the build, but the result is not exported. + Nothing uses those images, and writing one out is slow: a Go test phase's + image holds the toolchain. A build given neither `--output` nor `-t` writes + an untagged image and leaves it dangling, on every developer host and every + CI runner. `script/cibuild` and `script/docker` build the image that ships + and tag it, so each build replaces the previous image; each assigns the tag + on its own line before the build, so `set -e` stops it where + `script/projectname` fails. Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, `eslint`, `prettier` — never through `make lint` or `script/test`, which are @@ -143,66 +159,107 @@ style conventions are in separate documents: one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs exit non-zero with `parallel golangci-lint is running` — a status a caller cannot tell from real findings. Both have produced wrong verdicts in this - org, in both directions. A container has its own cache, its own `TMPDIR` and - a digest-pinned binary, so neither is reachable. + org, in both directions. A container has its own golangci-lint cache, its + own `TMPDIR` and a digest-pinned binary, so neither is reachable. -- **Any build that runs checks is built with `--no-cache`.** Docker invalidates - a `COPY` layer only when the copied content changes, so on an unchanged tree - the check `RUN` is served from cache, nothing executes, and the build still - exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: - `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the - four, and there is no fifth — `script/check` runs the two gate phases and - `script/fmt-check`, and builds no image of its own. A bare `docker build .` is - not evidence that anything ran: a sub-second build reporting success is a - cache hit, not a result. Never invalidate by pruning — `docker builder prune` - and friends destroy a build cache shared with every other build on the host. +- **Any build that runs checks is given a new `CACHEBUST` value.** Docker + invalidates a `COPY` layer only when the copied content changes, so on an + unchanged tree the check `RUN` is served from cache, nothing executes, and the + build still exits 0. Every `docker build` in `script/` therefore passes + `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process + id, so no two builds share a value, not even the three in one + `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and + `script/docker` are the four, and there is no fifth — `script/check` runs the + two gate phases and `script/fmt-check`, and builds no image of its own. Every + stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after + `FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps + are compared by content and need nothing. A stage without the `ARG` line is + served from cache on an unchanged tree, so a repository re-vendors the scripts + and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`: + it reruns every step too, but it also empties every cache mount, and Go's + build cache, which the canonical Go `Dockerfile` below mounts so that compiled + packages survive between check builds, would start empty every time. A bare + `docker build .` is not evidence that anything ran: a sub-second build + reporting success is a cache hit, not a result. Never invalidate by pruning — + `docker builder prune` and friends destroy a build cache shared with every + other build on the host. When a check is added or changed, prove it works by + planting a defect it must catch and watching the run fail on it, then revert + the defect. A green run alone shows neither that the check ran nor that it + covers what it should. - **The gate phases are separate stages, and the build stage depends on both.** The lint phase is based on the `golangci/golangci-lint` image (pinned by hash), so lint failures surface in seconds rather than after a full compile, - and the test phase is based on the Go image. The canonical Go repo + and the test phase is based on the Debian Go image. The canonical Go repo `Dockerfile`: ```dockerfile + # Every stage declares ARG CACHEBUST first: script/ passes a new value + # on every build, which reruns every RUN step after it. The steps that + # compile mount Go's build cache, which survives from build to build. + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint + ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN golangci-lint run --config .golangci.yml ./... + RUN --mount=type=cache,target=/root/.cache/go-build \ + golangci-lint run --config .golangci.yml ./... - # Test phase - # golang:1.x-alpine, YYYY-MM-DD + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. -count=1 because the mounted + # cache also holds test results, which go test would otherwise replay. + # golang:1.x, YYYY-MM-DD FROM golang@sha256:... AS test + ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN go test -timeout 90s -race -cover ./... || \ + RUN --mount=type=cache,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 90s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } # Build stage. Nothing is wanted from either phase above; the copies # are what make BuildKit build them first, so this stage cannot run # unless lint and test passed. # golang:1.x-alpine, YYYY-MM-DD FROM golang@sha256:... AS builder + ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null + RUN apk add --no-cache git + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN --mount=type=cache,target=/root/.cache/go-build \ + VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ # Runtime stage, and the last one FROM alpine@sha256:... + ARG CACHEBUST COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` @@ -217,23 +274,88 @@ style conventions are in separate documents: - Keep the runtime stage last, and if you add a stage after it, give it the same two copies. A plain `docker build .` builds the last stage's chain and nothing else. + - Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps + it for root), and only on the steps that compile: `golangci-lint run`, + both `go test` runs and `go build`. It keeps compiled packages between + check builds, and Go reuses one only when all its inputs are unchanged, so + it saves the compile without skipping a check. Nothing else is kept: + golangci-lint's own cache would replay stored findings in place of a lint + run, and Go builds from whatever files a kept module cache holds without + checking them again (only `go mod verify` does), so a file changed there + would go unnoticed. Go's build cache also holds test results, which is why + both `go test` runs carry `-count=1`. - If the project uses `//go:embed` directives that reference build artifacts (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint phase with `apk add`. - - `ARG VERSION=dev` is declared in the stage that compiles and supplied by - `script/docker` and `script/cibuild`; no stage may call `git describe`. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them: + + ```dockerfile + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. + A checkout whose `.git` is a file (a linked worktree, or a repository + checked out as a submodule) is the exception: that file points to a git + directory outside the build context, so the build cannot read the version + and a plain `docker build .` fails; pass the version with + `--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already + do. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` on push, and checks out the repo as its only other step. - That script bootstraps, runs the gate phases, and then builds the image, so a - successful run means every check passed; a bare `docker build .` does not + runs `script/cibuild` on push, and checks out the repo as its only other step, + with `persist-credentials: false`: `script/cibuild` needs no token, and + without it the checkout leaves the job's token in `.git/config` for every + later step. The checkout step also sets `fetch-depth: 0`, which fetches the + tags `git describe` needs: by default it clones shallow with no tags, and a + tagged repository's CI build would stamp a bare short commit id. The + workflow's `concurrency` block groups runs by workflow and branch + (`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`, + so a new push cancels the older run on the same branch, queued or running, and + no other: runs for replaced commits do not hold up the shared runner. + `script/cibuild` bootstraps, runs the gate phases, and then builds the image, + so a successful run means every check passed; a bare `docker build .` does not carry the same guarantee, because its gate phases may come from the cache. The - image build is uncached and so runs the gate phases a second time. That is the - price of the rule above, and it is worth paying: the image that ships is built - from a run of its own gates rather than from a cache entry. + image build gets a `CACHEBUST` value of its own and so runs the gate phases a + second time. That is the price of the rule above, and it is worth paying: the + image that ships is built from a run of its own gates rather than from a cache + entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the + shared runner after 20 minutes. That allows for the three Docker builds + described above (the test phase, the lint phase, then the image), each held to + the 5-minute Docker build limit below, plus the bootstrap. A separate workflow + limited to `main` by a `branches` list under `on: push` cannot be checked by + review: to try a change to it, add the feature branch to that list and push, + then remove the branch from the list again before merging. Keep any job in it + that publishes behind `if: github.ref_name == 'main'`, so the run from the + feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -286,17 +408,18 @@ style conventions are in separate documents: ``` `-count=1` is required on both invocations: it defeats Go's test _result_ - cache, so the target cannot report a pass it did not earn, and the rerun - reproduces a failure instead of replaying it. It leaves the build cache - alone, so it costs the runtime of the suite and no recompilation. + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. - Note that this is a second, independent cache, stacked below the Docker - layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26) - addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes; - it does not guarantee `go test` inside that step does any work, because the - `GOCACHE` baked into earlier image layers survives into the re-executed - step. They are two separate defects requiring two separate fixes, and a fix - for one must not be recorded as covering the other. + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. The directory lasts from one run to the next on a + developer's machine, where this target runs, and in the `test` phase of the + `Dockerfile`, which mounts it to keep compiled packages between check + builds, so the phase's `go test` runs carry `-count=1` too. A new + `CACHEBUST` value (above) is what makes that step run on an unchanged tree. Python example: @@ -324,9 +447,12 @@ style conventions are in separate documents: - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), - language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` - from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when - setting up a new repo. These patterns are written to `.gitignore`'s own + `node_modules/`, and the repo's own build outputs. Fetch the standard + `.gitignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up + a new repo. A repo's `.gitignore` is the standard file followed by the repo's + own entries, such as its binaries; a re-vendor replaces the standard part and + keeps those entries. These patterns are written to `.gitignore`'s own semantics, in which an unanchored pattern already matches at every depth; they are not a `.dockerignore` and must not be transplanted into one unmodified. @@ -340,7 +466,7 @@ style conventions are in separate documents: — which is more dangerous than a short file with no secret patterns at all, because it reads as solved and stops anyone looking. Give every depth-independent pattern the `**/` prefix and leave only genuinely - root-anchored entries unprefixed: `.git`, and the repo's own host-built + root-anchored entries unprefixed: `.claude`, and the repo's own host-built binary, written `/myapp` and never `**/myapp`, which would also match `cmd/myapp/` and delete the package directory from the context. Matching is case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so @@ -365,21 +491,24 @@ style conventions are in separate documents: directory, so a repo running agents in subdirectories still ships `services/api/.claude/` and must add its own anchored entry there. -- **Excluding `.git` means `git describe` cannot run inside any build stage, and - it fails quietly there.** In a build stage there is no repository, so - `git describe` writes nothing to stdout, `-X main.Version=` comes out empty, - the binary reports no version at all, and the build still exits 0. Compute the - version on the host and thread it in as a build arg. `script/docker` and - `script/cibuild` do this, byte-identically across repos: +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: ```sh - # Own line: a failing command substitution inside an argument does not - # trip `set -e`, so the inline form degrades to an empty constant. + # The version and the tag each get their own line: a failing command + # substitution inside an argument does not trip `set -e`, so the inline + # form degrades to an empty constant. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$(script/projectname)" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$(script/projectname)" . + -t "$tag" . ``` `--always` makes an untagged repo yield an abbreviated commit hash rather @@ -387,12 +516,12 @@ style conventions are in separate documents: fallback is applied — a live check that fires on a build from an export with no `.git` and on a repository with no commits yet. Do not fold it into the substitution as `|| echo unknown`, which makes the guard unreachable. The - Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared there because `ARG` is stage-scoped; passing `VERSION` to a repo whose Dockerfile declares no such `ARG` is ignored and costs nothing, which is why the scripts stay byte-identical. One consequence for CI: the standard - checkout action clones shallow and fetches no tags, so a repo that embeds a - tag-derived version must set `fetch-depth: 0` on its checkout step. + checkout action clones shallow and fetches no tags, so the canonical + `.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step. - **Verify `.dockerignore` by enumerating the image, not by reading the patterns.** Plant files at the root _and_ at least two directories deep, build @@ -426,12 +555,18 @@ style conventions are in separate documents: `test-support` depguard rule, where a repo names its own test-support packages by full import path. A repo adds entries there and changes nothing else, and a re-vendor carries its entries forward. The canonical golangci-lint version is - v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base image - (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`, - which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the - only pin, since no repo installs golangci-lint on the host: bumping the - version means changing it and nothing else. + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. - **`script/bootstrap` installs a pinned tool by comparing versions, never by testing presence.** An `if ! command -v ; then install; fi` guard tests @@ -453,8 +588,18 @@ style conventions are in separate documents: function defined and never invoked has the same exit status and the same empty output as one that worked. + Node is the exception to the whole-token comparison: the canonical + `script/bootstrap` compares only its major version, because the node 22 + alpine image the `Dockerfile` stages start from is not the exact pin, and + nvm cannot install a prebuilt node on alpine. + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. + - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -564,13 +709,17 @@ style conventions are in separate documents: Never edit existing migrations after release. - All repos should have an `.editorconfig` enforcing the project's indentation - settings. + settings: the standard file from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets + tabs for `Makefile` and Go files, followed by the repo's own sections, such as + one for another language it uses. A re-vendor replaces the standard part and + keeps those sections. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose body is a single call into `internal/` or `pkg/`, no project logic in @@ -601,3 +750,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/TODO.md b/TODO.md index 3c33482..22289af 100644 --- a/TODO.md +++ b/TODO.md @@ -18,7 +18,7 @@ pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work lands on `next`, and `main` receives only the milestone PR that the owner merges. `next` -is at the canonical `golangci-lint` v2.12.2 config and is green. Recent work +is at the canonical `golangci-lint` v2.14.0 config and is green. Recent work extracted the internal/magic, internal/allowlist, internal/httpfetcher, and internal/signature packages. The gosec findings from the 2026-07-06 survey are resolved. The disk cache is now size-bounded with LRU eviction @@ -30,6 +30,26 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the canonical files are re-vendored from `sneak/prompts` commit + `e3f79c4` (closes #221): `.dockerignore`, `.editorconfig`, + `.gitea/workflows/check.yml`, `.gitignore`, `.golangci.yml`, + `.prettierignore`, `.prettierrc`, `REPO_POLICIES.md` and the standard scripts + are its copies. pixa's own entries follow the standard part: `/bin`, `/data` + and the local config files in `.dockerignore`; Go's `*.log`, `*.out` and + `*.test`, `vendor/`, the binaries, `/data/`, `*.sqlite3` and the local config + files in `.gitignore`. The workflow keeps its `script/docker-smoke` step while + https://git.eeqj.de/sneak/pixa/issues/203 is open. The scripts pass a new + `CACHEBUST` build argument in place of `--no-cache`, and every `Dockerfile` + stage declares it first, so every `RUN` step still runs on every build while + Go's build cache, mounted on the steps that compile, is kept; both `go test` + runs keep `-count=1`. The lint and test builds write no image. The lint phase + runs golangci-lint v2.14.0, the test phase runs on the Debian Go image, and + the build stage marks `/src` safe for git. `script/bootstrap`, `script/fmt` + and `script/fmt-check` use the node on the `PATH` only when it has the pinned + major version. Of golangci-lint v2.14.0's two findings, an unused `nolint` + directive is removed, and gosec's new path check is marked as not applying to + the fixed config file locations. The agent guidance moves unchanged from + `CLAUDE.md` to a root `AGENTS.md`, where `REPO_POLICIES.md` keeps it. - 2026-10-15 the request metrics record any method that is not a standard one as `other` (closes #274): their `method` label was each request's method as sent, and Go's HTTP server accepts any method name, so a client sending made-up diff --git a/internal/config/config.go b/internal/config/config.go index d4b78a2..2eaf698 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -823,6 +823,7 @@ func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, erro // one whose path runs through a file, such as under a HOME of // /dev/null. One that cannot be read or does not parse is a // fatal startup error. + //nolint:gosec // G703: a fixed config location; HOME is the operator's _, statErr := os.Stat(cleanPath) if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) { continue diff --git a/internal/imgcache/cachesize.go b/internal/imgcache/cachesize.go index 51d2ee3..231a907 100644 --- a/internal/imgcache/cachesize.go +++ b/internal/imgcache/cachesize.go @@ -78,10 +78,6 @@ func (c *Cache) computeDefaultMaxBytes( computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator computed = min(computed, math.MaxInt64) - // gosec cannot see that min() above bounds computed, so it reads - // this conversion as potentially overflowing. It cannot: computed is - // at most math.MaxInt64 on every path here. - //nolint:gosec // G115: clamped to MaxInt64 by min above limit := int64(computed) limit = max(limit, DefaultCacheMaxBytesFloor) diff --git a/script/bootstrap b/script/bootstrap index fbada47..8fd0cbe 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,11 +3,13 @@ # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes NOTHING is present (not git, -# make, or go). Node is used directly if installed; otherwise it is -# installed at a pinned version via nvm (installing nvm itself first, -# from a hash-verified release archive, never curl | sh). The linter is -# never installed on the host: golangci-lint runs only in the lint phase -# of the Dockerfile (see script/lint). +# make, or go). The installed node is used only when its major version +# is the pinned one; otherwise (node missing, or another major) the +# pinned version is installed via nvm (installing nvm itself first, +# from a hash-verified release archive, never curl | sh), and yarn and +# the packages are installed under it. The linter is never installed on +# the host: golangci-lint runs only in the lint phase of the Dockerfile +# (see script/lint). # # script/bootstrap git, make, Go, and Node, Yarn and the # prettier in yarn.lock for script/fmt and @@ -110,31 +112,50 @@ ensure_nvm() { rm -rf "$tmp" } +# True when the node on PATH has the pinned major version. Only the +# major is compared, as REPO_POLICIES.md asks. Another major is not +# used: the pinned yarn 1 prints a deprecation warning under node 24. +# script/fmt and script/fmt-check carry this function unchanged, to run +# the yarn installed here. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + ensure_node() { - if ! missing node; then return 0; fi + if node_is_pinned_major; then + echo "bootstrap: using node $(node --version)" + return 0 + fi ensure_nvm nvm_sh "nvm install $NODE_VERSION" + nvm_sh "nvm use $NODE_VERSION >/dev/null && \ + echo \"bootstrap: using node \$(node --version)\"" } ensure_yarn() { + if ! node_is_pinned_major; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + return 0 + fi if ! missing yarn; then return 0; fi if ! missing corepack; then corepack enable corepack prepare "yarn@$YARN_VERSION" --activate - elif [ -s "$HOME/.nvm/nvm.sh" ]; then - nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ - corepack prepare yarn@$YARN_VERSION --activate" else npm install -g "yarn@$YARN_VERSION" fi } install_js_deps() { - if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + if node_is_pinned_major; then + yarn install --frozen-lockfile + else nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ yarn install --frozen-lockfile" - else - yarn install --frozen-lockfile fi } diff --git a/script/cibuild b/script/cibuild index 688299f..66fcca0 100755 --- a/script/cibuild +++ b/script/cibuild @@ -2,9 +2,11 @@ # script/cibuild: run the CI build. It bootstraps first: a CI runner # checks out and runs this and nothing else, and script/fmt-check runs # the formatter on the host, which a pristine checkout cannot do. -# --no-cache for the same reason as script/docker: the gate phases the -# final stage depends on are RUN steps, and a cached one is a check that -# did not run. +# A new CACHEBUST value reruns every RUN step, since a cached gate phase +# is a check that did not run; unlike --no-cache, it keeps Go's build cache. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# first; a stage without that line is served from cache on an unchanged +# tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -14,16 +16,17 @@ main() { cd "$ROOT" "$SCRIPT_DIR/bootstrap" "$SCRIPT_DIR/check" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. VERSION is computed here because .dockerignore - # excludes .git, so `git describe` in a build stage yields an empty - # version without failing. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/docker b/script/docker index c4688e8..360037a 100755 --- a/script/docker +++ b/script/docker @@ -1,8 +1,11 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# --no-cache because the gate phases the final stage depends on are RUN -# steps, and a cached one is a check that did not run. +# A new CACHEBUST value reruns every RUN step, since a cached gate phase +# is a check that did not run; unlike --no-cache, it keeps Go's build cache. +# This relies on every stage of the Dockerfile declaring ARG CACHEBUST +# first; a stage without that line is served from cache on an unchanged +# tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -10,16 +13,17 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - # Own line: a failing command substitution inside an argument does - # not trip `set -e`, so the inline form degrades silently to an - # empty constant. VERSION is computed here because .dockerignore - # excludes .git, so `git describe` in a build stage yields an empty - # version without failing. + # The version and the tag each get their own line: a failing + # command substitution inside an argument does not trip `set -e`, + # so the inline form degrades silently to an empty constant. The + # VERSION build argument takes precedence over the version a build + # stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - docker build --no-cache \ + tag="$("$SCRIPT_DIR/projectname")" + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --build-arg VERSION="$version" \ - -t "$("$SCRIPT_DIR/projectname")" . + -t "$tag" . } main "$@" diff --git a/script/fmt b/script/fmt index c7b584a..a110cc3 100755 --- a/script/fmt +++ b/script/fmt @@ -7,15 +7,25 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" # Must match the pin in script/bootstrap. NODE_VERSION="22.17.0" -# script/bootstrap installs node and yarn under nvm and leaves neither -# on the PATH of the shell that called it, so resolve the pinned -# toolchain here the way bootstrap's own install step does. nvm is a -# bash script, hence the subshell. +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. run_yarn() { - if command -v yarn >/dev/null 2>&1; then + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then exec yarn "$@" fi - if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then echo "fmt: no yarn; run script/bootstrap first" >&2 exit 1 fi diff --git a/script/fmt-check b/script/fmt-check index b8d3f00..9964da7 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -8,15 +8,25 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" # Must match the pin in script/bootstrap. NODE_VERSION="22.17.0" -# script/bootstrap installs node and yarn under nvm and leaves neither -# on the PATH of the shell that called it, so resolve the pinned -# toolchain here the way bootstrap's own install step does. nvm is a -# bash script, hence the subshell. +# True when the node on PATH has the pinned major version: the same +# function as in script/bootstrap, which uses it to decide where it +# installs yarn. +node_is_pinned_major() { + if ! command -v node >/dev/null 2>&1; then return 1; fi + installed="$(node --version)" + installed="${installed#v}" + [ "${installed%%.*}" = "${NODE_VERSION%%.*}" ] +} + +# Run the yarn script/bootstrap installed: the yarn on PATH when the +# node on PATH has the pinned major version, and otherwise yarn under +# the pinned node in nvm, which bootstrap leaves off the PATH of the +# shell that called it. nvm is a bash script, hence the subshell. run_yarn() { - if command -v yarn >/dev/null 2>&1; then + if node_is_pinned_major && command -v yarn >/dev/null 2>&1; then exec yarn "$@" fi - if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + if node_is_pinned_major || [ ! -s "$HOME/.nvm/nvm.sh" ]; then echo "fmt-check: no yarn; run script/bootstrap first" >&2 exit 1 fi diff --git a/script/lint b/script/lint index 2d8b075..c4c459e 100755 --- a/script/lint +++ b/script/lint @@ -5,9 +5,12 @@ # make its answer untrustworthy. # # The phase is not the last stage in the file, so it is built only when -# --target names it. --no-cache because a cached lint layer is a lint -# that did not run. The tag makes each build replace the previous image -# instead of leaving a dangling one behind. +# --target names it. A new CACHEBUST value reruns every RUN step, since +# a cached lint layer is a lint that did not run; unlike --no-cache, it +# keeps Go's build cache. This relies on every stage of the Dockerfile +# declaring ARG CACHEBUST first; a stage without that line is served +# from cache on an unchanged tree, with no warning. +# --output type=cacheonly writes no image, since nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -15,9 +18,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --no-cache \ + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --target lint \ - -t "$("$SCRIPT_DIR/projectname")-lint" . + --output type=cacheonly . } main "$@" diff --git a/script/test b/script/test index cd239f2..a56c4a1 100755 --- a/script/test +++ b/script/test @@ -2,8 +2,12 @@ # script/test: run the test suite. Testing is a phase of the Dockerfile # and this builds that phase alone, on the same terms as script/lint: # --target because a phase that is not the last stage is built only when -# named, --no-cache because a cached test layer is a test that did not -# run, and a tag so each build replaces the previous image. +# named, and a new CACHEBUST value because a cached test layer is a test +# that did not run; unlike --no-cache, it keeps Go's build cache. This +# relies on every stage of the Dockerfile declaring ARG CACHEBUST first; +# a stage without that line is served from cache on an unchanged tree, +# with no warning. --output type=cacheonly writes no image, since +# nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -11,9 +15,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --no-cache \ + docker build --build-arg CACHEBUST="$(date +%s)-$$" \ --target test \ - -t "$("$SCRIPT_DIR/projectname")-test" . + --output type=cacheonly . } main "$@" From f743930f714761bb6af2fb6db4a984e37eca5970 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 15 Oct 2026 23:07:22 +0200 Subject: [PATCH 077/131] Default max_concurrent_processing to half the CPUs pixa sees (closes #242) The default max_concurrent_processing was one image per CPU pixa sees, so in a container with no CPU limit on a 48-CPU host pixa processed 48 images at once. It is now half the CPUs, rounded down and at least 1, defined once in internal/config; PIXA_MAX_CONCURRENT_PROCESSING stays the override, and a value that is not a whole number of at least 1 stops startup. "Running under upaas" gives the worst-case memory, 4.2 GiB per image (the largest peak measured over every source format and kind at 8192x8192) plus the upstream buffers, and the temporary file libvips keeps under /tmp. The two existing tests expect the new default, as the owner allowed. Model: opus-5-5 --- README.md | 40 +++++++++++-- TODO.md | 12 ++++ configs/config.example.yml | 9 +-- .../concurrency_limits_internal_test.go | 57 +++++++++++++++++-- internal/config/config.go | 12 +++- internal/imageprocessor/imageprocessor.go | 15 +++-- ...max_concurrent_processing_internal_test.go | 32 +++++++++-- internal/imgcache/service.go | 2 +- 8 files changed, 149 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 6c46fe4..2666d03 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,34 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: the cache can grow to three quarters or more of the space they all have left, and to all of it when that is 500 MiB or less. - the rest are in the table under Configuration below +- **Memory:** pixa can use up to `max_concurrent_processing` times 4.2 GiB for + the images it processes at once, plus `upstream_connections` times + `upstream_max_response_size` for the source images it has fetched that wait + for processing. 4.2 GiB is the most resident memory was measured to grow while + pixa, with pixad's libvips settings, processed one 8192x8192 source, the + largest it accepts, saving it at full size to each output format in turn, one + at a time. The sources were a JPEG, a GIF, a WebP, an 8-bit and a 16-bit PNG, + an 8-bit and a 12-bit AVIF, and an 8-bit, a 16-bit and a 32-bit floating-point + JPEG XL; memory grew most for the 16-bit JPEG XL, of random pixels, saved as + WebP. At the defaults that is half the CPUs pixa sees times 4.2 GiB, plus 64 + times 50 MiB: on a host where pixa sees 48 CPUs, 24 times 4.2 GiB plus 3.1 + GiB, about 104 GiB. Set `PIXA_MAX_CONCURRENT_PROCESSING` and + `PIXA_UPSTREAM_CONNECTIONS` so that this fits the memory the container may + use. +- **Temporary files:** while pixa processes a large source, libvips writes its + decoded pixels to a file in `/tmp`. The file disappears from `/tmp` once it is + written, so a listing of `/tmp` does not show it, but it keeps its space, on + disk or, where `/tmp` is a tmpfs, in memory, until the image is done. For a + GIF, WebP, AVIF or JPEG XL source the decoded pixels are held in memory as + well as written to that file, and the 4.2 GiB above includes that memory. For + an 8192x8192 source the file was measured on pixad at 192 MiB for an 8-bit RGB + JPEG, 512 MiB for a 16-bit RGBA PNG or JPEG XL, and 1 GiB for a JPEG XL of + 32-bit floating-point RGBA, the largest measured: up to 1 GiB per image + processed at once, 24 GiB at the defaults on a host where pixa sees 48 CPUs. + This `/tmp` is in the container's own filesystem, not on the volume at + `/var/lib/pixa`, and `cache_max_bytes` does not count it. The 4.2 GiB above + does not include these files; where `/tmp` is a tmpfs, they are memory on top + of it. - **Health check:** the image's `HEALTHCHECK` requests `/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds after a deploy and marks the deploy failed unless it is `healthy`. The probe @@ -566,7 +594,7 @@ tell. With no file, pixa uses the environment and the defaults. | `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` | | `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` | | `PIXA_UPSTREAM_CONNECTIONS` | `upstream_connections` | Concurrent connections to all upstream hosts together; default `64` | -| `PIXA_MAX_CONCURRENT_PROCESSING` | `max_concurrent_processing` | Images processed at once; default the number of CPUs | +| `PIXA_MAX_CONCURRENT_PROCESSING` | `max_concurrent_processing` | Images processed at once; default half the CPUs, at least 1 | | `PIXA_IMAGE_REQUESTS_PER_MINUTE` | `image_requests_per_minute` | Image requests per client per minute; `0` turns it off; default `600` | | `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` | | `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB | @@ -652,10 +680,12 @@ Key settings in more detail: does, and `downstream_timeout` has not ended first, the request is answered 503 with the error `server busy, try again later` - `max_concurrent_processing` — the most images decoded and encoded at once; - default the number of CPUs pixa can use (`GOMAXPROCS`), which follows a - container's CPU limit. A request that finds all of them in use waits up to 10 - seconds for one to free up; if none does, and `downstream_timeout` has not - ended first, it is answered 503 the same way + default half the CPUs pixa can use (`GOMAXPROCS`, which follows a container's + CPU limit), rounded down, and at least 1. A value that is not a whole number + of at least 1 aborts startup. A request that finds all of them in use waits up + to 10 seconds for one to free up; if none does, and `downstream_timeout` has + not ended first, it is answered 503 the same way. Each one can take a lot of + memory: see "Running under upaas" - `image_requests_per_minute` — how many requests one client may send to the image routes per minute (see the limit on them under Routes); default `600`. `0` turns the limit off, and a negative or non-integer value aborts startup. diff --git a/TODO.md b/TODO.md index 22289af..1b59abf 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,18 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees, + rounded down, and at least 1 (closes #242): it defaulted to one image per CPU, + and a container with no CPU limit sees every CPU of the host, so on a 48-CPU + host pixa processed 48 images at once, each of which can take about 4.2 GiB. + `config.DefaultMaxConcurrentProcessing` gives the default to the config and to + `imageprocessor.New`; `PIXA_MAX_CONCURRENT_PROCESSING` still overrides it. + "Running under upaas" in `README.md` gives the worst-case memory, measured, + and tells the operator to set `PIXA_MAX_CONCURRENT_PROCESSING` and + `PIXA_UPSTREAM_CONNECTIONS` to fit the container's memory. It also gives the + file libvips writes to `/tmp` for each large source, up to 1 GiB, measured, + which is outside `/var/lib/pixa` and `cache_max_bytes` and is memory where + `/tmp` is a tmpfs. - 2026-10-15 the canonical files are re-vendored from `sneak/prompts` commit `e3f79c4` (closes #221): `.dockerignore`, `.editorconfig`, `.gitea/workflows/check.yml`, `.gitignore`, `.golangci.yml`, diff --git a/configs/config.example.yml b/configs/config.example.yml index cdf1022..6a0ab17 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -108,10 +108,11 @@ upstream_connections_per_host: 20 # downstream_timeout has ended first. upstream_connections: 64 -# Maximum number of images decoded and encoded at once (default: the -# number of CPUs pixa can use, which follows a container's CPU limit). A -# request that finds none free waits up to 10 seconds for one, and if none -# frees up it is answered 503, unless downstream_timeout has ended first. +# Maximum number of images decoded and encoded at once (default: half the +# number of CPUs pixa can use, at least 1; the number of CPUs follows a +# container's CPU limit). A request that finds none free waits up to 10 +# seconds for one, and if none frees up it is answered 503, unless +# downstream_timeout has ended first. # max_concurrent_processing: 4 # How many requests to the image routes (/v1/image/ and /v1/e/) one client diff --git a/internal/config/concurrency_limits_internal_test.go b/internal/config/concurrency_limits_internal_test.go index c5f4b20..51406e3 100644 --- a/internal/config/concurrency_limits_internal_test.go +++ b/internal/config/concurrency_limits_internal_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "runtime" "testing" ) @@ -12,8 +13,8 @@ const ( ) // TestOmittedConcurrencyLimitsUseDefaults checks that an omitted -// max_concurrent_processing is the number of CPUs Go uses and an omitted -// upstream_connections is 64. +// max_concurrent_processing is half the CPUs Go uses, and at least 1, and +// an omitted upstream_connections is 64. func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) { t.Parallel() @@ -22,9 +23,9 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) { t.Fatalf("minimal config should be valid, got error: %v", err) } - if c.MaxConcurrentProcessing != runtime.GOMAXPROCS(0) { - t.Errorf("MaxConcurrentProcessing = %d, want %d, one per CPU", - c.MaxConcurrentProcessing, runtime.GOMAXPROCS(0)) + if c.MaxConcurrentProcessing != max(runtime.GOMAXPROCS(0)/2, 1) { + t.Errorf("MaxConcurrentProcessing = %d, want %d, half the CPUs", + c.MaxConcurrentProcessing, max(runtime.GOMAXPROCS(0)/2, 1)) } if c.UpstreamConnections != 64 { @@ -32,6 +33,36 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) { } } +// TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne checks +// that an omitted max_concurrent_processing is half the CPUs Go uses, +// rounded down, and 1 when Go uses one CPU. +// +//nolint:paralleltest // it sets GOMAXPROCS, which other tests read +func TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne(t *testing.T) { + cpusBefore := runtime.GOMAXPROCS(0) + + t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) }) + + for _, tc := range []struct{ cpus, want int }{ + {1, 1}, {2, 1}, {3, 1}, {8, 4}, + } { + runtime.GOMAXPROCS(tc.cpus) + + c, err := configFromYAML(t, signingKeyLine) + if err != nil { + t.Fatalf("minimal config should be valid, got error: %v", err) + } + + t.Logf("GOMAXPROCS %d: MaxConcurrentProcessing %d", + tc.cpus, c.MaxConcurrentProcessing) + + if c.MaxConcurrentProcessing != tc.want { + t.Errorf("GOMAXPROCS %d: MaxConcurrentProcessing = %d, want %d", + tc.cpus, c.MaxConcurrentProcessing, tc.want) + } + } +} + // TestExplicitConcurrencyLimitsAreUsed checks that valid values for the // two limits are used as given. func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) { @@ -160,3 +191,19 @@ func TestConcurrencyLimitsFromEnvironment(t *testing.T) { }) } } + +// TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup checks +// that a PIXA_MAX_CONCURRENT_PROCESSING that is not a whole number of at +// least 1, an empty one included, aborts startup naming the variable and +// the key instead of taking the default. +func TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup(t *testing.T) { + for _, value := range []string{"", " ", "0.5", "half", "-1", "4 images"} { + t.Run(fmt.Sprintf("value %q", value), func(t *testing.T) { + t.Setenv(testMaxConcurrentProcessingVar, value) + + _, err := configFromYAML(t, signingKeyLine) + wantStartupError(t, err, + testMaxConcurrentProcessingVar, keyMaxConcurrentProcessing) + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 2eaf698..ff4ac6c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,6 +37,14 @@ const ( DefaultImageRequestsPerMinute = 600 ) +// DefaultMaxConcurrentProcessing returns the most images processed at once +// when max_concurrent_processing is omitted: half the CPUs Go uses +// (runtime.GOMAXPROCS(0), which follows a container's CPU limit), rounded +// down, and at least 1. +func DefaultMaxConcurrentProcessing() int { + return max(runtime.GOMAXPROCS(0)/2, 1) //nolint:mnd // half the CPUs +} + // Configuration key names. const ( keyDebug = "debug" @@ -297,10 +305,8 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost), UpstreamConnections: loader.intVal( keyUpstreamConnections, DefaultUpstreamConnections), - // Decoding and encoding are CPU-bound, so the default is one image - // per CPU Go uses, which follows a container's CPU limit. MaxConcurrentProcessing: loader.intVal( - keyMaxConcurrentProcessing, runtime.GOMAXPROCS(0)), + keyMaxConcurrentProcessing, DefaultMaxConcurrentProcessing()), UpstreamFetchTimeout: loader.durationVal( keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout), UpstreamMaxResponseSize: loader.int64Val( diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index ca033e7..2a1fbf5 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -7,11 +7,11 @@ import ( "errors" "fmt" "io" - "runtime" "sync" "time" "github.com/davidbyttow/govips/v2/vips" + "sneak.berlin/go/pixa/internal/config" ) // vipsOnce ensures vips is initialized exactly once. @@ -20,11 +20,10 @@ import ( var vipsOnce sync.Once // initVips initializes libvips with quiet logging, one worker thread per -// image and no operation cache. Process already works on one image per CPU -// by default, so more threads per image would only compete for the CPUs. -// Each request decodes different source bytes, so the operation cache -// would rarely be hit and would hold memory outside MaxConcurrentProcessing; -// repeated requests are served from pixa's disk cache instead. +// image and no operation cache. Each request decodes different source +// bytes, so the operation cache would rarely be hit and would hold memory +// outside MaxConcurrentProcessing; repeated requests are served from +// pixa's disk cache instead. func initVips() { vipsOnce.Do(func() { vips.LoggingSettings(nil, vips.LogLevelError) @@ -179,7 +178,7 @@ type Params struct { // If <= 0, DefaultMaxInputBytes is used. MaxInputBytes int64 // MaxConcurrentProcessing is the most images processed at once. - // If <= 0, the number of CPUs Go uses (runtime.GOMAXPROCS(0)) is used. + // If <= 0, config.DefaultMaxConcurrentProcessing() is used. MaxConcurrentProcessing int } @@ -195,7 +194,7 @@ func New(params Params) *ImageProcessor { maxConcurrentProcessing := params.MaxConcurrentProcessing if maxConcurrentProcessing <= 0 { - maxConcurrentProcessing = runtime.GOMAXPROCS(0) + maxConcurrentProcessing = config.DefaultMaxConcurrentProcessing() } return &ImageProcessor{ diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go index ba5388c..2dbbbbf 100644 --- a/internal/imageprocessor/max_concurrent_processing_internal_test.go +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -115,14 +115,14 @@ func waitForEntries(t *testing.T, entered <-chan struct{}, count int) { } } -func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) { +func TestNewDefaultsMaxConcurrentProcessingToHalfTheCPUs(t *testing.T) { t.Parallel() for _, limit := range []int{0, -1} { proc := New(Params{MaxConcurrentProcessing: limit}) - if got := cap(proc.processingSemaphore); got != runtime.GOMAXPROCS(0) { - t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, one per CPU", - limit, got, runtime.GOMAXPROCS(0)) + if got := cap(proc.processingSemaphore); got != max(runtime.GOMAXPROCS(0)/2, 1) { + t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, half the CPUs", + limit, got, max(runtime.GOMAXPROCS(0)/2, 1)) } } @@ -132,6 +132,30 @@ func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) { } } +// TestNewDefaultIsHalfTheCPUsAndAtLeastOne checks that New, given no +// MaxConcurrentProcessing, makes one slot for every two CPUs Go uses, +// rounded down, and one slot when Go uses one CPU. +// +//nolint:paralleltest // it sets GOMAXPROCS, which other tests read +func TestNewDefaultIsHalfTheCPUsAndAtLeastOne(t *testing.T) { + cpusBefore := runtime.GOMAXPROCS(0) + + t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) }) + + for _, tc := range []struct{ cpus, want int }{ + {1, 1}, {2, 1}, {3, 1}, {8, 4}, + } { + runtime.GOMAXPROCS(tc.cpus) + + got := cap(New(Params{}).processingSemaphore) + t.Logf("GOMAXPROCS %d: %d slots", tc.cpus, got) + + if got != tc.want { + t.Errorf("GOMAXPROCS %d: %d slots, want %d", tc.cpus, got, tc.want) + } + } +} + // TestProcessNeverExceedsMaxConcurrentProcessing starts more Process calls // than MaxConcurrentProcessing allows and holds the first ones inside // Process until the test lets them go. No more than the limit may be diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 1806a42..212e479 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -53,7 +53,7 @@ type ServiceConfig struct { // Allowlist is the list of hosts that don't require signatures Allowlist []string // MaxConcurrentProcessing is the most images processed at once; zero - // uses the image processor's default, one per CPU + // uses the image processor's default, half the CPUs MaxConcurrentProcessing int // Logger for logging Logger *slog.Logger From 22bfc432b676bf115b5bfc7b818e9d6e581359e9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 00:17:33 +0200 Subject: [PATCH 078/131] Document how a source query goes into an image URL, and accept it (closes #255) The README gave no way to put a source URL's query into a signed image URL, and GenerateSignedURL built one that failed: the route parsed chi's wildcard, which keeps the path as the client wrote it, so one %2F, %3B or %2C left the %3F undecoded and the signature failed. The route now parses the decoded path, GenerateSignedURL writes Go's own encoding of it, and the README says the ? before the query, any ? in it and every % must be percent-encoded, with a signed example. The path traversal check no longer reads the query and accepts a literal %, while a double-encoded .. stays refused. A percent-escape in a source path is now decoded once. Model: opus-5-5 --- README.md | 25 +++ TODO.md | 14 ++ internal/handlers/image.go | 16 +- .../handlers/image_signature_internal_test.go | 163 ++++++++++++++++++ internal/imgcache/urlparser.go | 47 +++-- internal/imgcache/urlparser_internal_test.go | 65 +++++++ internal/signature/signature.go | 33 ++-- internal/signature/signature_test.go | 38 ++++ 8 files changed, 353 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 2666d03..4fc97e9 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,20 @@ once, is refused with 400. both optional (values under Signature Specification). Both are part of what is cached, so each value of either is a separate cached image. +A source URL's query follows its path after `%3F`, the encoded `?`: +`/v1/image//%3F/.`. pixa decodes the +percent-escapes in the image URL's path once, then reads the host up to the +first `/`, the size and format after the last `/`, and between them the source +path up to the first `?` and the source query after it. So the `?` before the +query, any `?` in the query, and every `%` in the source path or query must be +percent-encoded, as `%3F` and `%25`. Any other character may be written as +itself or percent-encoded: `/`, `;`, `,`, `=` and `&` work either way. For +example, +`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg` at +800x600 WebP is +`/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp`, +signed as shown under Signature Specification. + A size with one side `0` has that side scaled to keep the source's proportions, and is refused with 400 when that side would come out above 8192, the most either side may be; so is a size with `fit=outside`, which scales the image to @@ -535,6 +549,17 @@ For the same image at quality 40 with fit `contain`, the input ends in and the URL is `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`. +For a source URL with a query, +`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg`, at +800x600 WebP with the same key and expiration and the default quality and fit: + +1. Build input, with the query's `:` written as `%3A` and its `%` as `%25`: + `cdn.example.com:/img:url=https%3A//origin.example.com/my%2520cat.jpg:800:600:webp:1704067200:85:cover` +2. The signature is `37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=` +3. URL, with the `?` before the query written as `%3F` and the query's `%` as + `%25` (see Routes): + `/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp?sig=37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=&exp=1704067200` + **Allowlist patterns:** - **Exact match**: `cdn.example.com` — matches only that host diff --git a/TODO.md b/TODO.md index 1b59abf..0ef9f6b 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,20 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the README says how a source URL with a query is written into an + image URL, and the image route accepts it (closes #255): the query follows the + source path after `%3F`. pixa now reads the image URL's path decoded once, so + the `?` before the query, any `?` in it and every `%` in the source path or + query must be percent-encoded, and any other character may be. The route used + to parse chi's wildcard, which holds the path as the client wrote it whenever + that is not Go's own encoding of it; a `/`, `;` or `,` in the query written as + `%2F`, `%3B` or `%2C`, as `GenerateSignedURL` wrote them, kept the `%3F` from + being read as `?`, and the request failed with 401. `GenerateSignedURL` now + writes Go's own encoding of the path. The README has an example with a query + and its signature. The path traversal check no longer reads the source query, + and keeps a `%` that does not start a percent-escape as it is instead of + refusing the URL, so a source path or query holding `%25` is accepted, while a + double-encoded `..` next to such a `%` is still refused. - 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees, rounded down, and at least 1 (closes #242): it defaulted to one image per CPU, and a container with no CPU limit sees every CPU of the host, so on a 48-CPU diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 9b28da1..96dad32 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -9,7 +9,6 @@ import ( "strconv" "time" - "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" @@ -80,20 +79,21 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc { } } -// parseImageRequest parses the wildcard path and query parameters into +// parseImageRequest parses the image URL's path and query parameters into // an ImageRequest. On invalid input it writes an error response and // returns false. func (s *Handlers) parseImageRequest( w http.ResponseWriter, r *http.Request, ) (*imgcache.ImageRequest, bool) { - // Get the wildcard path from chi - pathParam := chi.URLParam(r, "*") - - // Parse the URL path - parsed, err := imgcache.ParseImagePath(pathParam) + // The path is parsed decoded once, as r.URL.Path holds it, and not from + // chi's wildcard: chi routes on the path as the client wrote it whenever + // that is not Go's own encoding of it, so its wildcard would still hold + // the %3F before a source query when, say, a "/" in the query is written + // as %2F + parsed, err := imgcache.ParseImageURL(r.URL.Path) if err != nil { s.log.Warn("failed to parse image URL", - "path", pathParam, + "path", r.URL.Path, "error", err, ) s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest) diff --git a/internal/handlers/image_signature_internal_test.go b/internal/handlers/image_signature_internal_test.go index c057514..babdb1e 100644 --- a/internal/handlers/image_signature_internal_test.go +++ b/internal/handlers/image_signature_internal_test.go @@ -169,3 +169,166 @@ func TestHandleImage_InvalidExp_Returns400(t *testing.T) { }) } } + +// TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies sends URLs built +// by the service's signed-URL generator for source URLs with a query to the +// route, through a chi router: chi routes on the path as the client wrote it +// whenever that is not Go's own encoding of it. A 502 means the signature +// passed, so the route read the source path and query that were signed. +func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + }{ + {"slash and colon", "url=https://origin.example.com/cat.jpg"}, + {"semicolon", "a=1;b=2"}, + {"comma", "crop=0,0,50,50"}, + {"equals and ampersand", "a=1&b=2"}, + {"percent-escape", "name=my%20cat"}, + {"question mark", "next=/a?b=1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fix := setupTestHandler(t) + + signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{ + SourceHost: signedHost, + SourcePath: photoPath, + SourceQuery: tt.query, + Size: imgcache.Size{Width: 50, Height: 50}, + Format: imgcache.FormatJPEG, + }, time.Hour) + if err != nil { + t.Fatalf("GenerateSignedURL() error = %v", err) + } + + status := getImage(t, fix, signedURL) + if status != http.StatusBadGateway { + t.Errorf("status = %d, want %d", status, http.StatusBadGateway) + } + }) + } +} + +// TestHandleImage_SourceQueryEncodedAnyWay signs one source URL with a query +// and writes it into the image URL in ways the README allows: the "?" before +// the query and each "%" in it percent-encoded, and every other character as +// itself or percent-encoded. A 502 means the signature passed. +func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) { + t.Parallel() + + expires := time.Now().Add(time.Hour) + signer := signature.New("test-signing-key-must-be-32-chars") + sig := signer.Sign(&signature.Request{ + SourceHost: signedHost, + SourcePath: photoPath, + SourceQuery: "url=https://origin.example.com/my%20cat.jpg", + Width: 50, + Height: 50, + Format: string(imgcache.FormatJPEG), + Quality: 85, + FitMode: string(imgcache.FitCover), + Expires: expires, + }) + + tests := []struct { + name string + writtenQuery string + }{ + {"only ? and % encoded", + "%3Furl=https://origin.example.com/my%2520cat.jpg"}, + {"every reserved character encoded", + "%3Furl%3Dhttps%3A%2F%2Forigin.example.com%2Fmy%2520cat.jpg"}, + {"lower-case hex digits", + "%3furl=https://origin.example.com/my%2520cat.jpg"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + target := fmt.Sprintf("/v1/image/%s%s%s/50x50.jpeg?sig=%s&exp=%d", + signedHost, photoPath, tt.writtenQuery, sig, expires.Unix()) + + status := getImage(t, setupTestHandler(t), target) + if status != http.StatusBadGateway { + t.Errorf("status = %d, want %d", status, http.StatusBadGateway) + } + }) + } +} + +// TestHandleImage_SourceURLHoldingPercent25Verifies signs a source path and a +// source query that each hold %25, the percent-escape of a literal "%", and +// sends each to the route written as the README says, with that "%" written as +// %25, and as GenerateSignedURL writes it. A 502 means the signature passed. +func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sourcePath string + sourceQuery string + // writtenPathAndQuery is the source path and query as the README says + // to write them into the image URL + writtenPathAndQuery string + }{ + {"path", "/sale/50%25off.jpg", "", "/sale/50%2525off.jpg"}, + {"query", photoPath, "discount=50%25", photoPath + "%3Fdiscount=50%2525"}, + } + + for _, tt := range tests { + t.Run(tt.name+" written by the README rule", func(t *testing.T) { + t.Parallel() + + expires := time.Now().Add(time.Hour) + signer := signature.New("test-signing-key-must-be-32-chars") + sig := signer.Sign(&signature.Request{ + SourceHost: signedHost, + SourcePath: tt.sourcePath, + SourceQuery: tt.sourceQuery, + Width: 50, + Height: 50, + Format: string(imgcache.FormatJPEG), + Quality: 85, + FitMode: string(imgcache.FitCover), + Expires: expires, + }) + + target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d", + signedHost, tt.writtenPathAndQuery, sig, expires.Unix()) + + status := getImage(t, setupTestHandler(t), target) + if status != http.StatusBadGateway { + t.Errorf("status = %d, want %d", status, http.StatusBadGateway) + } + }) + + t.Run(tt.name+" written by GenerateSignedURL", func(t *testing.T) { + t.Parallel() + + fix := setupTestHandler(t) + + signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{ + SourceHost: signedHost, + SourcePath: tt.sourcePath, + SourceQuery: tt.sourceQuery, + Size: imgcache.Size{Width: 50, Height: 50}, + Format: imgcache.FormatJPEG, + }, time.Hour) + if err != nil { + t.Fatalf("GenerateSignedURL() error = %v", err) + } + + status := getImage(t, fix, signedURL) + if status != http.StatusBadGateway { + t.Errorf("status = %d, want %d", status, http.StatusBadGateway) + } + }) + } +} diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index 4247be6..1794d1f 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -47,6 +47,9 @@ func ValidateDimension(name string, value int) error { var sizeFormatRegex = regexp.MustCompile( `^(\d+)x(\d+)(?:\.(\w+))?$|^(orig)(?:\.(\w+))?$`) +// percentEscapeRegex matches one percent-escape, such as "%2e" +var percentEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}`) + // ParsedURL contains the parsed components of an image proxy URL. type ParsedURL struct { // Host is the source origin host (e.g., "cdn.example.com") @@ -61,9 +64,8 @@ type ParsedURL struct { Format ImageFormat } -// ParseImagePath parses the path captured by chi's wildcard: -// //., or // for JPEG XL -// This is the primary entry point when using chi routing. +// ParseImagePath parses an image URL's path after /v1/image/, decoded once: +// //., or // for JPEG XL. // Examples: // - cdn.example.com/photos/cat.jpg/800x600.webp // - cdn.example.com/photos/cat.jpg/0x0.jpeg @@ -79,9 +81,10 @@ func ParseImagePath(path string) (*ParsedURL, error) { return parseImageComponents(path) } -// ParseImageURL parses a full URL path like /v1/image///., -// or /v1/image/// for JPEG XL -// Use ParseImagePath instead when working with chi's wildcard capture. +// ParseImageURL parses an image URL's path, decoded once as r.URL.Path holds +// it: /v1/image///., or /v1/image/// +// for JPEG XL. A source query follows the source path after a "?", which the +// URL writes as %3F. func ParseImageURL(urlPath string) (*ParsedURL, error) { // Remove the /v1/image/ prefix const prefix = "/v1/image/" @@ -100,12 +103,6 @@ func ParseImageURL(urlPath string) (*ParsedURL, error) { // parseImageComponents parses //., or // // for JPEG XL. func parseImageComponents(remainder string) (*ParsedURL, error) { - // Check for path traversal before any other processing - err := checkPathTraversal(remainder) - if err != nil { - return nil, err - } - // Find the last path segment, which holds "size" or "size.format" lastSlash := strings.LastIndex(remainder, "/") if lastSlash == -1 { @@ -161,6 +158,13 @@ func parseImageComponents(remainder string) (*ParsedURL, error) { path = path[:qIndex] } + // The source query names no file on the upstream, so only the host and + // the source path are checked for path traversal + err = checkPathTraversal(host + path) + if err != nil { + return nil, err + } + // Ensure path starts with / if !strings.HasPrefix(path, "/") { path = "/" + path @@ -186,12 +190,7 @@ func checkPathTraversal(path string) error { // Decode multiple times to catch double-encoding decoded := path for range 3 { - newDecoded, err := url.PathUnescape(decoded) - if err != nil { - // Malformed encoding is suspicious - return ErrPathTraversal - } - + newDecoded := decodePercentEscapes(decoded) if newDecoded == decoded { break } @@ -232,6 +231,18 @@ func checkPathTraversal(path string) error { return nil } +// decodePercentEscapes decodes every percent-escape in s, such as %2e, and +// keeps a "%" that does not start one as it is, where url.PathUnescape would +// refuse the whole string. A source path holding %25, a literal "%", has such +// a "%" after one more decoding. +func decodePercentEscapes(s string) string { + return percentEscapeRegex.ReplaceAllStringFunc(s, func(escape string) string { + decoded, _ := url.PathUnescape(escape) + + return decoded + }) +} + // parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600" func parseSizeFormat(s string) (Size, ImageFormat, error) { matches := sizeFormatRegex.FindStringSubmatch(s) diff --git a/internal/imgcache/urlparser_internal_test.go b/internal/imgcache/urlparser_internal_test.go index 518a04e..603d516 100644 --- a/internal/imgcache/urlparser_internal_test.go +++ b/internal/imgcache/urlparser_internal_test.go @@ -407,6 +407,71 @@ func TestParseImagePath_PathTraversal(t *testing.T) { } } +// TestParseImageURL_SourceURLHoldingPercent25 verifies that a source path and +// a source query holding %25, the percent-escape of a literal "%", are +// accepted as they are, although decoding them once more leaves a "%" that +// does not start a percent-escape. +func TestParseImageURL_SourceURLHoldingPercent25(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + want *ParsedURL + }{ + { + name: "path", + input: "/v1/image/cdn.example.com/sale/50%25off.jpg/800x600.webp", + want: &ParsedURL{ + Host: testHostCDN, Path: "/sale/50%25off.jpg", + Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + }, + }, + { + name: "query", + input: "/v1/image/cdn.example.com/photos/cat.jpg?discount=50%25/800x600.webp", + want: &ParsedURL{ + Host: testHostCDN, Path: testPathCat, Query: "discount=50%25", + Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := ParseImageURL(tt.input) + if err != nil { + t.Fatalf("ParseImageURL() unexpected error = %v", err) + } + + assertParsedURL(t, got, tt.want) + }) + } +} + +// TestParseImageURL_DoubleEncodedTraversalNextToPercent verifies that a +// double-encoded ".." is still refused when the path also holds a "%" that +// does not start a percent-escape, either as written or after one more +// decoding. +func TestParseImageURL_DoubleEncodedTraversalNextToPercent(t *testing.T) { + t.Parallel() + + for _, input := range []string{ + "/v1/image/cdn.example.com/photos/%252e%252e/secret/50%25off.jpg/800x600.jpeg", + "/v1/image/cdn.example.com/photos/%252e%252e/secret/100%/800x600.jpeg", + } { + _, err := ParseImageURL(input) + t.Logf("ParseImageURL(%q) error = %v", input, err) + + if !errors.Is(err, ErrPathTraversal) { + t.Errorf("ParseImageURL(%q) error = %v, want ErrPathTraversal", + input, err) + } + } +} + // errorIs checks if err matches target (handles wrapped errors). func errorIs(err, target error) bool { if errors.Is(err, target) { diff --git a/internal/signature/signature.go b/internal/signature/signature.go index a683e39..a507240 100644 --- a/internal/signature/signature.go +++ b/internal/signature/signature.go @@ -126,31 +126,20 @@ func (s *Signer) GenerateSignedURL( sizeStr = fmt.Sprintf("%dx%d", req.Width, req.Height) } - // Build the path. - // When a source query is present, it is embedded as a path segment - // (e.g. /host/path?query/size.fmt) so that the URL parser can extract - // it from the last-slash split. The "?" inside a path segment is - // percent-encoded by clients but chi delivers it decoded, which is - // exactly what the URL parser expects. - var path string + // Build the path. A source query follows the source path after a "?". + // The image route reads the path decoded once, so the whole path is + // written in Go's own percent-encoding of a URL path, which writes "?" + // as %3F and "%" as %25, and leaves "/", ";", ",", "=" and "&" as they + // are. + sourcePathAndQuery := req.SourcePath if req.SourceQuery != "" { - path = fmt.Sprintf("/v1/image/%s%s%%3F%s/%s.%s", - req.SourceHost, - req.SourcePath, - url.PathEscape(req.SourceQuery), - sizeStr, - req.Format, - ) - } else { - path = fmt.Sprintf("/v1/image/%s%s/%s.%s", - req.SourceHost, - req.SourcePath, - sizeStr, - req.Format, - ) + sourcePathAndQuery += "?" + req.SourceQuery } - return path, sig, exp + unescapedPath := fmt.Sprintf("/v1/image/%s%s/%s.%s", + req.SourceHost, sourcePathAndQuery, sizeStr, req.Format) + + return (&url.URL{Path: unescapedPath}).EscapedPath(), sig, exp } // buildSignatureData creates the string to be signed. diff --git a/internal/signature/signature_test.go b/internal/signature/signature_test.go index 1712fca..559d19c 100644 --- a/internal/signature/signature_test.go +++ b/internal/signature/signature_test.go @@ -475,6 +475,44 @@ func TestGenerateSignedURL_WithoutQueryString(t *testing.T) { } } +// TestGenerateSignedURL_ReadmeQueryExample checks the README's example of a +// source URL with a query, the image at +// https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg: its +// signature with the README's key, and the path GenerateSignedURL writes for +// it, with the "?" written as %3F and the "%" as %25. +func TestGenerateSignedURL_ReadmeQueryExample(t *testing.T) { + t.Parallel() + + signer := signature.New("example-signing-key-for-documentation") + + req := signature.Request{ + SourceHost: testHost, + SourcePath: "/img", + SourceQuery: "url=https://origin.example.com/my%20cat.jpg", + Width: 800, + Height: 600, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + Expires: time.Unix(1704067200, 0), + } + + const wantSig = "37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=" + + gotSig := signer.Sign(&req) + if gotSig != wantSig { + t.Errorf("Sign() = %q, want %q", gotSig, wantSig) + } + + const wantPath = "/v1/image/cdn.example.com" + + "/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp" + + gotPath, _, _ := signer.GenerateSignedURL(&req, time.Hour) + if gotPath != wantPath { + t.Errorf("GenerateSignedURL() path = %q, want %q", gotPath, wantPath) + } +} + func TestParseParams(t *testing.T) { t.Parallel() From 62b4f4791eebc5a8e2d9e24dcebf253795e57d9a Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 01:14:51 +0200 Subject: [PATCH 079/131] Refuse an image URL whose host is not a host name or IP address (closes #275) A ? written into the host segment as %3F reached the parser decoded, so cdn.example.com?x=1 passed the allowlist as cdn.example.com, fetched the host's root path with that query, and was cached under its own keys: another unsigned spelling of one upstream request. The host segment is now accepted only as a host name, an IPv4 address or a bracketed IPv6 address, with an optional port, and anything else is refused with 400. An IPv6 address without brackets is now refused instead of needing a signature; a host name outside ASCII is refused, its punycode accepted. Model: opus-5-5 --- README.md | 6 +- TODO.md | 11 +++ .../handlers/image_errors_internal_test.go | 76 +++++++++++++++++++ internal/imgcache/urlparser.go | 27 ++++++- 4 files changed, 115 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4fc97e9..85d08c0 100644 --- a/README.md +++ b/README.md @@ -575,7 +575,11 @@ pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so `CDN.Example.com` is matched, fetched, cached and counted against `upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature -even when the host is allowlisted. A host holding `@` is refused with 400. +even when the host is allowlisted. The host must be a host name or an IPv4 +address (letters, digits, hyphens, underscores and dots) or an IPv6 address in +brackets, either one optionally followed by `:` and a port. Any other host, such +as one holding `@`, or a `?`, `#`, `%`, `\` or space written percent-encoded, is +refused with 400. ### Configuration diff --git a/TODO.md b/TODO.md index 0ef9f6b..6b6417e 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 the host of a `/v1/image/` URL must be a host name or an IPv4 + address (letters, digits, hyphens, underscores and dots) or an IPv6 address in + brackets, either one optionally followed by `:` and a port, and any other host + is refused with 400 (closes #275): only `@` was refused, so a `?` written as + `%3F` gave the host `cdn.example.com?x=1`, which Go's URL parser reads as + `cdn.example.com` with a query. Such a URL passed the allowlist, and each text + after the `?` fetched the host's root path and cached the answer under keys of + its own. A `#`, `%`, `\` or space written percent-encoded is refused the same + way, and so is an IPv6 address without brackets, which needed a signature + before. Underscores are kept because `allowlist_hosts` accepts host names + holding them. - 2026-10-15 the README says how a source URL with a query is written into an image URL, and the image route accepts it (closes #255): the query follows the source path after `%3F`. pixa now reads the image URL's path decoded once, so diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index fbbcf6f..eac170f 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -264,3 +264,79 @@ func TestHandleImage_AllowlistOrSignature(t *testing.T) { }) } } + +// TestHandleImage_InvalidHost_Returns400 sends image URLs whose host no URL +// host can be, and checks that each is refused with 400 naming the host as +// decoded: allowlistedHost with a character after it that a URL host cannot +// hold, percent-encoded, an IPv6 address without brackets, and brackets +// around something that is not an IPv6 address. A "?" or "#" after +// allowlistedHost would otherwise end the host when the source URL is read, +// so the request would pass the allowlist and be fetched and cached under a +// host that is not the one fetched. +func TestHandleImage_InvalidHost_Returns400(t *testing.T) { + t.Parallel() + + photos := newPhotoFetcher(t, allowlistedHost) + + tests := []struct { + name string + writtenHost string + decodedHost string + }{ + {"? written as %3F", allowlistedHost + "%3Fx=1", allowlistedHost + "?x=1"}, + {"# written as %23", allowlistedHost + "%23x", allowlistedHost + "#x"}, + {"% written as %25", allowlistedHost + "%253F", allowlistedHost + "%3F"}, + {`\ written as %5C`, allowlistedHost + "%5Cx", allowlistedHost + `\x`}, + {"space written as %20", allowlistedHost + "%20", allowlistedHost + " "}, + {"IPv6 address without brackets", "2001:db8::1", "2001:db8::1"}, + {"IPv4 address in brackets", "[203.0.113.5]", "[203.0.113.5]"}, + {"no address in brackets", "[abc]", "[abc]"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := sendGet(t, newImageRoute(t, photos), photoURL(tt.writtenHost)) + checkErrorBody(t, rec, http.StatusBadRequest, + fmt.Sprintf("invalid image URL: invalid source host: %q", + tt.decodedHost)) + }) + } +} + +// TestHandleImage_HostForms_Served checks that the image route serves, with +// a valid signature, the image at photoPath on a host name, an IPv4 address, +// an IPv6 address in brackets, and a host followed by ":" with or without a +// port. +func TestHandleImage_HostForms_Served(t *testing.T) { + t.Parallel() + + hosts := []string{ + allowlistedHost, + "img_1.example.com", + "203.0.113.5", + "[2001:db8::1]", + allowlistedHost + ":8443", + "[2001:db8::1]:8443", + allowlistedHost + ":", + } + + photos := newPhotoFetcher(t, hosts...) + exp := time.Now().Add(time.Hour) + + for _, host := range hosts { + t.Run(host, func(t *testing.T) { + t.Parallel() + + sig := photoSignature(testSigningKey, host, exp) + + rec := sendGet(t, newImageRoute(t, photos), + photoURLWithSig(host, sig, exp)) + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d: %s", + rec.Code, http.StatusOK, rec.Body) + } + }) + } +} diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index 1794d1f..467c8d9 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -3,6 +3,7 @@ package imgcache import ( "errors" "fmt" + "net/netip" "net/url" "regexp" "strconv" @@ -50,6 +51,14 @@ var sizeFormatRegex = regexp.MustCompile( // percentEscapeRegex matches one percent-escape, such as "%2e" var percentEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}`) +// hostRegex matches the forms the host of an image URL may take: a host name +// or an IPv4 address (letters, digits, "-", "_" and "."), or an IPv6 address +// in brackets (hex digits, ":" and "."), either one optionally followed by ":" +// and a port, which may be empty. Its one group holds the text in brackets, +// which parseImageComponents also checks is an IPv6 address. +var hostRegex = regexp.MustCompile( + `^(?:[A-Za-z0-9_.-]+|\[([0-9A-Fa-f:.]+)\])(?::[0-9]*)?$`) + // ParsedURL contains the parsed components of an image proxy URL. type ParsedURL struct { // Host is the source origin host (e.g., "cdn.example.com") @@ -141,10 +150,20 @@ func parseImageComponents(remainder string) (*ParsedURL, error) { return nil, ErrMissingHost } - // A URL reads what comes before "@" as a user name and password, and the - // rest as the host - if strings.Contains(host, "@") { - return nil, fmt.Errorf("%w: %q holds @", ErrInvalidHost, host) + // Only the forms hostRegex describes are accepted. A "?", "#" or "@" in + // the host would make Go's URL parser, which the allowlist check and the + // fetcher use, read a different host from the one allowlisted and cached + hostMatch := hostRegex.FindStringSubmatch(host) + if hostMatch == nil { + return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host) + } + + // Go's URL parser refuses brackets around anything but an IPv6 address + if inBrackets := hostMatch[1]; inBrackets != "" { + addr, err := netip.ParseAddr(inBrackets) + if err != nil || !addr.Is6() { + return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host) + } } // Host names do not depend on letter case, so the allowlist, the upstream From 1fd5cf8bd07c377b528815185404d9d3568f219b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 01:36:56 +0200 Subject: [PATCH 080/131] Refuse an image URL whose source path or query holds a # (closes #286) A # written as %23 in the source path or query reached the parser decoded and stayed in the source URL, where Go reads it as the start of a fragment and drops it before fetching. So each text after the # was another unsigned spelling of one upstream request, cached under its own keys. An image URL whose decoded source path or query holds a # is now refused with 400, signed or not, and README.md's rule for writing a source path and query says a # cannot be part of either. Model: opus-5-5 --- README.md | 5 +- TODO.md | 5 ++ .../handlers/image_errors_internal_test.go | 48 +++++++++++++++++++ internal/imgcache/urlparser.go | 8 ++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 85d08c0..b84f4a1 100644 --- a/README.md +++ b/README.md @@ -323,8 +323,9 @@ first `/`, the size and format after the last `/`, and between them the source path up to the first `?` and the source query after it. So the `?` before the query, any `?` in the query, and every `%` in the source path or query must be percent-encoded, as `%3F` and `%25`. Any other character may be written as -itself or percent-encoded: `/`, `;`, `,`, `=` and `&` work either way. For -example, +itself or percent-encoded: `/`, `;`, `,`, `=` and `&` work either way. Neither +the source path nor the source query may hold a `#`, and an image URL that +writes one as `%23` is refused with 400. For example, `https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg` at 800x600 WebP is `/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp`, diff --git a/TODO.md b/TODO.md index 6b6417e..4b27407 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,11 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a `/v1/image/` URL whose source path or source query holds a `#`, + written as `%23`, is refused with 400 (closes #286): the source URL built from + it ended at the `#` when fetched, so each text after it fetched the same + upstream image, without a signature for an allowlisted host, and cached it + under keys of its own. `README.md` says neither may hold a `#`. - 2026-10-15 the host of a `/v1/image/` URL must be a host name or an IPv4 address (letters, digits, hyphens, underscores and dots) or an IPv6 address in brackets, either one optionally followed by `:` and a port, and any other host diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index eac170f..25fe061 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -340,3 +340,51 @@ func TestHandleImage_HostForms_Served(t *testing.T) { }) } } + +// TestHandleImage_HashInSourcePathOrQuery_Returns400 sends image URLs whose +// source path or source query holds a "#", written as %23, and checks that +// each is refused with 400, the one signed for that source path included. The +// source URL would otherwise end at the "#" when it is fetched, so each text +// after it would fetch the image at photoPath and cache it under keys of its +// own. +func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) { + t.Parallel() + + photos := newPhotoFetcher(t, allowlistedHost, signedHost) + exp := time.Now().Add(time.Hour) + + sig := signature.New(testSigningKey).Sign(&signature.Request{ + SourceHost: signedHost, + SourcePath: photoPath + "#x", + Width: 50, + Height: 50, + Format: string(imgcache.FormatJPEG), + Quality: 85, + FitMode: string(imgcache.FitCover), + Expires: exp, + }) + + tests := []struct { + name string + target string + }{ + {"# in the source path", + "/v1/image/" + allowlistedHost + photoPath + "%23x/50x50.jpeg"}, + {"# in the source query", + "/v1/image/" + allowlistedHost + photoPath + "%3Fv=1%23x/50x50.jpeg"}, + {"# in the source path, signed", + fmt.Sprintf("/v1/image/%s%s%%23x/50x50.jpeg?sig=%s&exp=%d", + signedHost, photoPath, sig, exp.Unix())}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := sendGet(t, newImageRoute(t, photos), tt.target) + checkErrorBody(t, rec, http.StatusBadRequest, + `invalid image URL: invalid image path: `+ + `the source path or query holds a "#"`) + }) + } +} diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index 467c8d9..59d95fb 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -171,6 +171,14 @@ func parseImageComponents(remainder string) (*ParsedURL, error) { // host host = strings.ToLower(host) + // Go's URL parser, which the fetcher uses, reads a "#" as the start of a + // fragment and drops the rest, so each text after it would be another + // spelling, cached under its own keys, of one upstream request + if strings.Contains(path, "#") { + return nil, fmt.Errorf( + `%w: the source path or query holds a "#"`, ErrInvalidPath) + } + // Extract query string if present in path if qIndex := strings.Index(path, "?"); qIndex != -1 { query = path[qIndex+1:] From 9a96740463dbef17693486014296f4d75637c307 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 02:23:18 +0200 Subject: [PATCH 081/131] Crop the centre before resizing for fit=cover (closes #282) For fit=cover, the default fit, libvips' thumbnail scaled the whole source to cover the requested size and copied that into memory before cutting out the centre, so an 8192x64 source at 8192x8192 asked for about 24 GiB, and 8192x1 failed with 500. resize now first cuts the centre of the source, widened to whole source pixels, and gives only that to thumbnail, which scales both sides by the same amount as before; the image it holds exceeds the output by less than one scaled source pixel on each side. Output size and scale are unchanged; its position can move by up to one output pixel. Model: opus-5-5 --- TODO.md | 12 + .../imageprocessor/fit_cover_internal_test.go | 268 ++++++++++++++++++ internal/imageprocessor/imageprocessor.go | 36 ++- 3 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 internal/imageprocessor/fit_cover_internal_test.go diff --git a/TODO.md b/TODO.md index 4b27407..cc2a818 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,18 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-15 a thin source requested with `fit=cover` no longer makes libvips + build a huge image (closes #282): libvips' thumbnail scaled the whole source + to cover the requested size and copied that into memory before it cut out the + centre, so an 8192x64 source at 8192x8192 made it allocate about 24 GiB, and + an 8192x1 source answered 500. The image processor now first cuts the centre + of the source, widened to whole source pixels, and thumbnail scales and cuts + only that, by the same amount on both sides as before. What it copies into + memory is then larger than the output by less than one scaled source pixel on + each side. With the runtime image's libvips and JPEG output at 8192x8192, + resident memory peaked at about 242 MiB for the 8192x64 source and 431 MiB for + the 8192x1 one. The output can sit a fraction of an output pixel from where it + was. - 2026-10-15 a `/v1/image/` URL whose source path or source query holds a `#`, written as `%23`, is refused with 400 (closes #286): the source URL built from it ended at the `#` when fetched, so each text after it fetched the same diff --git a/internal/imageprocessor/fit_cover_internal_test.go b/internal/imageprocessor/fit_cover_internal_test.go new file mode 100644 index 0000000..0b01211 --- /dev/null +++ b/internal/imageprocessor/fit_cover_internal_test.go @@ -0,0 +1,268 @@ +package imageprocessor + +import ( + "bytes" + "image" + "image/color" + "image/png" + "math" + "testing" +) + +// TestImageProcessor_FitCoverServesThinSource verifies that FitCover serves +// a thin source at the requested size. An 8192x1 source scaled to cover +// 8192x8192 is 8192 times as long as the output on its long side. The centre +// of an 8192x1 source with the proportions of 1x8192 is 1/8192 of a pixel +// wide. Each case is also turned on its side. +func TestImageProcessor_FitCoverServesThinSource(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputWidth, inputHeight int + size Size + }{ + {"8192x1 at 8192x8192", 8192, 1, Size{Width: 8192, Height: 8192}}, + {"8192x1 at 1x8192", 8192, 1, Size{Width: 1, Height: 8192}}, + {"1x8192 at 8192x8192", 1, 8192, Size{Width: 8192, Height: 8192}}, + {"1x8192 at 8192x1", 1, 8192, Size{Width: 8192, Height: 1}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover}) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + defer func() { _ = result.Content.Close() }() + + t.Logf("Process() made %dx%d", result.Width, result.Height) + + if result.Width != tt.size.Width || result.Height != tt.size.Height { + t.Errorf("Process() made %dx%d, want %dx%d", + result.Width, result.Height, tt.size.Width, tt.size.Height) + } + }) + } +} + +// createBandsPNG creates a PNG of three equal bands, red, green and blue, +// side by side when it is wider than tall and top to bottom otherwise. +func createBandsPNG(t *testing.T, width, height int) []byte { + t.Helper() + + bands := []color.RGBA{ + {R: 255, A: 255}, + {G: 255, A: 255}, + {B: 255, A: 255}, + } + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + band := 3 * y / height + if width > height { + band = 3 * x / width + } + + img.Set(x, y, bands[band]) + } + } + + return encodePNG(t, img) +} + +// TestImageProcessor_FitCoverCropsTheCentre verifies that FitCover keeps the +// centre of the source: a 300x100 source of red, green and blue bands side +// by side, or a 100x300 one with them top to bottom, comes out at 50x50 as +// its green band. +func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputWidth, inputHeight int + }{ + {"bands side by side", 300, 100}, + {"bands top to bottom", 100, 300}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + input := createBandsPNG(t, tt.inputWidth, tt.inputHeight) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{ + Size: Size{Width: 50, Height: 50}, + Format: FormatPNG, + FitMode: FitCover, + }) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + defer func() { _ = result.Content.Close() }() + + output, err := png.Decode(result.Content) + if err != nil { + t.Fatalf("failed to decode output: %v", err) + } + + bounds := output.Bounds() + t.Logf("Process() made %dx%d", bounds.Dx(), bounds.Dy()) + + if bounds.Dx() != 50 || bounds.Dy() != 50 { + t.Fatalf("Process() made %dx%d, want 50x50", + bounds.Dx(), bounds.Dy()) + } + + // Resampling may blend a little of the bands beside the + // green one into the output's edges. + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + r, g, b, _ := output.At(x, y).RGBA() + if r>>8 > 64 || g>>8 < 192 || b>>8 > 64 { + t.Fatalf("output pixel %d,%d is %d,%d,%d, want green", + x, y, r>>8, g>>8, b>>8) + } + } + } + }) + } +} + +// The greys of createSquarePNG, and the grey halfway between them. +const ( + backgroundGrey = 192 + squareGrey = 64 + halfwayGrey = (backgroundGrey + squareGrey) / 2 +) + +// createSquarePNG creates a grey PNG side pixels square with a darker grey +// square squareSide pixels wide in its centre. +func createSquarePNG(t *testing.T, side, squareSide int) []byte { + t.Helper() + + first := (side - squareSide) / 2 + last := first + squareSide - 1 + + img := image.NewGray(image.Rect(0, 0, side, side)) + for y := range side { + for x := range side { + grey := uint8(backgroundGrey) + if x >= first && x <= last && y >= first && y <= last { + grey = squareGrey + } + + img.SetGray(x, y, color.Gray{Y: grey}) + } + } + + return encodePNG(t, img) +} + +// squareLength returns how long the square is along greys, the greys of a +// row or column of the output that crosses it: the distance between the +// first and the last point where greys passes halfwayGrey, or 0 when there +// is no such point. Resampling blurs the square's edges over several +// pixels, and each point is placed between the two pixels on either side +// of it in proportion to their greys, so the length is measured to a +// fraction of a pixel. +func squareLength(greys []float64) float64 { + first, last := -1.0, -1.0 + + for i := 1; i < len(greys); i++ { + before, after := greys[i-1], greys[i] + if (before > halfwayGrey) == (after > halfwayGrey) { + continue + } + + point := float64(i-1) + (before-halfwayGrey)/(before-after) + if first < 0 { + first = point + } + + last = point + } + + return last - first +} + +// TestImageProcessor_FitCoverKeepsProportions verifies that FitCover scales +// both sides of the source by the same amount: a square in the centre of +// the source comes out square. Its width and height are measured along the +// output's middle row and column. +func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + side, squareSide int + size Size + }{ + {"16x16 at 100x60", 16, 8, Size{Width: 100, Height: 60}}, + {"16x16 at 60x100", 16, 8, Size{Width: 60, Height: 100}}, + {"10x10 at 100x35", 10, 2, Size{Width: 100, Height: 35}}, + {"10x10 at 35x100", 10, 2, Size{Width: 35, Height: 100}}, + {"64x64 at 300x100", 64, 16, Size{Width: 300, Height: 100}}, + {"64x64 at 100x300", 64, 16, Size{Width: 100, Height: 300}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + input := createSquarePNG(t, tt.side, tt.squareSide) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{Size: tt.size, Format: FormatPNG, FitMode: FitCover}) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + defer func() { _ = result.Content.Close() }() + + output, err := png.Decode(result.Content) + if err != nil { + t.Fatalf("failed to decode output: %v", err) + } + + bounds := output.Bounds() + if bounds.Dx() != tt.size.Width || bounds.Dy() != tt.size.Height { + t.Fatalf("Process() made %dx%d, want %dx%d", + bounds.Dx(), bounds.Dy(), tt.size.Width, tt.size.Height) + } + + row := make([]float64, bounds.Dx()) + for x := range row { + r, _, _, _ := output.At(x, bounds.Dy()/2).RGBA() + row[x] = float64(r >> 8) + } + + column := make([]float64, bounds.Dy()) + for y := range column { + r, _, _, _ := output.At(bounds.Dx()/2, y).RGBA() + column[y] = float64(r >> 8) + } + + width, height := squareLength(row), squareLength(column) + t.Logf("square is %.2f wide and %.2f high", width, height) + + if width == 0 || math.Abs(width-height) > 0.5 { + t.Errorf("square is %.2f wide and %.2f high, want it square", + width, height) + } + }) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 2a1fbf5..716b9ad 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -475,8 +475,8 @@ func (p *ImageProcessor) resize( ) error { switch fit { case FitCover, "": - // Resize and crop to fill exact dimensions (default) - return img.Thumbnail(width, height, vips.InterestingCentre) + // Crop the centre and resize to fill exact dimensions (default) + return cropCentreAndResize(img, width, height) case FitContain: // Resize to fit within dimensions, maintaining aspect ratio @@ -530,6 +530,38 @@ func (p *ImageProcessor) resize( } } +// cropCentreAndResize scales img by the same amount on both sides, the +// least that makes it cover width x height, and cuts the centre width x +// height out of it, as libvips' thumbnail does with InterestingCentre. +// thumbnail alone scales the whole image and copies the result into memory +// before it cuts, and for a thin image that is far larger than the image or +// the output: 8192x64 at 8192x8192 is 1048576x8192, about 24 GiB. So the +// centre is cut from img first, widened to whole pixels of img, which adds +// less than one pixel of img on each side, and thumbnail scales and cuts +// only that. +func cropCentreAndResize(img *vips.ImageRef, width, height int) error { + imgW, imgH := img.Width(), img.Height() + + // An image wider than the requested proportions keeps its height and + // loses width down to imgH*width/height, any other keeps its width and + // loses height down to imgW*height/width. Half of what is lost comes + // off each side, rounded down to whole pixels, so the part kept holds + // all of the centre. + left, top := 0, 0 + if imgW*height > imgH*width { + left = (imgW*height - imgH*width) / (2 * height) //nolint:mnd // each side + } else { + top = (imgH*width - imgW*height) / (2 * width) //nolint:mnd // each side + } + + err := img.ExtractArea(left, top, imgW-2*left, imgH-2*top) + if err != nil { + return err + } + + return img.Thumbnail(width, height, vips.InterestingCentre) +} + const defaultQuality = 85 // encode encodes an image to the specified format. govips gives pixa no way From 4a4bc9758fca0c4ab05f78c83cadafc4661fb9f2 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 03:31:15 +0200 Subject: [PATCH 082/131] Fetch the source path and query URL-encoded, not joined as text (closes #288) The source URL was built by joining the decoded path and query as text, so a space in the query reached the upstream raw and failed with 502, 100%25.png made a URL Go could not parse and answered 401, and %20 and %2520 fetched the same path under different cache keys. The source URL is now built with net/url, so each decoded path and query is one upstream request matching its signature and cache key; the query encodes the characters a browser encodes, ' included. The generator page compares what pixa would fetch with the URL as entered, and answers 400 with the URL to use when they differ, or for a user name and password or a .. segment. Model: opus-5-5 --- README.md | 34 +- TODO.md | 13 + internal/handlers/auth.go | 48 +++ internal/imgcache/imgcache.go | 24 +- internal/imgcache/urlparser.go | 28 +- internal/server/source_url_internal_test.go | 371 ++++++++++++++++++++ 6 files changed, 502 insertions(+), 16 deletions(-) create mode 100644 internal/server/source_url_internal_test.go diff --git a/README.md b/README.md index b84f4a1..a8a9242 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,20 @@ writes one as `%23` is refused with 400. For example, `/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp`, signed as shown under Signature Specification. +The source path is the path with its percent-escapes decoded, and pixa fetches +it percent-encoded again: every character other than letters, digits, `-`, `.`, +`_`, `~`, `/`, `$`, `&`, `+`, `,`, `:`, `;`, `=` and `@` is sent as a +percent-escape. So `my%20cat.jpg` in the image URL is the source path +`/my cat.jpg`, fetched as `/my%20cat.jpg`; `my%2520cat.jpg` is the source path +`/my%20cat.jpg`, fetched as `/my%2520cat.jpg`; and `100%25.png` is fetched as +`/100%25.png`. The source query is the source URL's query as that URL writes it, +its percent-escapes included, and is sent as it is, except that the characters a +browser percent-encodes in a query (a space, `"`, `'`, `<`, `>`, control +characters and characters outside ASCII) and a `%` that does not begin a +percent-escape are percent-encoded. pixa signs and caches the query in that +form, so `img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are +the same source query, `name=my%20cat`. + A size with one side `0` has that side scaled to keep the source's proportions, and is refused with 400 when that side would come out above 8192, the most either side may be; so is a size with `fit=outside`, which scales the image to @@ -445,7 +459,13 @@ nor change what it asks for. sent with an empty format, or none, also makes a JPEG XL URL. Width and height both empty or `0` keep the original size; if only one of them is empty or `0`, that side is scaled to keep the image's proportions. A URL whose - scaled side would come out above 8192 answers 400. + scaled side would come out above 8192 answers 400, and so does a source URL + whose path holds a `..` segment. So does a source URL that the image URL + would fetch written another way, and the page then shows the URL pixa would + fetch: one with a user name and password, which pixa does not send, or one + whose path holds `%2F`, fetched as `/`, or `(`, fetched as `%28`, as pixa + keeps the source path decoded and fetches it percent-encoded again (see + Routes). 3. The page shows the URL, `https:///v1/e//img.`, and when it expires. `` is the host the page was opened on, and the URL starts with `http` instead while `debug` is on. The name after the token is ignored @@ -507,11 +527,13 @@ Where: - `host` — source host as the URL writes it up to the first `/`, in lower case, keeping an IPv6 address's brackets and any `:` and port after it (e.g. `cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`) -- `path` — source path (e.g. `/photos/cat.jpg`), with each `%` written as `%25` - and each `:` as `%3A`, so `/photos/a:b.jpg` is signed as `/photos/a%3Ab.jpg` -- `query` — source query string, empty string if none, with `%` and `:` written - as in `path`; neither then holds a `:`, so path `/a:b` with no query and path - `/a` with query `b:` are signed differently +- `path` — source path, with its percent-escapes decoded as under Routes (e.g. + `/photos/cat.jpg`, or `/photos/my cat.jpg` for `/photos/my%20cat.jpg`), with + each `%` written as `%25` and each `:` as `%3A`, so `/photos/a:b.jpg` is + signed as `/photos/a%3Ab.jpg` +- `query` — source query string as under Routes, empty string if none, with `%` + and `:` written as in `path`; neither then holds a `:`, so path `/a:b` with no + query and path `/a` with query `b:` are signed differently - `width` — requested width in pixels, `0` for original - `height` — requested height in pixels, `0` for original - `format` — output format, one of those listed under Routes, with `original` diff --git a/TODO.md b/TODO.md index cc2a818..4c56de7 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,19 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 the source URL a request fetches is built with `net/url` from the + source host, path and query, not joined as text (closes #288). The source + path, decoded, is fetched percent-encoded again, so `100%25.png` fetches + `/100%25.png` where it answered 401, and `my%20cat.png` and `my%2520cat.png` + fetch different paths where both fetched `/my%20cat.png` under two cache keys. + The source query stays as the source URL writes it; the characters a browser + percent-encodes in a query, such as `'` or a space, which was sent raw and + answered 502, are percent-encoded when the image URL is read, so the query is + signed and cached as it is sent. The generator page answers 400, showing the + URL pixa would fetch, for a source URL its image URL would fetch written + another way, such as with `%2F` in the path, whatever else the path holds, or + with a user name and password. It also answers 400 for a source path holding a + `..` segment, as `/v1/image/` refuses one. - 2026-10-15 a thin source requested with `fit=cover` no longer makes libvips build a huge image (closes #282): libvips' thumbnail scaled the whole source to cover the requested size and copied that into memory before it cut out the diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index dff2763..d51b569 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -9,7 +9,9 @@ import ( "math" "net/http" "net/url" + "slices" "strconv" + "strings" "time" "sneak.berlin/go/pixa/internal/clientip" @@ -133,6 +135,16 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { return } + // /v1/image/ refuses a ".." segment in the source path, and so does + // the generator. The path is decoded, so %2e%2e counts too + if slices.Contains(strings.Split(parsed.Path, "/"), "..") { + s.renderGeneratorWithForm(w, r, + "Source URL path cannot hold a .. segment", r.Form, + http.StatusBadRequest) + + return + } + payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form) if err != nil { s.renderGeneratorWithForm(w, r, err.Error(), r.Form, @@ -141,6 +153,19 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { return } + // The image URL must fetch the source URL as it was written + sourceRequest := payload.ToImageRequest() + sourceRequest.AllowHTTP = s.config.AllowHTTP + fetchURL := sourceRequest.SourceURL() + + if !fetchedAsWritten(parsed, fetchURL) { + s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+ + "written: pixa would fetch "+fetchURL, + r.Form, http.StatusBadRequest) + + return + } + // Generate encrypted token token, err := s.encGen.Generate(payload) if err != nil { @@ -173,6 +198,29 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { } } +// fetchedAsWritten reports whether fetchURL, the URL pixa fetches for an +// image URL made from the source URL source, is source as it was written, +// apart from the scheme, which the token does not keep. It is not when source +// holds a user name and password, which the token does not keep either, or a +// "?" with no query after it. The token keeps the source path decoded and +// pixa fetches it percent-encoded again, so a path written another way, such +// as with %2F, is fetched as another path. url.URL keeps the path as written +// in RawPath when it is not the decoded path percent-encoded again. +// EscapedPath ignores RawPath when it also holds a character net/url encodes, +// such as a space, so RawPath is compared itself. +func fetchedAsWritten(source *url.URL, fetchURL string) bool { + writtenPath := source.RawPath + if writtenPath == "" { + writtenPath = source.EscapedPath() + } + + fetched, err := url.Parse(fetchURL) + + return err == nil && source.User == nil && !source.ForceQuery && + fetched.EscapedPath() == writtenPath && + fetched.RawQuery == source.RawQuery +} + // buildGeneratePayload parses the numeric form fields and assembles the // encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A // non-numeric or out-of-range width, height, quality or ttl, or an diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index 47acca6..51b33b3 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "net/url" "time" ) @@ -86,9 +87,11 @@ func ValidateImageRequest(req *ImageRequest) error { type ImageRequest struct { // SourceHost is the origin host (e.g., "cdn.example.com") SourceHost string - // SourcePath is the path on the origin (e.g., "/photos/cat.jpg") + // SourcePath is the path on the origin with its percent-escapes decoded, + // as url.URL's Path holds it (e.g., "/photos/my cat.jpg") SourcePath string - // SourceQuery is the optional query string for the origin URL + // SourceQuery is the optional query string for the origin URL as the URL + // writes it, percent-escapes included, as url.URL's RawQuery holds it SourceQuery string // Size is the requested output dimensions Size Size @@ -107,20 +110,25 @@ type ImageRequest struct { AllowHTTP bool } -// SourceURL returns the full upstream URL to fetch. -// Uses http:// scheme when AllowHTTP is true, otherwise https://. +// SourceURL returns the full upstream URL to fetch: the source path +// percent-encoded again, as net/url writes a path, so each source path is one +// upstream path, and the source query as it is, with the characters +// escapeQuery encodes percent-encoded. Uses http:// scheme when AllowHTTP is +// true, otherwise https://. func (r *ImageRequest) SourceURL() string { scheme := "https" if r.AllowHTTP { scheme = "http" } - url := scheme + "://" + r.SourceHost + r.SourcePath - if r.SourceQuery != "" { - url += "?" + r.SourceQuery + sourceURL := url.URL{ + Scheme: scheme, + Host: r.SourceHost, + Path: r.SourcePath, + RawQuery: escapeQuery(r.SourceQuery), } - return url + return sourceURL.String() } // ImageResponse represents a processed image ready to serve diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index 59d95fb..6c58c20 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -51,6 +51,13 @@ var sizeFormatRegex = regexp.MustCompile( // percentEscapeRegex matches one percent-escape, such as "%2e" var percentEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}`) +// queryEscapeRegex matches a percent-escape, or one character that a browser +// percent-encodes in the query of an http or https URL: one outside the +// printable ASCII characters "!" to "~", such as a space, a control character +// or a character outside ASCII, or one of `"#'<>`. A "%" that does not begin a +// percent-escape matches too. +var queryEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}|[^!-~]|["#'<>%]`) + // hostRegex matches the forms the host of an image URL may take: a host name // or an IPv4 address (letters, digits, "-", "_" and "."), or an IPv6 address // in brackets (hex digits, ":" and "."), either one optionally followed by ":" @@ -179,9 +186,12 @@ func parseImageComponents(remainder string) (*ParsedURL, error) { `%w: the source path or query holds a "#"`, ErrInvalidPath) } - // Extract query string if present in path + // Extract query string if present in path. It is the source URL's query + // as that URL writes it. A character a URL's query cannot hold as it is, + // such as a space written %20 in the image URL, is percent-encoded here, + // so the query is signed and cached as it is sent upstream if qIndex := strings.Index(path, "?"); qIndex != -1 { - query = path[qIndex+1:] + query = escapeQuery(path[qIndex+1:]) path = path[:qIndex] } @@ -270,6 +280,20 @@ func decodePercentEscapes(s string) string { }) } +// escapeQuery returns a source query as a URL's query holds it: each +// character that queryEscapeRegex names, such as a space, percent-encoded, +// and the rest, the query's percent-escapes included, kept as they are. The +// result is unchanged by a second escapeQuery. +func escapeQuery(query string) string { + return queryEscapeRegex.ReplaceAllStringFunc(query, func(match string) string { + if percentEscapeRegex.MatchString(match) { + return match + } + + return url.PathEscape(match) + }) +} + // parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600" func parseSizeFormat(s string) (Size, ImageFormat, error) { matches := sizeFormatRegex.FindStringSubmatch(s) diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go new file mode 100644 index 0000000..beaf1d5 --- /dev/null +++ b/internal/server/source_url_internal_test.go @@ -0,0 +1,371 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "slices" + "strings" + "sync" + "testing" + + "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/session" +) + +// percentSignPath is a source path holding a "%", which a URL writes %25. +const percentSignPath = "/100%25.png" + +// generateFormPattern extracts the CSRF token rendered into the generator +// page's form. +var generateFormPattern = regexp.MustCompile( + `(?s)action="/generate".*?name="gorilla\.csrf\.Token" value="([^"]+)"`) + +// generatedURLPattern extracts the path of the encrypted URL the generator +// page shows. +var generatedURLPattern = regexp.MustCompile(`value="[^"]*(/v1/e/[^"]+)"`) + +// recordingUpstream is a test upstream that answers every request with a PNG +// and records the request target, the path and query as sent, of each request +// it receives. +type recordingUpstream struct { + *httptest.Server + + mu sync.Mutex + received []string +} + +// startRecordingUpstream starts a recordingUpstream, which is closed when the +// test ends. +func startRecordingUpstream(t *testing.T) *recordingUpstream { + t.Helper() + + source := encodeTestPNG(t, 64, 48) + upstream := &recordingUpstream{} + + upstream.Server = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + upstream.mu.Lock() + upstream.received = append(upstream.received, r.RequestURI) + upstream.mu.Unlock() + + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(source) + })) + t.Cleanup(upstream.Close) + + return upstream +} + +// requestTargets returns the request targets the upstream has received, in +// the order it received them. +func (u *recordingUpstream) requestTargets() []string { + u.mu.Lock() + defer u.mu.Unlock() + + return slices.Clone(u.received) +} + +// pageOrigin is the origin of the login and generator pages below, opened +// over HTTPS; httptest gives every request the host example.com. +const pageOrigin = "https://example.com" + +// openRoot sends GET / to s over HTTPS, with cookie when it is not nil, and +// returns the page: the login page, or the generator page for a login session. +func openRoot( + t *testing.T, s *Server, cookie *http.Cookie, +) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, + pageOrigin+"/", nil) + if cookie != nil { + req.AddCookie(cookie) + } + + page := httptest.NewRecorder() + s.ServeHTTP(page, req) + + return page +} + +// submitForm posts form to path on s as a browser submits it from page over +// HTTPS: with the CSRF token tokenPattern finds in page, the cookies page set, +// cookie when it is not nil, and a Referer naming the page, which pixa needs +// while debug is off, as it is in startImageProxy's config. +func submitForm( + t *testing.T, s *Server, page *httptest.ResponseRecorder, + tokenPattern *regexp.Regexp, path string, form url.Values, + cookie *http.Cookie, +) *httptest.ResponseRecorder { + t.Helper() + + match := tokenPattern.FindStringSubmatch(page.Body.String()) + if match == nil { + t.Fatalf("the page has no form for %s with a CSRF token", path) + } + + form.Set("gorilla.csrf.Token", match[1]) + + req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, + pageOrigin+path, strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Referer", pageOrigin+"/") + + for _, c := range page.Result().Cookies() { + req.AddCookie(c) + } + + if cookie != nil { + req.AddCookie(cookie) + } + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + + return rec +} + +// postGenerate logs in to s with the signing key, submits the generator page's +// form with sourceURL, and returns the response. +func postGenerate( + t *testing.T, s *Server, sourceURL string, +) *httptest.ResponseRecorder { + t.Helper() + + login := submitForm(t, s, openRoot(t, s, nil), csrfFieldPattern, "/", + url.Values{"key": {testSigningKey}}, nil) + + var sessionCookie *http.Cookie + + for _, c := range login.Result().Cookies() { + if c.Name == session.CookieName { + sessionCookie = c + } + } + + if sessionCookie == nil { + t.Fatalf("logging in answered %d and set no session cookie", login.Code) + } + + rec := submitForm(t, s, openRoot(t, s, sessionCookie), generateFormPattern, + "/generate", url.Values{"url": {sourceURL}}, sessionCookie) + t.Logf("POST /generate with url %s: %d", sourceURL, rec.Code) + + return rec +} + +// TestImageURLFetchesItsSourcePathAndQuery requests image URLs whose source +// query holds a space, written percent-encoded, or a "'", and whose source +// path holds a "%", written percent-encoded, and checks the one request the +// upstream receives: the source path percent-encoded again, and the source +// query with the space written %20 and the "'" written %27, as a browser +// writes them in a query. +func TestImageURLFetchesItsSourcePathAndQuery(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + // pathAndQuery is the source path and query as the image URL writes + // them + pathAndQuery string + // wantRequest is the path and query the upstream must receive + wantRequest string + }{ + {"space in the query", "/photo.png%3Fname=my%20cat", + "/photo.png?name=my%20cat"}, + {"apostrophe in the query", "/photo.png%3Fname=it's", + "/photo.png?name=it%27s"}, + {"percent sign in the path", percentSignPath, percentSignPath}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + getImage(t, s, "/v1/image/"+upstreamHost+tt.pathAndQuery+"/orig.png") + + got := upstream.requestTargets() + t.Logf("upstream received %q", got) + + if !slices.Equal(got, []string{tt.wantRequest}) { + t.Errorf("upstream received %q, want [%q]", got, tt.wantRequest) + } + }) + } +} + +// TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart requests twice each +// of the source paths /my cat.png and /my%20cat.png, which the image URL +// writes my%20cat.png and my%2520cat.png. Each is fetched once, as a request +// of its own, and its second request is served from the cache. +func TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + imageURLs := []string{ + "/v1/image/" + upstreamHost + "/my%20cat.png/orig.png", + "/v1/image/" + upstreamHost + "/my%2520cat.png/orig.png", + } + + for _, imageURL := range imageURLs { + getImage(t, s, imageURL) + } + + for _, imageURL := range imageURLs { + cacheStatus := getImage(t, s, imageURL).Header().Get("X-Pixa-Cache") + if cacheStatus != string(imgcache.CacheHit) { + t.Errorf("second GET %s: X-Pixa-Cache = %q, want HIT", + imageURL, cacheStatus) + } + } + + got := upstream.requestTargets() + t.Logf("upstream received %q", got) + + want := []string{"/my%20cat.png", "/my%2520cat.png"} + if !slices.Equal(got, want) { + t.Errorf("upstream received %q, want %q", got, want) + } +} + +// TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest requests the source query +// name=my%20cat written as the README says, name=my%2520cat, and with the +// space written %20, name=my%20cat. Both are one source query: the upstream +// receives one request, and the second image URL is served from the cache. +func TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + getImage(t, s, "/v1/image/"+upstreamHost+"/photo.png%3Fname=my%2520cat/orig.png") + + second := getImage(t, s, + "/v1/image/"+upstreamHost+"/photo.png%3Fname=my%20cat/orig.png") + if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) { + t.Errorf("second X-Pixa-Cache = %q, want HIT", got) + } + + got := upstream.requestTargets() + t.Logf("upstream received %q", got) + + want := []string{"/photo.png?name=my%20cat"} + if !slices.Equal(got, want) { + t.Errorf("upstream received %q, want %q", got, want) + } +} + +// TestGeneratedURLFetchesTheSourceURLAsEntered makes an encrypted URL on the +// generator page for source URLs holding %20, %25 and %23 in the path or the +// query, requests it, and checks that the upstream receives the path and +// query exactly as entered. +func TestGeneratedURLFetchesTheSourceURLAsEntered(t *testing.T) { + t.Parallel() + + // The names hold no "%": each test's state directory is named after it + // and goes into the database URL, which decodes percent-escapes + tests := []struct { + name string + pathAndQuery string + }{ + {"space in the path", "/my%20cat.png"}, + {"percent sign in the path", percentSignPath}, + {"number sign in the path", "/a%23b.png"}, + {"all three in the query", "/photo.png?name=my%20cat%25%23"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + rec := postGenerate(t, s, "https://"+upstreamHost+tt.pathAndQuery) + + match := generatedURLPattern.FindStringSubmatch(rec.Body.String()) + if rec.Code != http.StatusOK || match == nil { + t.Fatalf("POST /generate: %d with no encrypted URL; body %s", + rec.Code, rec.Body.String()) + } + + getImage(t, s, match[1]) + + got := upstream.requestTargets() + t.Logf("upstream received %q", got) + + if !slices.Equal(got, []string{tt.pathAndQuery}) { + t.Errorf("upstream received %q, want [%q]", got, tt.pathAndQuery) + } + }) + } +} + +// TestGeneratorRefusesSourceURLItCannotFetchAsWritten submits source URLs the +// generator page must make no encrypted URL for, and checks that it answers +// 400 and shows why. pixa keeps the source path decoded and fetches it +// percent-encoded again, so a path holding %2F would be fetched with a "/" +// there, whatever else the path holds. pixa sends no user name or password, +// and no "?" with no query after it. For these the page shows the URL pixa +// would fetch. A source path holding a ".." segment is refused, as +// /v1/image/ refuses it. +func TestGeneratorRefusesSourceURLItCannotFetchAsWritten(t *testing.T) { + t.Parallel() + + // The test config sets allow_http, so pixa would fetch over plain HTTP + const fetchedFrom = "http://" + upstreamHost + + tests := []struct { + name string + // sourceURL is the source URL after "https://" + sourceURL string + // wantShown is what the page must show: the URL pixa would fetch, or + // why it makes no encrypted URL + wantShown string + }{ + {"encoded slash", upstreamHost + "/a%2Fb.png", + fetchedFrom + "/a/b.png"}, + {"encoded slash and a space", upstreamHost + "/a%2Fb c.png", + fetchedFrom + "/a/b%20c.png"}, + {"encoded slash and a bar", upstreamHost + "/a%2Fb|c.png", + fetchedFrom + "/a/b%7Cc.png"}, + {"encoded slash and a letter outside ASCII", upstreamHost + "/café%2Fx.png", + fetchedFrom + "/caf%C3%A9/x.png"}, + {"user name and password", "user:pass@" + upstreamHost + "/a.png", + fetchedFrom + "/a.png"}, + {"question mark with no query", upstreamHost + "/a.png?", + fetchedFrom + "/a.png"}, + {"two-dot segment", upstreamHost + "/a/../b.png", + "cannot hold a .. segment"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + s, _, _ := startImageProxy(t, startRecordingUpstream(t).Server) + + rec := postGenerate(t, s, "https://"+tt.sourceURL) + body := rec.Body.String() + + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest) + } + + if strings.Contains(body, "/v1/e/") { + t.Error("the page shows an encrypted URL") + } + + if !strings.Contains(body, tt.wantShown) { + t.Errorf("the page does not show %s; body %s", tt.wantShown, body) + } + }) + } +} From 6d271d4617a71d7913cd115e550741cabb6c7991 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 04:04:19 +0200 Subject: [PATCH 083/131] Make one image URL per source on the generator page for host case and an empty path (closes #293) The generator page passed the source host and path into the encrypted URL as entered, so a host spelled in another letter case, or a source with no path against the same source with /, made separate cache keys and fetched one upstream image twice. The page now lower-cases the source host, as /v1/image/ does, and writes an empty source path as /, before it builds the encrypted URL. The checks the page gained for https://git.eeqj.de/sneak/pixa/issues/288 still hold. Model: opus-5-5 --- README.md | 8 ++- TODO.md | 5 ++ internal/handlers/auth.go | 21 ++++++- internal/server/source_url_internal_test.go | 64 +++++++++++++++++++++ 4 files changed, 94 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a8a9242..3ab10a7 100644 --- a/README.md +++ b/README.md @@ -473,9 +473,11 @@ nor change what it asks for. `jxl` for a form with no format. The token holds the source's host, path and query and the size, format, quality, -fit and expiry, encrypted with a key derived from `signing_key`. The source -URL's scheme is not kept: the image is fetched like any other (see Routes), and -the blocked networks still apply. +fit and expiry, encrypted with a key derived from `signing_key`. The host is +kept in lower case and an empty path as `/`, as for `/v1/image/`, so +`https://CDN.Example.com` and `https://cdn.example.com/` are one source image, +fetched and cached once. The source URL's scheme is not kept: the image is +fetched like any other (see Routes), and the blocked networks still apply. How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never. The expiry is fixed in the token when the URL is made and cannot be changed or diff --git a/TODO.md b/TODO.md index 4c56de7..3817c02 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,11 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 the generator page lower-cases the source host and writes an empty + source path as `/` before it makes the encrypted URL, as `/v1/image/` does + (closes #293). A host written in upper and in lower case, or a source URL with + no path and with the path `/`, made two cache keys and two upstream requests + for one image. - 2026-10-16 the source URL a request fetches is built with `net/url` from the source host, path and query, not joined as text (closes #288). The source path, decoded, is fetched percent-encoded again, so `100%25.png` fetches diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index d51b569..27105f2 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -127,7 +127,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { sourceURL := r.FormValue("url") // Validate source URL - parsed, err := url.Parse(sourceURL) + parsed, err := parseSourceURL(sourceURL) if err != nil || parsed.Host == "" { s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form, http.StatusBadRequest) @@ -198,6 +198,25 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { } } +// parseSourceURL parses the source URL entered on the generator page with its +// host lower-cased and an empty path written as "/", as /v1/image/ reads a +// source URL, so that each spelling of one source URL makes one cache key and +// one upstream request. +func parseSourceURL(sourceURL string) (*url.URL, error) { + parsed, err := url.Parse(sourceURL) + if err != nil { + return nil, err + } + + parsed.Host = strings.ToLower(parsed.Host) + + if parsed.Path == "" { + parsed.Path = "/" + } + + return parsed, nil +} + // fetchedAsWritten reports whether fetchURL, the URL pixa fetches for an // image URL made from the source URL source, is source as it was written, // apart from the scheme, which the token does not keep. It is not when source diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index beaf1d5..d006c44 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -308,6 +308,70 @@ func TestGeneratedURLFetchesTheSourceURLAsEntered(t *testing.T) { } } +// TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL enters two +// spellings of one source URL on the generator page, requests the encrypted +// URL made for each, and checks that the upstream receives one request and +// that the second encrypted URL is served from the cache. One pair writes the +// host in upper and in lower case, which /v1/image/ lower-cases. The host is +// an IPv6 address, whose hex digits have letter case, because the fetcher +// would look a host name up in DNS. It has a port, as the fetcher refuses an +// IPv6 address written without one. The other pair is a source URL with no +// path and with the path "/", both of which fetch "/". +func TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + // sourceURLs are the two spellings, after "https://" + sourceURLs []string + // wantRequest is the path the upstream must receive + wantRequest string + }{ + {"host in upper and in lower case", + []string{"[2001:DB8::A]:8443/a.png", "[2001:db8::a]:8443/a.png"}, + "/a.png"}, + {"no path and the path written as a slash", + []string{upstreamHost, upstreamHost + "/"}, "/"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + var imageURLs []string + + for _, sourceURL := range tt.sourceURLs { + rec := postGenerate(t, s, "https://"+sourceURL) + + match := generatedURLPattern.FindStringSubmatch(rec.Body.String()) + if rec.Code != http.StatusOK || match == nil { + t.Fatalf("POST /generate: %d with no encrypted URL; body %s", + rec.Code, rec.Body.String()) + } + + imageURLs = append(imageURLs, match[1]) + } + + getImage(t, s, imageURLs[0]) + + second := getImage(t, s, imageURLs[1]) + if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) { + t.Errorf("second X-Pixa-Cache = %q, want HIT", got) + } + + got := upstream.requestTargets() + t.Logf("upstream received %q", got) + + if !slices.Equal(got, []string{tt.wantRequest}) { + t.Errorf("upstream received %q, want [%q]", got, tt.wantRequest) + } + }) + } +} + // TestGeneratorRefusesSourceURLItCannotFetchAsWritten submits source URLs the // generator page must make no encrypted URL for, and checks that it answers // 400 and shows why. pixa keeps the source path decoded and fetches it From 8b4dcda549ad4945d8d0e506d49816d05527eb76 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 04:17:59 +0200 Subject: [PATCH 084/131] Keep a side computed for a size with one side 0 at 1 or more (closes #283) A size with one side 0 has that side worked out from the source's proportions by integer division, and nothing kept it at 1 or more. When it rounded down to 0, libvips was asked for a square: an 8192x1 source at 100x0 came out 100x100, at 1300x0 it asked for one block of about 39 GB, and some shapes answered 500. Each computed side, and each side contain, inside and outside work out again from the scale, is now at least 1. README.md now says what fit=outside does and what a 0 side means; with fit=outside an 8192x1 source at 100x0 comes out 8192x1, as covering the size requires. Model: opus-5-5 --- README.md | 14 +++- TODO.md | 14 ++++ .../computed_side_internal_test.go | 66 +++++++++++++++++++ .../fit_outside_internal_test.go | 42 ++++++++++++ internal/imageprocessor/imageprocessor.go | 32 +++++---- internal/imgcache/urlparser.go | 4 +- 6 files changed, 156 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 3ab10a7..d9b3c86 100644 --- a/README.md +++ b/README.md @@ -536,8 +536,13 @@ Where: - `query` — source query string as under Routes, empty string if none, with `%` and `:` written as in `path`; neither then holds a `:`, so path `/a:b` with no query and path `/a` with query `b:` are signed differently -- `width` — requested width in pixels, `0` for original -- `height` — requested height in pixels, `0` for original +- `width` — requested width in pixels, or `0`: the width is then worked out from + the source's proportions and the requested height, rounded down but at least + 1, or is the source's own width when the height is `0` too (the `orig` size) +- `height` — requested height in pixels, or `0`: the height is then worked out + from the source's proportions and the requested width, rounded down but at + least 1, or is the source's own height when the width is `0` too (the `orig` + size) - `format` — output format, one of those listed under Routes, with `original` signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no format has the signature of the same URL ending in `.jxl`; `auto` is signed as @@ -550,7 +555,10 @@ Where: with 400 - `fit` — the URL's `fit` query parameter (cover, contain, fill, inside, outside), or `cover` when the URL has no `fit`; a request whose `fit` is - anything else, an empty `fit=` included, is refused with 400 + anything else, an empty `fit=` included, is refused with 400. `outside` scales + the image to cover the requested size, keeping its proportions, so it can come + out larger than that size on one side: an 8192x1 source at `100x0`, which is + 100x1 by its proportions, comes out at 8192x1 The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa diff --git a/TODO.md b/TODO.md index 3817c02..a00b1ef 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,20 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 a size with one side `0` keeps the side computed from the source's + proportions at 1 or more (closes #283): it was rounded down, so for a thin + source it came out at 0, and libvips then made the size square. An 8192x1 + source at `100x0` came out at 100x100, and at `1300x0` with `fit=cover` made + libvips ask for about 39 GB at once; an 8192x2 source at `100x0` answered 500. + `fit=contain` and `fit=inside` work both sides out again from the smaller of + the two scales and `fit=outside` from the larger, and each keeps both sides at + 1 or more too, so a 1x8192 source at `0x100` with `fit=contain`, and a 49x8192 + source at `0x100` or a 49x49 source at `1x1` with `fit=outside`, no longer + answer 500. An 8192x1 source at `1300x0` now comes out at 1300x1, and the + process running it alone peaked at about 34 MiB resident. With `fit=outside` + the image is scaled to cover the size, so that source at `100x0` comes out at + 8192x1. `README.md` now says this under Signature Specification, and says how + a `0` side is worked out where it said `0` meant the original. - 2026-10-16 the generator page lower-cases the source host and writes an empty source path as `/` before it makes the encrypted URL, as `/v1/image/` does (closes #293). A host written in upper and in lower case, or a source URL with diff --git a/internal/imageprocessor/computed_side_internal_test.go b/internal/imageprocessor/computed_side_internal_test.go index f5c3d5a..231087d 100644 --- a/internal/imageprocessor/computed_side_internal_test.go +++ b/internal/imageprocessor/computed_side_internal_test.go @@ -50,3 +50,69 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) { }) } } + +// TestImageProcessor_ComputedSideIsAtLeastOne verifies that, for a size with +// one side 0, the side computed from the input's proportions is 1 where it +// would round down to 0, and that Process makes that size in every fit mode +// but FitOutside, which scales the input to cover the size and so can make +// it longer. Each case checks the size targetDimensions gives before it +// calls Process: with that side at 0, an 8192x1 input at 1300x0 makes +// libvips ask for about 39 GB with FitCover. +func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputWidth, inputHeight int + size Size + wantWidth, wantHeight int + }{ + {"8192x1 at 100x0", 8192, 1, Size{Width: 100}, 100, 1}, + {"8192x1 at 1300x0", 8192, 1, Size{Width: 1300}, 1300, 1}, + {"8192x2 at 100x0", 8192, 2, Size{Width: 100}, 100, 1}, + {"1x8192 at 0x100", 1, 8192, Size{Height: 100}, 1, 100}, + {"1x8192 at 0x1300", 1, 8192, Size{Height: 1300}, 1, 1300}, + {"2x8192 at 0x100", 2, 8192, Size{Height: 100}, 1, 100}, + } + + fits := []FitMode{FitCover, FitContain, FitFill, FitInside, FitOutside} + + for _, tt := range tests { + for _, fit := range fits { + t.Run(tt.name+" "+string(fit), func(t *testing.T) { + t.Parallel() + + width, height, err := targetDimensions(tt.size, + tt.inputWidth, tt.inputHeight) + if err != nil || width != tt.wantWidth || height != tt.wantHeight { + t.Fatalf("targetDimensions() = %dx%d, %v, want %dx%d", + width, height, err, tt.wantWidth, tt.wantHeight) + } + + input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{Size: tt.size, Format: FormatJPEG, FitMode: fit}) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + defer func() { _ = result.Content.Close() }() + + t.Logf("Process() made %dx%d", result.Width, result.Height) + + ok := result.Width == tt.wantWidth && result.Height == tt.wantHeight + if fit == FitOutside { + ok = result.Width >= tt.wantWidth && result.Height >= tt.wantHeight + } + + if !ok { + t.Errorf("Process() made %dx%d, want %dx%d (at least, for %s)", + result.Width, result.Height, tt.wantWidth, tt.wantHeight, + FitOutside) + } + }) + } + } +} diff --git a/internal/imageprocessor/fit_outside_internal_test.go b/internal/imageprocessor/fit_outside_internal_test.go index ab3737f..62d9674 100644 --- a/internal/imageprocessor/fit_outside_internal_test.go +++ b/internal/imageprocessor/fit_outside_internal_test.go @@ -60,3 +60,45 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) { }) } } + +// TestImageProcessor_FitOutsideSideIsAtLeastOne verifies that FitOutside +// keeps each side at 1 or more when it works both sides out again from the +// larger scale. A side of 49 scaled to 1 is 49 times 1/49, which in floating +// point is just under 1 and so rounded down to 0, a width libvips refuses. +func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputWidth, inputHeight int + size Size + wantWidth, wantHeight int + }{ + {"49x8192 at 0x100", 49, 8192, Size{Height: 100}, 1, 167}, + {"49x49 at 1x1", 49, 49, Size{Width: 1, Height: 1}, 1, 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) + + result, err := New(Params{}).Process(t.Context(), + bytes.NewReader(input), + &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside}) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + + defer func() { _ = result.Content.Close() }() + + t.Logf("Process() made %dx%d", result.Width, result.Height) + + if result.Width != tt.wantWidth || result.Height != tt.wantHeight { + t.Errorf("Process() made %dx%d, want %dx%d", + result.Width, result.Height, tt.wantWidth, tt.wantHeight) + } + }) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 716b9ad..7b83a41 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -311,8 +311,10 @@ func (p *ImageProcessor) Process( // targetDimensions calculates the output dimensions for a requested size, // scaling proportionally when only one dimension is given and keeping the -// original dimensions when both are zero. A side scaled proportionally can -// come out above MaxOutputDimension, which is ErrOutputTooLarge. +// original dimensions when both are zero. A side scaled proportionally is +// rounded down but kept at 1 or more, as for a thin image it would round +// down to 0, a side of no pixels. It can come out above MaxOutputDimension, +// which is ErrOutputTooLarge. func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) { width, height := size.Width, size.Height @@ -322,10 +324,10 @@ func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) { return origWidth, origHeight, nil case width == 0: // Only height specified: calculate width proportionally - width = origWidth * height / origHeight + width = max(1, origWidth*height/origHeight) case height == 0: // Only width specified: calculate height proportionally - height = origHeight * width / origWidth + height = max(1, origHeight*width/origWidth) } if width > MaxOutputDimension || height > MaxOutputDimension { @@ -479,13 +481,16 @@ func (p *ImageProcessor) resize( return cropCentreAndResize(img, width, height) case FitContain: - // Resize to fit within dimensions, maintaining aspect ratio + // Resize to fit within dimensions, maintaining aspect ratio. Scaled + // by the smaller scale, the short side of a thin image can round + // down to 0, which libvips refuses as a width, so each side is at + // least 1. imgW, imgH := img.Width(), img.Height() scaleW := float64(width) / float64(imgW) scaleH := float64(height) / float64(imgH) scale := min(scaleW, scaleH) - newW := int(float64(imgW) * scale) - newH := int(float64(imgH) * scale) + newW := max(1, int(float64(imgW)*scale)) + newH := max(1, int(float64(imgH)*scale)) return img.Thumbnail(newW, newH, vips.InterestingNone) @@ -503,19 +508,22 @@ func (p *ImageProcessor) resize( scaleW := float64(width) / float64(imgW) scaleH := float64(height) / float64(imgH) scale := min(scaleW, scaleH) - newW := int(float64(imgW) * scale) - newH := int(float64(imgH) * scale) + newW := max(1, int(float64(imgW)*scale)) + newH := max(1, int(float64(imgH)*scale)) return img.Thumbnail(newW, newH, vips.InterestingNone) case FitOutside: - // Resize so smallest dimension fits, may exceed target on other dimension + // Resize so smallest dimension fits, may exceed target on other + // dimension. A side scaled to 1 can come out just under 1 in + // floating point, as 49 times 1/49 does, and round down to 0, which + // libvips refuses as a width, so each side is at least 1. imgW, imgH := img.Width(), img.Height() scaleW := float64(width) / float64(imgW) scaleH := float64(height) / float64(imgH) scale := max(scaleW, scaleH) - newW := int(float64(imgW) * scale) - newH := int(float64(imgH) * scale) + newW := max(1, int(float64(imgW)*scale)) + newH := max(1, int(float64(imgH)*scale)) // The dimension that exceeds the target is held to the maximum too if newW > MaxOutputDimension || newH > MaxOutputDimension { diff --git a/internal/imgcache/urlparser.go b/internal/imgcache/urlparser.go index 6c58c20..46bba16 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imgcache/urlparser.go @@ -29,7 +29,9 @@ var ( const MaxDimension = imageprocessor.MaxOutputDimension // ValidateDimension checks one requested width or height; name ("width" or -// "height") appears in the error. 0 means "original size" and is valid. +// "height") appears in the error. 0 is valid: that side is then worked out from +// the source's proportions and the other requested side, rounded down but at +// least 1, or is the source's own side when the other side is 0 too. func ValidateDimension(name string, value int) error { if value < 0 { return fmt.Errorf("%w: %s is negative", ErrInvalidSize, name) From efa431ba187fce1e5aa41de17363f7d583387494 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 04:37:54 +0200 Subject: [PATCH 085/131] Escape state_dir in the db_url derived from it (closes #290) When db_url is omitted, pixa derives it from state_dir, and it put the directory name into a file: URL as it was, so a state_dir holding %, ? or # opened another file and lost the journal mode. The derived URL is now built with net/url, which percent-encodes the directory name; an ordinary path gives the same URL as before. When the path written into the URL starts with //, as for a state_dir of / or one starting with //, the empty host is kept so SQLite does not read the first name in the path as a host. Model: opus-5-5 --- README.md | 13 ++- TODO.md | 8 ++ internal/config/config.go | 27 ++++- .../config/config_validation_internal_test.go | 98 +++++++++++++++++++ 4 files changed, 139 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d9b3c86..7a71a5d 100644 --- a/README.md +++ b/README.md @@ -725,11 +725,14 @@ Key settings in more detail: - `signing_key` — HMAC secret for URL signatures - `db_url` — the SQLite database to open; omitted, it is `file:/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the - database in WAL mode. pixa opens one connection to it, so its own reads and - writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to any - `db_url`, so a write that finds another program writing to the file waits up - to five seconds for it instead of failing. WAL mode comes only from the URL: - keep `_pragma=journal_mode(WAL)` in one you set + database in WAL mode. `state_dir` is percent-encoded in it, so a `%`, `?` or + `#` in the name is written as `%25`, `%3F` or `%23`. For a `state_dir` of `/` + or one starting with `//` it starts `file:////`, so SQLite does not read the + first name in the path as a host. pixa opens one connection to it, so its own + reads and writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to + any `db_url`, so a write that finds another program writing to the file waits + up to five seconds for it instead of failing. WAL mode comes only from the + URL: keep `_pragma=journal_mode(WAL)` in one you set - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk cache entirely; omitted defaults to 75% of the sum of the free space on the filesystem containing `/cache/` and the bytes of source and diff --git a/TODO.md b/TODO.md index a00b1ef..da1345b 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 a `state_dir` whose name holds `%`, `?` or `#` keeps its database + at `state.sqlite3` in that directory (closes #290): the `db_url` derived from + it put the name into a `file:` URL as it was, so SQLite read `%41` as `A`, a + `?` or `#` ended the path early, and the journal mode parameter was lost. The + derived `db_url` is now built with Go's `net/url`, which writes those + characters as `%25`, `%3F` and `%23`. For a `state_dir` of `/` or one starting + with `//` it starts `file:////`, so SQLite no longer reads the first name in + the path as a host. - 2026-10-16 a size with one side `0` keeps the side computed from the source's proportions at 1 or more (closes #283): it was rounded down, so for a thin source it came out at 0, and libvips then made the size square. An 8192x1 diff --git a/internal/config/config.go b/internal/config/config.go index ff4ac6c..40eb799 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -338,8 +338,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { settingName(keyDBURL), errValueEmpty) } - // The driver sets the journal mode only through a _pragma parameter. - c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_pragma=journal_mode(WAL)", c.StateDir) + c.DBURL = defaultDBURL(c.StateDir) } if loader.err != nil { @@ -354,6 +353,30 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { return c, nil } +// defaultDBURL returns the db_url used when db_url is omitted: the file +// state.sqlite3 in stateDir, in WAL mode, which the driver sets only +// through a _pragma parameter. String() percent-encodes any %, ? or # in +// the directory name, which SQLite decodes back, so the URL names exactly +// this file. OmitHost writes file:/var/lib/pixa/... instead of +// file:///var/lib/pixa/..., and keeps a relative stateDir from being read +// as a host name. The path written into the URL is stateDir followed by +// /state.sqlite3; when it starts with //, as it does for a stateDir of / +// or one starting with //, the empty host is kept, so the URL starts +// file://// and SQLite does not read the first name in the path as a +// host. +func defaultDBURL(stateDir string) string { + dbPath := stateDir + "/state.sqlite3" + + dbURL := url.URL{ + Scheme: "file", + OmitHost: !strings.HasPrefix(dbPath, "//"), + Path: dbPath, + RawQuery: "_pragma=journal_mode(WAL)", + } + + return dbURL.String() +} + // validateKnownKeys rejects configuration files containing keys the // application does not understand, so typos fail at startup instead of // being silently ignored, and rejects keys that are explicitly set to diff --git a/internal/config/config_validation_internal_test.go b/internal/config/config_validation_internal_test.go index abfdee7..1324050 100644 --- a/internal/config/config_validation_internal_test.go +++ b/internal/config/config_validation_internal_test.go @@ -135,6 +135,104 @@ func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) { } } +// TestDerivedDBURLOpensStateDirHoldingURLCharacters opens the db_url +// derived from a state_dir whose name holds %41, ? and #, and checks that +// the database is state.sqlite3 in that directory, in WAL mode. Put into +// a file: URL as it is, the name would lose its meaning: %41 is read as +// A, and the path ends at the ?, which also loses the journal mode. +func TestDerivedDBURLOpensStateDirHoldingURLCharacters(t *testing.T) { + t.Parallel() + + stateDir := filepath.Join(t.TempDir(), "a%41b?c#d") + + err := os.Mkdir(stateDir, 0o750) + if err != nil { + t.Fatalf("failed to create %q: %v", stateDir, err) + } + + // Quoted so YAML takes the whole name as it is. + c, err := configFromYAML(t, signingKeyLine+"state_dir: '"+stateDir+"'\n") + if err != nil { + t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err) + } + + t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL) + + checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(stateDir, "state.sqlite3")) +} + +// TestDerivedDBURLOpensStateDirStartingWithTwoSlashes opens the db_url +// derived from a state_dir that starts with //, and checks that the +// database is state.sqlite3 in that directory, in WAL mode. Written right +// after file:, such a name gives a URL starting file://, and SQLite reads +// the first directory name as a host and refuses to open it. +func TestDerivedDBURLOpensStateDirStartingWithTwoSlashes(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + stateDir := "/" + dir + + c, err := configFromYAML(t, signingKeyLine+"state_dir: "+stateDir+"\n") + if err != nil { + t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err) + } + + t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL) + + checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(dir, "state.sqlite3")) +} + +// TestDerivedDBURLForRootStateDir checks the db_url derived from a +// state_dir of /. The path written into the URL is then +// //state.sqlite3, and written right after file: it gives a URL starting +// file://, where SQLite reads state.sqlite3 as a host and refuses to open +// it. The test only checks the URL, since it cannot write to /. +func TestDerivedDBURLForRootStateDir(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+"state_dir: /\n") + if err != nil { + t.Fatalf("config with state_dir / should be valid, got: %v", err) + } + + t.Logf("state_dir / gives db_url %q", c.DBURL) + + wantDBURL := "file:////state.sqlite3?_pragma=journal_mode(WAL)" + if c.DBURL != wantDBURL { + t.Errorf("DBURL = %q, want %q", c.DBURL, wantDBURL) + } +} + +// checkDBURLOpensFileInWALMode opens dbURL with the SQLite driver pixad +// uses, and checks that the database is in WAL mode and is the file +// dbFile. +func checkDBURLOpensFileInWALMode(t *testing.T, dbURL, dbFile string) { + t.Helper() + + db, err := sql.Open("sqlite", dbURL) + if err != nil { + t.Fatalf("failed to open %q: %v", dbURL, err) + } + + t.Cleanup(func() { _ = db.Close() }) + + var journalMode string + + err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode) + if err != nil { + t.Fatalf("failed to read the journal mode of %q: %v", dbURL, err) + } + + if journalMode != "wal" { + t.Errorf("journal mode of %q = %q, want wal", dbURL, journalMode) + } + + _, err = os.Stat(dbFile) + if err != nil { + t.Errorf("opening %q did not create %q: %v", dbURL, dbFile, err) + } +} + func TestExplicitValidValuesAreUsed(t *testing.T) { t.Parallel() From 08d6c3c4592029f4faf65c9e6fcd3224ad0b4bd0 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 05:00:47 +0200 Subject: [PATCH 086/131] Fetch an IPv6 source host written without a port (closes #295) The fetcher took the host from the source URL and stripped a port with net.SplitHostPort, which fails when there is no port, so a bracketed IPv6 host such as [2001:db8::a] kept its brackets, its name lookup failed, and the request answered 500. validateURL now takes the host with url.URL.Hostname(), which drops the brackets and any port, so such a host is fetched like one with a port, through /v1/image/ and through an encrypted URL, and a private IPv6 address is still refused. Model: opus-5-5 --- TODO.md | 6 ++ internal/httpfetcher/httpfetcher.go | 16 +-- internal/server/source_url_internal_test.go | 110 ++++++++++++++++++++ 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/TODO.md b/TODO.md index da1345b..9ae228e 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,12 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 a source host that is an IPv6 address in brackets with no port, + such as `[2001:db8::a]`, is fetched like one with a port, through `/v1/image/` + and through an encrypted URL (closes #295). The fetcher's check of the source + URL kept the brackets when no port followed them, could not look the host up, + and the request answered 500. It now checks the address without brackets, so a + private one is refused with 403, as with a port. - 2026-10-16 a `state_dir` whose name holds `%`, `?` or `#` keeps its database at `state.sqlite3` in that directory (closes #290): the `db_url` derived from it put the name into a `file:` URL as it was, so SQLite read `%41` as `A`, a diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 1428040..159ee68 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -473,16 +473,16 @@ func validateURL(ctx context.Context, rawURL string, allowHTTP bool) error { return ErrUnsupportedScheme } - // Parse to extract host - host := extractHost(rawURL) - if host == "" { - return ErrInvalidHost + parsedURL, err := neturl.Parse(rawURL) + if err != nil { + return fmt.Errorf("%w: %w", ErrInvalidHost, err) } - // Remove port if present - h, _, err := net.SplitHostPort(host) - if err == nil { - host = h + // Hostname drops any port, and the brackets of an IPv6 address whether + // or not a port follows them + host := parsedURL.Hostname() + if host == "" { + return ErrInvalidHost } // Block obvious localhost patterns diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index d006c44..0a5b4a1 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -6,12 +6,16 @@ import ( "net/url" "regexp" "slices" + "strconv" "strings" "sync" "testing" + "time" + "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/session" + "sneak.berlin/go/pixa/internal/signature" ) // percentSignPath is a source path holding a "%", which a URL writes %25. @@ -433,3 +437,109 @@ func TestGeneratorRefusesSourceURLItCannotFetchAsWritten(t *testing.T) { }) } } + +// ipv6SourcePath is the source path of the image URLs signedImageURL and +// encryptedImageURL make. +const ipv6SourcePath = "/ipv6.png" + +// TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort requests ipv6SourcePath +// on an IPv6 source host, written with and without a port, through a +// /v1/image/ URL with a signature and through an encrypted URL. The public +// address 2001:db8::a, a documentation address (RFC 3849) the fetcher accepts +// as public, answers 200, and the upstream receives the request for +// ipv6SourcePath. The private address fd00::1 answers 403, and the upstream +// receives no request: the test's fetcher connects every address to the test +// upstream, so only the fetcher's check of the source URL can refuse it. +func TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sourceHost string + wantStatus int + // wantRequests are the request targets the upstream must receive + wantRequests []string + }{ + {"public address without a port", "[2001:db8::a]", + http.StatusOK, []string{ipv6SourcePath}}, + {"public address with a port", "[2001:db8::a]:8443", + http.StatusOK, []string{ipv6SourcePath}}, + {"private address without a port", "[fd00::1]", + http.StatusForbidden, nil}, + {"private address with a port", "[fd00::1]:8443", + http.StatusForbidden, nil}, + } + + for _, tt := range tests { + imageURLs := map[string]string{ + "image URL with a signature": signedImageURL(t, tt.sourceHost), + "encrypted URL": encryptedImageURL(t, tt.sourceHost), + } + + for route, imageURL := range imageURLs { + t.Run(tt.name+" "+route, func(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, imageURL, nil)) + t.Logf("GET %s: %d", imageURL, rec.Code) + + if rec.Code != tt.wantStatus { + t.Errorf("status = %d, want %d; body %s", + rec.Code, tt.wantStatus, rec.Body.String()) + } + + got := upstream.requestTargets() + t.Logf("upstream received %q", got) + + if !slices.Equal(got, tt.wantRequests) { + t.Errorf("upstream received %q, want %q", got, tt.wantRequests) + } + }) + } + } +} + +// signedImageURL returns the /v1/image/ URL of ipv6SourcePath on sourceHost at +// its original size as PNG, with a signature made with the test signing key, +// which a host not on the allowlist needs. The URL has no q or fit, so it is +// signed with the quality 85 and the fit cover. +func signedImageURL(t *testing.T, sourceHost string) string { + t.Helper() + + path, sig, exp := signature.New(testSigningKey).GenerateSignedURL( + &signature.Request{ + SourceHost: sourceHost, + SourcePath: ipv6SourcePath, + Format: string(imgcache.FormatPNG), + Quality: 85, + FitMode: string(imgcache.FitCover), + }, time.Hour) + + return path + "?sig=" + sig + "&exp=" + strconv.FormatInt(exp, 10) +} + +// encryptedImageURL returns the encrypted URL of ipv6SourcePath on sourceHost +// at its original size as PNG, made with the test signing key. +func encryptedImageURL(t *testing.T, sourceHost string) string { + t.Helper() + + generator, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator: %v", err) + } + + token, err := generator.Generate(&encurl.Payload{ + SourceHost: sourceHost, SourcePath: ipv6SourcePath, + Format: imgcache.FormatPNG, + }) + if err != nil { + t.Fatalf("encurl Generate: %v", err) + } + + return "/v1/e/" + token + "/img.png" +} From 3a6b3bc202b62563dd6f5ed42ac6e3850b8bab52 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 05:31:55 +0200 Subject: [PATCH 087/131] Refuse an IPv6 address that holds an internal IPv4 address (closes #297) The check that keeps pixa from fetching internal addresses read an IPv6 address as IPv4 only in the mapped form (::ffff:a.b.c.d), so the IPv4-compatible form, such as ::127.0.0.1, was judged as IPv6 and passed. An address in ::/96 is now judged by the IPv4 address it holds, so one holding a private, loopback or link-local address is refused on both image routes before any upstream request. The NAT64 range 64:ff9b::/96 stays blocked whole, as it was on next. Model: opus-5-5 --- TODO.md | 8 ++++ internal/httpfetcher/httpfetcher.go | 7 +++ .../httpfetcher/httpfetcher_internal_test.go | 38 ++++++++++++++++ internal/server/source_url_internal_test.go | 43 +++++++++++++++++++ 4 files changed, 96 insertions(+) diff --git a/TODO.md b/TODO.md index 9ae228e..5009531 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal + addresses judges an IPv6 address in the IPv4-compatible form (`::/96`), such + as `::127.0.0.1` or `::10.0.0.1`, by the IPv4 address in its last four bytes + (closes #297). It read only the mapped form (`::ffff:a.b.c.d`) as IPv4, so a + source host such as `[::127.0.0.1]` was checked as a public IPv6 address and + fetched, which on a network that routes it reaches the internal IPv4 host; it + is now refused with 403. The NAT64 form (`64:ff9b::/96`) was already refused + as a whole, as one of the built-in blocked ranges, and still is. - 2026-10-16 a source host that is an IPv6 address in brackets with no port, such as `[2001:db8::a]`, is fetched like one with a port, through `/v1/image/` and through an encrypted URL (closes #295). The fetcher's check of the source diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 159ee68..53d5b20 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -543,6 +543,13 @@ func isPrivateIP(ip net.IP) bool { return true } + // An IPv4-compatible IPv6 address (in ::/96, such as ::127.0.0.1) holds + // an IPv4 address in its last four bytes and is judged as that address + if addr, ok := netip.AddrFromSlice(ip); ok && + netip.MustParsePrefix("::/96").Contains(addr) { + return isPrivateIP(ip[12:]) + } + // Check for loopback if ip.IsLoopback() { return true diff --git a/internal/httpfetcher/httpfetcher_internal_test.go b/internal/httpfetcher/httpfetcher_internal_test.go index a774614..c7a8970 100644 --- a/internal/httpfetcher/httpfetcher_internal_test.go +++ b/internal/httpfetcher/httpfetcher_internal_test.go @@ -186,6 +186,44 @@ func TestIsPrivateIP(t *testing.T) { } } +// TestIsPrivateIPJudgesEmbeddedIPv4Address checks IPv6 addresses that hold an +// IPv4 address in their last four bytes: the IPv4-compatible form (::/96) and +// the NAT64 form (64:ff9b::/96). One holding a loopback, private or +// link-local IPv4 address is refused as that address is, and one holding a +// public IPv4 address in the IPv4-compatible form passes. +func TestIsPrivateIPJudgesEmbeddedIPv4Address(t *testing.T) { + t.Parallel() + + tests := []struct { + ip string + want bool + }{ + {"::127.0.0.1", true}, // loopback + {"::10.0.0.1", true}, // private + {"::169.254.169.254", true}, // link-local + {"64:ff9b::7f00:1", true}, // NAT64 form of 127.0.0.1 + {"::8.8.8.8", false}, // public + } + + for _, tc := range tests { + t.Run(tc.ip, func(t *testing.T) { + t.Parallel() + + ip := net.ParseIP(tc.ip) + if ip == nil { + t.Fatalf("failed to parse IP %q", tc.ip) + } + + got := isPrivateIP(ip) + t.Logf("isPrivateIP(%s) = %v", ip, got) + + if got != tc.want { + t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want) + } + }) + } +} + func TestValidateURL_RejectsNonHTTPS(t *testing.T) { t.Parallel() diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index 0a5b4a1..172de90 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -543,3 +543,46 @@ func encryptedImageURL(t *testing.T, sourceHost string) string { return "/v1/e/" + token + "/img.png" } + +// TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address requests +// ipv6SourcePath on IPv6 source hosts that hold a loopback or private IPv4 +// address in their last four bytes, in the IPv4-compatible form (::/96) or the +// NAT64 form (64:ff9b::/96), through a /v1/image/ URL with a signature and +// through an encrypted URL. Each answers 403, and the upstream receives no +// request: the test's fetcher connects every address to the test upstream, so +// only the fetcher's check of the source URL can refuse it. +func TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address(t *testing.T) { + t.Parallel() + + upstream := startRecordingUpstream(t) + s, _, _ := startImageProxy(t, upstream.Server) + + sourceHosts := []string{ + "[::127.0.0.1]", // IPv4-compatible form of a loopback address + "[::10.0.0.1]", // IPv4-compatible form of a private address + "[64:ff9b::7f00:1]", // NAT64 form of 127.0.0.1 + } + + for _, sourceHost := range sourceHosts { + imageURLs := []string{ + signedImageURL(t, sourceHost), + encryptedImageURL(t, sourceHost), + } + + for _, imageURL := range imageURLs { + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, imageURL, nil)) + t.Logf("GET %s (source host %s): %d", imageURL, sourceHost, rec.Code) + + if rec.Code != http.StatusForbidden { + t.Errorf("source host %s: status = %d, want %d; body %s", + sourceHost, rec.Code, http.StatusForbidden, rec.Body.String()) + } + } + } + + if got := upstream.requestTargets(); len(got) != 0 { + t.Errorf("upstream received %q, want no request", got) + } +} From faad959bea547d07d000234627e35d8f465d0663 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 16 Oct 2026 06:52:17 +0200 Subject: [PATCH 088/131] Refuse local-use NAT64 addresses, and IPv4-compatible ones in blocked_networks (closes #299) The check that keeps pixa from fetching internal addresses let through 64:ff9b:1::/48, the local-use NAT64 range, and matched an address in the IPv4-compatible form (::/96) against blocked_networks only as IPv6, so ::203.0.113.5 passed with 203.0.113.0/24 listed. The local-use NAT64 range is now blocked next to 64:ff9b::/96, and an address in ::/96 is refused when blocked_networks holds it as written or the IPv4 address it holds. The mapped form stays matched as IPv4 only, as Go keeps plain IPv4 addresses in that form; README.md says so. Model: opus-5-5 --- README.md | 5 +- TODO.md | 12 +++ internal/httpfetcher/httpfetcher.go | 48 ++++++++---- internal/server/image_proxy_internal_test.go | 53 ++++++++++++++ internal/server/source_url_internal_test.go | 77 ++++++++++++++++++++ 5 files changed, 180 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7a71a5d..6338c40 100644 --- a/README.md +++ b/README.md @@ -694,7 +694,10 @@ Key settings in more detail: and generator pages. Default: empty - `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added to the always-enforced built-in ranges (loopback, private, link-local, CGNAT, - benchmark, NAT64, and the like); an invalid CIDR aborts startup + benchmark, NAT64, and the like); an invalid CIDR aborts startup. An IPv6 + address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 + address it holds, so an IPv6 range such as `::/0` refuses neither it nor a + plain IPv4 address - `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one of these ranges; the logged and login-recorded client address is then the diff --git a/TODO.md b/TODO.md index 5009531..596940e 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,18 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal + addresses refuses the local-use NAT64 range (`64:ff9b:1::/48`, RFC 8215) as it + refuses `64:ff9b::/96`, and refuses an IPv6 address in the IPv4-compatible + form (`::/96`) when `blocked_networks` contains either the address as written + or the IPv4 address it holds (closes #299). A source host such as + `[64:ff9b:1::a00:1]` was fetched, which on a network that runs NAT64 on that + range reaches the IPv4 address it holds, here `10.0.0.1`; and with + `203.0.113.0/24` in `blocked_networks`, `[::ffff:203.0.113.5]` was refused but + `[::203.0.113.5]` was fetched. Both now answer 403. The mapped form + (`::ffff:a.b.c.d`) is still matched only as the IPv4 address it holds: Go + keeps a plain IPv4 address in that form, and an IPv6 entry such as `::/0` must + not refuse every IPv4 address. - 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal addresses judges an IPv6 address in the IPv4-compatible form (`::/96`), such as `::127.0.0.1` or `::10.0.0.1`, by the IPv4 address in its last four bytes diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 53d5b20..29373f4 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -59,15 +59,17 @@ const ( // builtinBlockedPrefixes are internal or special-use ranges that Go's // net.IP predicates (IsPrivate, IsLinkLocalUnicast, and the like) do not // already cover. They are always blocked, in addition to any -// operator-supplied networks. IPv4-mapped IPv6 addresses are unmapped -// before matching, so these IPv4 ranges are caught in both forms. +// operator-supplied networks. IPv4-mapped and IPv4-compatible IPv6 +// addresses are read as the IPv4 address they hold before matching, so +// these IPv4 ranges are caught in all three forms. // //nolint:gochecknoglobals // immutable built-in blocklist var builtinBlockedPrefixes = []netip.Prefix{ - netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT / carrier-grade NAT - netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments - netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range - netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4) + netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT / carrier-grade NAT + netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments + netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range + netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4) + netip.MustParsePrefix("64:ff9b:1::/48"), // RFC 8215 local-use NAT64 (maps onto IPv4) } // Fetcher errors. @@ -543,12 +545,8 @@ func isPrivateIP(ip net.IP) bool { return true } - // An IPv4-compatible IPv6 address (in ::/96, such as ::127.0.0.1) holds - // an IPv4 address in its last four bytes and is judged as that address - if addr, ok := netip.AddrFromSlice(ip); ok && - netip.MustParsePrefix("::/96").Contains(addr) { - return isPrivateIP(ip[12:]) - } + // An IPv4-compatible IPv6 address is judged as the IPv4 address it holds + ip = ipv4CompatibleToIPv4(ip) // Check for loopback if ip.IsLoopback() { @@ -601,7 +599,12 @@ func isPrivateIP(ip net.IP) bool { } // isBlockedIP reports whether ip is refused, either by the built-in -// internal-range check or by one of the operator-supplied prefixes. +// internal-range check or by one of the operator-supplied prefixes. An +// IPv4-compatible address (in ::/96) is refused when a prefix contains the +// address as written or the IPv4 address it holds. An IPv4-mapped address (in +// ::ffff:0:0/96) is matched only as the IPv4 address it holds: Go keeps a +// plain IPv4 address in that form too, and an IPv6 prefix such as ::/0 must +// not refuse every IPv4 address. func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool { if isPrivateIP(ip) { return true @@ -614,11 +617,28 @@ func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool { addr = addr.Unmap() + // ip read as IPv4 where it is or holds an IPv4 address, otherwise the + // zero netip.Addr, which no prefix contains. Matching asIPv4 as well as + // addr refuses more only for an IPv4-compatible address. + asIPv4, _ := netip.AddrFromSlice(ipv4CompatibleToIPv4(ip).To4()) + return slices.ContainsFunc(blocked, func(prefix netip.Prefix) bool { - return prefix.Contains(addr) + return prefix.Contains(addr) || prefix.Contains(asIPv4) }) } +// ipv4CompatibleToIPv4 returns the IPv4 address that an IPv4-compatible IPv6 +// address (in ::/96, such as ::127.0.0.1) holds in its last four bytes, and +// any other address unchanged. +func ipv4CompatibleToIPv4(ip net.IP) net.IP { + addr, ok := netip.AddrFromSlice(ip) + if ok && netip.MustParsePrefix("::/96").Contains(addr) { + return ip[12:] + } + + return ip +} + // ssrfSafeDialer validates IP addresses against the built-in blocked ranges // before connecting. New wraps dialSSRFSafe with the operator-supplied // blocklist; this entry point enforces the built-in ranges alone. diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go index 24b7961..ba56e4f 100644 --- a/internal/server/image_proxy_internal_test.go +++ b/internal/server/image_proxy_internal_test.go @@ -14,10 +14,12 @@ import ( "net" "net/http" "net/http/httptest" + "net/netip" "os" "path/filepath" "sync/atomic" "testing" + "time" "go.uber.org/fx" "go.uber.org/fx/fxtest" @@ -183,6 +185,57 @@ func startImageProxy( return s, db.DB(), stateDir } +// startImageProxyWithBlockedNetworks starts the components pixad's fx app +// builds, from a config with a fresh state directory and blockedNetworks as +// blocked_networks. Unlike startImageProxy it provides no upstream fetcher, so +// the handlers build the one pixad uses, which checks every address it +// connects to against the built-in blocked ranges and blocked_networks. It +// returns the server with its routes. +func startImageProxyWithBlockedNetworks( + t *testing.T, blockedNetworks []netip.Prefix, +) *Server { + t.Helper() + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + BlockedNetworks: blockedNetworks, + // A fetch that gets past the checks and finds nothing at the address + // fails within this time at the latest. + UpstreamFetchTimeout: 2 * time.Second, + // With no connection slots, the fetch would fail before connecting. + UpstreamConnections: config.DefaultUpstreamConnections, + DownstreamTimeout: config.DefaultDownstreamTimeout, + } + + var ( + h *handlers.Handlers + mw *middleware.Middleware + ) + + app := fxtest.New(t, + fx.Supply(cfg), + fx.Provide( + globals.New, + logger.New, + database.New, + healthcheck.New, + handlers.New, + middleware.New, + ), + fx.Populate(&h, &mw), + ) + app.RequireStart() + t.Cleanup(app.RequireStop) + + s := &Server{config: cfg, mw: mw, h: h} + s.SetupRoutes() + + return s +} + // getImage sends a GET for target to s and fails unless it answers 200. func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder { t.Helper() diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index 172de90..27b963b 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -3,6 +3,7 @@ package server import ( "net/http" "net/http/httptest" + "net/netip" "net/url" "regexp" "slices" @@ -586,3 +587,79 @@ func TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address(t *testing.T) t.Errorf("upstream received %q, want no request", got) } } + +// TestImageRoutesCheckIPv4AddressInIPv6SourceHost requests ipv6SourcePath on +// source hosts that hold an IPv4 address, through a /v1/image/ URL with a +// signature and through an encrypted URL, with the fetcher pixad uses. +// [64:ff9b:1::a00:1], in the local-use NAT64 range (64:ff9b:1::/48, RFC 8215), +// answers 403. [::203.0.113.5] holds the public address 203.0.113.5 in the +// IPv4-compatible form (::/96) and answers 403 when blocked_networks lists +// 203.0.113.0/24 or ::/96. [::ffff:203.0.113.5] holds it in the IPv4-mapped +// form (::ffff:0:0/96) and answers 403 when blocked_networks lists +// 203.0.113.0/24. Go keeps the plain address 203.0.113.5 in the mapped form +// too, and an IPv6 range must not refuse it: with ::/0 listed it is not +// refused. Neither is [::203.0.113.5] with blocked_networks empty. Where a +// host is not refused, the fetcher tries to connect to it, finds nothing +// there, and the request fails with another status. +func TestImageRoutesCheckIPv4AddressInIPv6SourceHost(t *testing.T) { + t.Parallel() + + const ( + ipv4CompatibleHost = "[::203.0.113.5]" + ipv4MappedHost = "[::ffff:203.0.113.5]" + ) + + tests := []struct { + name string + sourceHost string + blockedNetworks []netip.Prefix + wantForbidden bool + }{ + {"local-use NAT64 form of 10.0.0.1", "[64:ff9b:1::a00:1]", nil, true}, + {"IPv4-compatible form of an address in blocked_networks", + ipv4CompatibleHost, + []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, true}, + {"IPv4-compatible form with ::/96 in blocked_networks", + ipv4CompatibleHost, + []netip.Prefix{netip.MustParsePrefix("::/96")}, true}, + {"IPv4-mapped form of an address in blocked_networks", + ipv4MappedHost, + []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, true}, + {"plain IPv4 address with ::/0 in blocked_networks", + "203.0.113.5", + []netip.Prefix{netip.MustParsePrefix("::/0")}, false}, + {"IPv4-compatible form of an address with blocked_networks empty", + ipv4CompatibleHost, nil, false}, + } + + for _, tt := range tests { + imageURLs := map[string]string{ + "image URL with a signature": signedImageURL(t, tt.sourceHost), + "encrypted URL": encryptedImageURL(t, tt.sourceHost), + } + + for route, imageURL := range imageURLs { + t.Run(tt.name+" "+route, func(t *testing.T) { + t.Parallel() + + s := startImageProxyWithBlockedNetworks(t, tt.blockedNetworks) + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, imageURL, nil)) + t.Logf("GET %s: %d", imageURL, rec.Code) + + forbidden := rec.Code == http.StatusForbidden + if tt.wantForbidden && !forbidden { + t.Errorf("status = %d, want %d; body %s", + rec.Code, http.StatusForbidden, rec.Body.String()) + } + + if !tt.wantForbidden && forbidden { + t.Errorf("status = %d, want any other status; body %s", + rec.Code, rec.Body.String()) + } + }) + } + } +} From 0ab15324f7d0d82a97fd482c5408fce5bb317014 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 19 Oct 2026 03:00:43 +0200 Subject: [PATCH 089/131] Read a blocked_networks entry in the IPv4-mapped form as its IPv4 range (closes #301) The fetcher matches an IPv4-mapped address (::ffff:a.b.c.d) as the IPv4 address it holds, so a blocked_networks entry written in that form was accepted and matched nothing. Such an entry is now kept as the IPv4 range it covers: ::ffff:10.0.0.0/104 becomes 10.0.0.0/8, and ::ffff:0:0/96 becomes 0.0.0.0/0. One shorter than /96 covers more than IPv4 addresses and aborts startup naming the setting. README.md says both. trusted_proxies shares the list parser, has the same fault and is unchanged here (issue 303). Model: opus-5-5 --- README.md | 4 +- TODO.md | 3 ++ .../config/blocked_networks_internal_test.go | 53 +++++++++++++++++++ internal/config/config.go | 36 ++++++++++++- 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6338c40..c069b90 100644 --- a/README.md +++ b/README.md @@ -697,7 +697,9 @@ Key settings in more detail: benchmark, NAT64, and the like); an invalid CIDR aborts startup. An IPv6 address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 address it holds, so an IPv6 range such as `::/0` refuses neither it nor a - plain IPv4 address + plain IPv4 address. An entry written in that form is read as the IPv4 range it + covers, so `::ffff:10.0.0.0/104` refuses what `10.0.0.0/8` refuses; such an + entry shorter than `/96` covers more than IPv4 addresses and aborts startup - `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one of these ranges; the logged and login-recorded client address is then the diff --git a/TODO.md b/TODO.md index 596940e..06e7082 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,9 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-18 a `blocked_networks` entry in the IPv4-mapped form refused nothing; + `::ffff:10.0.0.0/104` is now read as `10.0.0.0/8`, and such an entry shorter + than `/96` aborts startup (closes #301). - 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal addresses refuses the local-use NAT64 range (`64:ff9b:1::/48`, RFC 8215) as it refuses `64:ff9b::/96`, and refuses an IPv6 address in the IPv4-compatible diff --git a/internal/config/blocked_networks_internal_test.go b/internal/config/blocked_networks_internal_test.go index 25373be..475b1d4 100644 --- a/internal/config/blocked_networks_internal_test.go +++ b/internal/config/blocked_networks_internal_test.go @@ -88,3 +88,56 @@ func TestBlockedNetworksInvalidAbortsStartup(t *testing.T) { }, }) } + +// TestBlockedNetworksMappedEntryIsIPv4Range checks that an entry written in +// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers, +// since the fetcher matches an IPv4-mapped address as the IPv4 address it +// holds. /96, the shortest such entry, covers every IPv4 address. +func TestBlockedNetworksMappedEntryIsIPv4Range(t *testing.T) { + t.Parallel() + + yamlContent := signingKeyLine + `blocked_networks: + - "::ffff:10.0.0.0/104" + - "::ffff:0:0/96" +` + + c, err := configFromYAML(t, yamlContent) + if err != nil { + t.Fatalf("IPv4-mapped blocked_networks entries should load, got error: %v", err) + } + + want := []string{"10.0.0.0/8", "0.0.0.0/0"} + if len(c.BlockedNetworks) != len(want) { + t.Fatalf("BlockedNetworks = %v, want %d entries", c.BlockedNetworks, len(want)) + } + + for i, w := range want { + if got := c.BlockedNetworks[i].String(); got != w { + t.Errorf("BlockedNetworks[%d] = %q, want %q", i, got, w) + } + } +} + +// TestBlockedNetworksMappedEntryShorterThan96AbortsStartup checks that an +// entry in the IPv4-mapped form shorter than /96, which covers more than the +// IPv4 addresses, aborts startup with an error naming the key and the value. +func TestBlockedNetworksMappedEntryShorterThan96AbortsStartup(t *testing.T) { + t.Parallel() + + runAbortCases(t, []abortCase{ + { + name: "mapped-shorter-than-96", + yaml: signingKeyLine + `blocked_networks: + - "::ffff:0.0.0.0/80" +`, + wantErrSubstrings: []string{keyBlockedNetworks, "::ffff:0.0.0.0/80"}, + }, + { + name: "mapped-95", + yaml: signingKeyLine + `blocked_networks: + - "::ffff:0:0/95" +`, + wantErrSubstrings: []string{keyBlockedNetworks}, + }, + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index 40eb799..4a7bd72 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -91,6 +91,7 @@ var ( errNotABoolean = errors.New("not a boolean") errNotAStringList = errors.New("not a list of strings") errNotAValidCIDR = errors.New("not a valid CIDR network") + errMappedTooShort = errors.New("in the IPv4-mapped form but shorter than /96") errNotAMetricsMap = errors.New("not a map of metrics settings") errEmptyListEntry = errors.New("list contains an empty entry") errEmptyEntry = errors.New("contains an empty entry") @@ -271,7 +272,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { return nil, err } - blockedNetworks, err := parseCIDRList(sc, keyBlockedNetworks) + blockedNetworks, err := parseBlockedNetworks(sc) if err != nil { return nil, err } @@ -1278,6 +1279,39 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { return prefixes, nil } +// parseBlockedNetworks parses blocked_networks with parseCIDRList, and keeps +// an entry written in the IPv4-mapped form (::ffff:a.b.c.d) as the IPv4 +// range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. The fetcher +// matches an IPv4-mapped address as the IPv4 address it holds, so the entry +// as written would match nothing. Such an entry shorter than /96 covers more +// than the IPv4 addresses and aborts startup. +func parseBlockedNetworks(sc *smartconfig.Config) ([]netip.Prefix, error) { + prefixes, err := parseCIDRList(sc, keyBlockedNetworks) + if err != nil { + return nil, err + } + + // An IPv4-mapped address is the 96 bits of ::ffff:0:0/96 followed by + // the 32 bits of the IPv4 address it holds. + const mappedPrefixBits = 96 + + for i, prefix := range prefixes { + if !prefix.Addr().Is4In6() { + continue + } + + if prefix.Bits() < mappedPrefixBits { + return nil, fmt.Errorf("%s: value %q is %w", + settingName(keyBlockedNetworks), prefix, errMappedTooShort) + } + + prefixes[i] = netip.PrefixFrom(prefix.Addr().Unmap(), + prefix.Bits()-mappedPrefixBits) + } + + return prefixes, nil +} + // parseHostList parses the value of the named config key into host patterns, // or returns nil if the key is omitted. It accepts a YAML list of strings or a // comma-separated string. An explicitly null value, a wrong type, an empty From 9eacfa64064e9c731fe26272abd53f8bf7f17102 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 19 Oct 2026 05:00:48 +0200 Subject: [PATCH 090/131] Space reconciliation passes by ten times the last pass's time (closes #250) The reconciliation pass reads a row and stats a file for every cached file, and it ran on every eviction tick, so on a large cache a pass longer than the tick was followed by the next one at once. It still runs at startup and is considered on each tick, but it now starts no sooner than ten times as long after the last pass ended as that pass took, so it takes at most about a tenth of the time however large the cache. Eviction and the removal of expired negative cache rows still run on every tick. The comments that bound the drift, and README.md, give the new bound. No existing test changed. Model: opus-5-5 --- README.md | 6 +- TODO.md | 3 + internal/imgcache/eviction.go | 68 +++++++++++------ internal/imgcache/eviction_internal_test.go | 83 +++++++++++++++++++++ 4 files changed, 137 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c069b90..b473c1a 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,11 @@ into place, so a crash leaves no file under its own name with part of its data. When pixa starts, and every five minutes after, it removes each cached source or transformed image whose file is empty, or not the size the database records for it, with a warning naming the file; the next request for it fetches the source -or converts the image again. +or converts the image again. Looking for those files reads the database once for +every cached file, so pixa skips each five-minute look that would start less +than ten times as long after the last look ended as that look took. After a look +that took one minute, the next one starts 10 to 15 minutes after that look +ended. pixa's target is 1-5k r/s, which has not been measured at that rate (see Load Test). Toward it, pixa keeps in memory the content types of the 10,000 diff --git a/TODO.md b/TODO.md index 06e7082..44ae4c1 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,9 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-18 the next reconciliation pass waits at least ten times as long as + the last one took, so passes on a large cache no longer run back to back + (closes #250). - 2026-10-18 a `blocked_networks` entry in the IPv4-mapped form refused nothing; `::ffff:10.0.0.0/104` is now read as `10.0.0.0/8`, and such an entry shorter than `/96` aborts startup (closes #301). diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 2fe34fb..83828c4 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -18,6 +18,14 @@ import ( // write-pressure wakeups triggered by stores. const DefaultEvictionInterval = 5 * time.Minute +// reconciliationWaitFactor is how many times as long as a +// reconciliation pass took the eviction loop waits at least, after that +// pass ends, before it starts the next one. A pass reads a row and stats a +// file for every cached file, so its time grows with the cache; waiting +// ten times as long keeps reconciliation to at most a tenth of the time, +// however large the cache is. +const reconciliationWaitFactor = 10 + // evictionBatchSize is how many LRU candidates of each class (variants // and source blobs) one eviction pass fetches from the database. const evictionBatchSize = 100 @@ -438,10 +446,12 @@ func (c *Cache) notifyWritePressure() { } // StartEviction launches the background eviction goroutine, which -// reconciles the database accounting with the cache directories and // removes the expired negative cache rows at startup and again on every -// periodic tick thereafter, and evicts to the configured limit on the -// given periodic interval and on write-pressure notifications. On a +// periodic tick thereafter, reconciles the database accounting with the +// cache directories at startup and again on the first periodic tick +// that comes at least ten times as long after a reconciliation pass +// ended as that pass took, and evicts to the configured limit on the given +// periodic interval and on write-pressure notifications. On a // disabled cache it only removes the expired negative cache rows, as // the negative cache is used either way. It is a no-op when already // started. The goroutine outlives the caller, so it runs with its own @@ -482,7 +492,7 @@ func (c *Cache) StopEviction(ctx context.Context) error { func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { defer close(c.evictionDone) - c.runReconciliationPass(ctx) + nextReconciliation := c.runReconciliationPass(ctx) c.runCleanExpired(ctx) c.runEvictionPass(ctx) @@ -494,16 +504,20 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { case <-ctx.Done(): return case <-ticker.C: - // Reconciliation walks the cache directories, so it only - // runs on the periodic ticker rather than on every - // write-pressure wakeup, keeping it off the per-store hot - // path. Reusing the eviction interval itself (rather than a - // separate, longer one) is a deliberate choice: it is the - // simplest option that still bounds how long a store's - // best-effort accounting insert can stay silently - // unaccounted for to one interval, on a process that is - // already running this loop regardless. - c.runReconciliationPass(ctx) + // Reconciliation walks the cache directories and reads a + // row for every file, so it runs only on the periodic + // ticker rather than on every write-pressure wakeup, and + // only on the first tick at or after nextReconciliation, + // which is ten times as long after the last pass ended as + // that pass took. Passes then never run back to back, + // however long one takes. A file whose best-effort + // accounting insert failed in a store stays unaccounted for + // until the next pass, up to about one eviction interval + // plus ten times the last pass's time. + if !time.Now().Before(nextReconciliation) { + nextReconciliation = c.runReconciliationPass(ctx) + } + c.runCleanExpired(ctx) case <-c.evictionPressure: } @@ -527,17 +541,25 @@ func (c *Cache) runEvictionPass(ctx context.Context) { } // runReconciliationPass runs one reconciliation pass, logging failures -// instead of propagating them (the loop must keep running). It does -// nothing once ctx is cancelled. -func (c *Cache) runReconciliationPass(ctx context.Context) { +// instead of propagating them (the loop must keep running), and returns +// the earliest time the next pass may start: reconciliationWaitFactor +// times as long after this pass ended as it took. It does nothing once +// ctx is cancelled. +func (c *Cache) runReconciliationPass(ctx context.Context) time.Time { if ctx.Err() != nil { - return + return time.Now() } + started := time.Now() + err := c.reconcileAccounting(ctx) if err != nil { c.log.Warn("cache accounting reconciliation failed", "error", err) } + + ended := time.Now() + + return ended.Add(reconciliationWaitFactor * ended.Sub(started)) } // runCleanExpired runs CleanExpired, logging a failure instead of @@ -556,7 +578,8 @@ func (c *Cache) runCleanExpired(ctx context.Context) { // reconcileAccounting synchronizes the database size accounting with // the actual contents of the cache directories. It runs at startup and -// again on every periodic eviction tick thereafter, off the request +// then on the first periodic eviction tick that comes at least ten times +// as long after the last pass ended as that pass took, off the request // hot path: it adopts variant files that predate the accounting table // (or whose accounting insert failed, e.g. StoreVariant's best-effort // insert under transient DB contention), drops accounting rows whose @@ -566,9 +589,10 @@ func (c *Cache) runCleanExpired(ctx context.Context) { // files left behind by crashed writes, and last checks the total cache // usage against the tables. It reads the tables a page at a time. // Running it periodically, not just once, bounds how long such drift can -// accumulate unaccounted for on a long-running process to one eviction -// interval. Once ctx is cancelled, it stops at the next file or row and -// returns ctx's error. +// accumulate unaccounted for on a long-running process to about one +// eviction interval plus ten times the last pass's time. +// Once ctx is cancelled, it stops at the next file or row and returns +// ctx's error. func (c *Cache) reconcileAccounting(ctx context.Context) error { if c.disabled { return nil diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 7d54df1..d1ddf31 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -918,6 +918,89 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { } } +// TestReconciliationPassesAreSpacedByTenTimesTheirTime makes the hook +// called after every read of the reconciliation pass sleep 10 ms, runs +// the eviction loop on a 10 ms interval, and records when each of those +// calls starts and ends. A pass starts before its first call and ends +// after its last, so the time from the first call's start to the last +// call's end is at most the pass's time. The next pass's first call must +// start at least ten times that long after the last call ended. +func TestReconciliationPassesAreSpacedByTenTimesTheirTime(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + // A pass made here, before the loop starts, counts the reads one + // pass makes on this empty cache, so the reads recorded below can be + // split into passes. + readsPerPass := 0 + cache.reconciliationReadTestHook = func(int) { readsPerPass++ } + + err := cache.reconcileAccounting(t.Context()) + if err != nil { + t.Fatalf("reconcileAccounting failed: %v", err) + } + + t.Logf("one reconciliation pass makes %d reads", readsPerPass) + + if readsPerPass == 0 { + t.Fatal("a reconciliation pass made no read") + } + + // The pass at startup and two after it. + const passes = 3 + + type read struct{ start, end time.Time } + + // The channel holds every read the test looks at, so none of those + // is dropped; reads after them are, and never block the loop. + reads := make(chan read, passes*readsPerPass) + + cache.reconciliationReadTestHook = func(int) { + start := time.Now() + + time.Sleep(10 * time.Millisecond) + + select { + case reads <- read{start: start, end: time.Now()}: + default: + } + } + + cache.StartEviction(10 * time.Millisecond) + defer func() { _ = cache.StopEviction(t.Context()) }() + + recorded := make([]read, 0, passes*readsPerPass) + timeout := time.After(15 * time.Second) + + for len(recorded) < passes*readsPerPass { + select { + case r := <-reads: + recorded = append(recorded, r) + case <-timeout: + t.Fatalf("saw %d reconciliation reads in 15 s, want %d (%d passes)", + len(recorded), passes*readsPerPass, passes) + } + } + + for pass := range passes - 1 { + first := recorded[pass*readsPerPass] + last := recorded[(pass+1)*readsPerPass-1] + next := recorded[(pass+1)*readsPerPass] + + passTime := last.end.Sub(first.start) + wait := next.start.Sub(last.end) + + t.Logf("pass %d took at least %v, and the next pass started %v after it", + pass+1, passTime, wait) + + if wait < 10*passTime { + t.Errorf("pass %d took at least %v, and the next pass started %v "+ + "after it, want at least %v", pass+1, passTime, wait, 10*passTime) + } + } +} + // TestStopEvictionInterruptsPassInProgress holds the test database's // only connection, so the startup reconciliation pass waits for it, and // checks that StopEviction stops that pass instead of waiting for the From 930e43961eb36ab14d7a54b549967405342c372c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 19 Oct 2026 05:29:05 +0200 Subject: [PATCH 091/131] Read a trusted_proxies entry in the IPv4-mapped form as its IPv4 range (closes #303) An entry such as ::ffff:10.0.0.0/104 in trusted_proxies was accepted but trusted nothing, because the client address is matched as the IPv4 address an IPv4-mapped address holds. The conversion that blocked_networks gained in https://git.eeqj.de/sneak/pixa/issues/301 now lives in parseCIDRList, which reads both settings: such an entry is read as the IPv4 range it covers, and one shorter than /96 stops startup with an error naming the setting being read and quoting the entry as written. README.md says so for trusted_proxies. Model: opus-5-5 --- README.md | 26 ++++++---- TODO.md | 4 ++ internal/config/config.go | 52 +++++++------------ .../config/trusted_proxies_internal_test.go | 38 ++++++++++++++ 4 files changed, 78 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index b473c1a..9eae101 100644 --- a/README.md +++ b/README.md @@ -713,16 +713,22 @@ Key settings in more detail: omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a private network; an explicitly empty list (`[]`) trusts no one, and an - explicit list replaces the default. An invalid CIDR aborts startup. Set this - to the address pixa sees for requests that come through your proxy, such as - `172.17.0.1/32`, when the defaults do not cover it, or to trust nothing else - (see the login limit and `image_requests_per_minute` under Routes). For a - proxy on the Docker host that connects to pixa over `127.0.0.1`, that address - is the gateway of the container's Docker network (`172.17.0.1` on the default - bridge), not the proxy's own address; a proxy that connects through another of - the host's addresses is seen with that address. To be sure which address it - is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request - through the proxy, and read `remoteIP` in pixa's request log line for it + explicit list replaces the default. An invalid CIDR aborts startup. An IPv6 + address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 + address it holds, so an IPv6 range such as `::/0` trusts neither it nor a + plain IPv4 address. An entry written in that form is read as the IPv4 range it + covers, so `::ffff:10.0.0.0/104` trusts what `10.0.0.0/8` trusts; such an + entry shorter than `/96` covers more than IPv4 addresses and aborts startup. + Set this to the address pixa sees for requests that come through your proxy, + such as `172.17.0.1/32`, when the defaults do not cover it, or to trust + nothing else (see the login limit and `image_requests_per_minute` under + Routes). For a proxy on the Docker host that connects to pixa over + `127.0.0.1`, that address is the gateway of the container's Docker network + (`172.17.0.1` on the default bridge), not the proxy's own address; a proxy + that connects through another of the host's addresses is seen with that + address. To be sure which address it is, set this to `[]` (or + `PIXA_TRUSTED_PROXIES` to empty), send a request through the proxy, and read + `remoteIP` in pixa's request log line for it - `upstream_fetch_timeout` — time allowed for one fetch from an upstream host, as a duration such as `30s` (the default) or `2m` - `upstream_max_response_size` — largest upstream response accepted, in bytes; diff --git a/TODO.md b/TODO.md index 44ae4c1..284d19d 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,10 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-19 a `trusted_proxies` entry in the IPv4-mapped form trusted nothing; + it is now read as the IPv4 range it covers by the same code that reads + `blocked_networks`, and such an entry shorter than `/96` aborts startup + (closes #303). - 2026-10-18 the next reconciliation pass waits at least ten times as long as the last one took, so passes on a large cache no longer run back to back (closes #250). diff --git a/internal/config/config.go b/internal/config/config.go index 4a7bd72..eff51ce 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -272,7 +272,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { return nil, err } - blockedNetworks, err := parseBlockedNetworks(sc) + blockedNetworks, err := parseCIDRList(sc, keyBlockedNetworks) if err != nil { return nil, err } @@ -1249,6 +1249,13 @@ func defaultTrustedProxies() []netip.Prefix { // wrong type, an empty entry, a non-string entry, or an unparseable CIDR // aborts startup naming the key and the offending value; the default // (an empty list) applies only to an omitted key. +// +// An entry written in the IPv4-mapped form (::ffff:a.b.c.d) is kept as the +// IPv4 range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. Both +// blocked_networks and trusted_proxies are matched against an IPv4-mapped +// address as the IPv4 address it holds, so the entry as written would match +// nothing. Such an entry shorter than /96 covers more than the IPv4 +// addresses and aborts startup. func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { raw, ok := lookupValue(sc, key) if !ok { @@ -1264,6 +1271,10 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { return nil, err } + // An IPv4-mapped address is the 96 bits of ::ffff:0:0/96 followed by + // the 32 bits of the IPv4 address it holds. + const mappedPrefixBits = 96 + prefixes := make([]netip.Prefix, 0, len(entries)) for _, entry := range entries { @@ -1273,40 +1284,17 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { settingName(key), entry, errNotAValidCIDR) } - prefixes = append(prefixes, prefix) - } - - return prefixes, nil -} - -// parseBlockedNetworks parses blocked_networks with parseCIDRList, and keeps -// an entry written in the IPv4-mapped form (::ffff:a.b.c.d) as the IPv4 -// range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. The fetcher -// matches an IPv4-mapped address as the IPv4 address it holds, so the entry -// as written would match nothing. Such an entry shorter than /96 covers more -// than the IPv4 addresses and aborts startup. -func parseBlockedNetworks(sc *smartconfig.Config) ([]netip.Prefix, error) { - prefixes, err := parseCIDRList(sc, keyBlockedNetworks) - if err != nil { - return nil, err - } - - // An IPv4-mapped address is the 96 bits of ::ffff:0:0/96 followed by - // the 32 bits of the IPv4 address it holds. - const mappedPrefixBits = 96 - - for i, prefix := range prefixes { - if !prefix.Addr().Is4In6() { - continue - } - - if prefix.Bits() < mappedPrefixBits { + if prefix.Addr().Is4In6() && prefix.Bits() < mappedPrefixBits { return nil, fmt.Errorf("%s: value %q is %w", - settingName(keyBlockedNetworks), prefix, errMappedTooShort) + settingName(key), entry, errMappedTooShort) } - prefixes[i] = netip.PrefixFrom(prefix.Addr().Unmap(), - prefix.Bits()-mappedPrefixBits) + if prefix.Addr().Is4In6() { + prefix = netip.PrefixFrom(prefix.Addr().Unmap(), + prefix.Bits()-mappedPrefixBits) + } + + prefixes = append(prefixes, prefix) } return prefixes, nil diff --git a/internal/config/trusted_proxies_internal_test.go b/internal/config/trusted_proxies_internal_test.go index ae61bfd..41a184d 100644 --- a/internal/config/trusted_proxies_internal_test.go +++ b/internal/config/trusted_proxies_internal_test.go @@ -83,3 +83,41 @@ func TestTrustedProxiesInvalidAbortsStartup(t *testing.T) { }, }) } + +// TestTrustedProxiesMappedEntryIsIPv4Range checks that an entry written in +// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers, +// since the client address is matched as the IPv4 address it holds. /96, the +// shortest such entry, covers every IPv4 address. +func TestTrustedProxiesMappedEntryIsIPv4Range(t *testing.T) { + t.Parallel() + + c, err := configFromYAML(t, signingKeyLine+ + `trusted_proxies: ["::ffff:10.0.0.0/104", "::ffff:0:0/96"]`+"\n") + if err != nil { + t.Fatalf("IPv4-mapped trusted_proxies entries should load: %v", err) + } + + got := make([]string, len(c.TrustedProxies)) + for i, p := range c.TrustedProxies { + got[i] = p.String() + } + + if joined := strings.Join(got, ","); joined != "10.0.0.0/8,0.0.0.0/0" { + t.Errorf("TrustedProxies = %v, want 10.0.0.0/8 and 0.0.0.0/0", got) + } +} + +// TestTrustedProxiesMappedEntryShorterThan96AbortsStartup checks that an +// entry in the IPv4-mapped form shorter than /96, which covers more than the +// IPv4 addresses, aborts startup with an error naming the key and the value. +func TestTrustedProxiesMappedEntryShorterThan96AbortsStartup(t *testing.T) { + t.Parallel() + + runAbortCases(t, []abortCase{ + { + name: "mapped-95", + yaml: signingKeyLine + `trusted_proxies: ["::ffff:0:0/95"]` + "\n", + wantErrSubstrings: []string{keyTrustedProxies, "::ffff:0:0/95"}, + }, + }) +} From 957ec85f97a6b4a5c0b7c9ce2c1d65f5018bea18 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 19 Oct 2026 06:11:43 +0200 Subject: [PATCH 092/131] Re-vendor the canonical files from sneak/prompts at 0653579 (closes #221) REPO_POLICIES.md, script/cibuild, script/docker, script/lint and script/test are their copies at that commit. In every Dockerfile stage the system package install and the Go module download now come before ARG CACHEBUST, so check builds keep those layers until the base image, the install line, go.mod or go.sum change; every step that lints, tests or compiles still reruns. golangci-lint run also mounts golangci-lint's cache. The stages install their packages with apt-get and apk directly instead of through script/bootstrap --cgo. The test phase installs no recommended packages and names libheif's AVIF encoder, libheif-plugin-aomenc, which the tests use. Model: opus-5-5 --- Dockerfile | 53 ++++++++++++++---------- README.md | 36 ++++++++++------- REPO_POLICIES.md | 103 +++++++++++++++++++++++++++++++---------------- TODO.md | 4 ++ script/bootstrap | 5 +-- script/cibuild | 9 +++-- script/docker | 9 +++-- script/lint | 12 +++--- script/test | 11 ++--- 9 files changed, 149 insertions(+), 93 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2d7c80..9cc15d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,16 @@ -# Every stage declares ARG CACHEBUST first: script/ passes a new value -# on every build, which reruns every RUN step after it. The steps that -# compile mount Go's build cache, which survives from build to build. +# In every stage ARG CACHEBUST comes right after the steps that install +# system packages and download Go's modules, or right after FROM in a +# stage with neither, and every step that lints, tests or compiles +# comes after it. script/ passes a new value on every build, which +# reruns every RUN step after it. Docker keeps the layers above it +# until the base image, their own lines or go.mod and go.sum change. +# The steps that compile mount Go's build cache, and golangci-lint run +# also mounts golangci-lint's; both survive from build to build. # Lint phase. script/lint builds it alone. The linter is run directly: # `make lint` and script/lint are themselves a docker build. # golangci/golangci-lint:v2.14.0, 2026-10-15 FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint -ARG CACHEBUST # The linter compiles every package, and govips needs the libvips # headers for that. REPO_POLICIES.md has the lint phase install them @@ -18,25 +22,29 @@ RUN apt-get update \ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download +ARG CACHEBUST COPY . . RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ golangci-lint run --config .golangci.yml ./... # Test phase. script/test builds it alone. -race needs cgo and so a C # compiler, which the Debian Go image ships and the alpine one does not. # golang:1.25.4-trixie, 2026-10-15 FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test -ARG CACHEBUST + +# govips compiles against the libvips and libheif headers, and the tests +# need libvips' JPEG XL support; libvips-dev brings all three. The tests +# also encode AVIF, which needs libheif's AVIF encoder, +# libheif-plugin-aomenc: Debian only recommends it, so it is named here. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev libheif-plugin-aomenc \ + && rm -rf /var/lib/apt/lists/* WORKDIR /src - -# script/bootstrap --cgo installs the build dependencies (the libvips -# and libheif headers, and libvips' JPEG XL support, which the tests -# need) and downloads the Go modules. -COPY script/ ./script/ COPY go.mod go.sum ./ -RUN script/bootstrap --cgo - +RUN go mod download +ARG CACHEBUST COPY . . # Without -v first; on a failure, again with -v for the details, and @@ -52,22 +60,23 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; } # Build stage. Nothing is wanted from the two phases above: these copies -# make BuildKit build them first, so this stage runs only when lint and -# test passed. +# make BuildKit build them first, so nothing after them runs unless lint +# and test passed. # golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder + +# git for the version, and for govips a C compiler, pkg-config and the +# libvips and libheif headers +RUN apk add --no-cache git build-base pkgconfig vips-dev libheif-dev + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download ARG CACHEBUST COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null -WORKDIR /src - -# Build dependencies, git among them, and Go modules, as in the test -# phase -COPY script/ ./script/ -COPY go.mod go.sum ./ -RUN script/bootstrap --cgo # A tar-stream context keeps the sender's file owners, which git refuses. RUN git config --system --add safe.directory /src @@ -102,7 +111,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # the libvips and musl it was built with. # alpine:3.22, 2026-10-08 FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 -ARG CACHEBUST # Install runtime dependencies only. vips-jxl is libvips' JPEG XL # support, without which pixad does not start. @@ -113,6 +121,7 @@ RUN apk add --no-cache \ ca-certificates \ tzdata \ su-exec +ARG CACHEBUST # Copy binary from builder COPY --from=builder /pixad /usr/local/bin/pixad diff --git a/README.md b/README.md index 9eae101..2e0f3d2 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL support, which on Alpine is the `vips-jxl` package and which the nix and brew packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04 on. Building pixa also needs the development files of libvips and libheif, -`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these, -as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which +`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these; +the `Dockerfile` installs its own packages. Plain `script/bootstrap`, which `script/setup` and `script/cibuild` run, installs git, make and Go, and Node, Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but none of the C libraries: the checks compile pixa in Docker. Docker itself must @@ -837,19 +837,25 @@ provide: `script/precommit` Every `docker build` in these scripts passes a new value of the `CACHEBUST` -build argument, which every stage of the `Dockerfile` declares first, so every -`RUN` step, the lint and test phases included, runs on every build instead of -coming from the build cache. Go's build cache is kept from build to build: the -steps that compile mount it, so a package whose inputs are unchanged is not -compiled again, and both `go test` runs pass `-count=1` so that no stored test -result is replayed. The `test` phase runs on the Debian Go image, with Debian's -libvips and libheif; the image pixa ships is built and runs on Alpine. -`script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`, -`script/setup` and `script/install-precommit` are the standard copies from -`sneak/prompts`, kept identical to them. `script/fmt` and `script/fmt-check` are -the standard copies with pixa's `gofmt` step kept before prettier. prettier -formats the markdown only, not the HTML templates, as it cannot parse a Go -template action inside a tag. +build argument, which every stage of the `Dockerfile` declares, so every `RUN` +step after that line, the lint and test steps included, runs on every build +instead of coming from the build cache. Above the line are the steps that +install system packages and download the Go modules; Docker keeps their layers +until the base image, the step itself, `go.mod` or `go.sum` changes. Go's build +cache is kept from build to build: the steps that compile mount it, so a package +whose inputs are unchanged is not compiled again, and both `go test` runs pass +`-count=1` so that no stored test result is replayed. The lint step also mounts +golangci-lint's cache, which reuses a package's findings only when its files, +those of the packages it imports, `go.mod`, the linters and their settings in +`.golangci.yml`, and the golangci-lint version are all unchanged. The `test` +phase runs on the Debian Go image, with Debian's libvips and libheif; the image +pixa ships is built and runs on Alpine. `script/check`, `script/cibuild`, +`script/docker`, `script/lint`, `script/test`, `script/setup` and +`script/install-precommit` are the standard copies from `sneak/prompts`, kept +identical to them. `script/fmt` and `script/fmt-check` are the standard copies +with pixa's `gofmt` step kept before prettier. prettier formats the markdown +only, not the HTML templates, as it cannot parse a Go template action inside a +tag. ## Load Test diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 5b24098..1067698 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-10-15 +last_modified: 2026-10-18 --- This document covers repository structure, tooling, and workflow standards. Code @@ -159,8 +159,11 @@ style conventions are in separate documents: one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs exit non-zero with `parallel golangci-lint is running` — a status a caller cannot tell from real findings. Both have produced wrong verdicts in this - org, in both directions. A container has its own golangci-lint cache, its - own `TMPDIR` and a digest-pinned binary, so neither is reachable. + org, in both directions. A container has its own `TMPDIR`, so the lock is + out of reach. The lint phase does keep golangci-lint's cache between check + builds (below), but it lints every checkout at `/src`, and a finding is + taken from that cache only for the same files, settings and golangci-lint + version. - **Any build that runs checks is given a new `CACHEBUST` value.** Docker invalidates a `COPY` layer only when the copied content changes, so on an @@ -170,15 +173,21 @@ style conventions are in separate documents: id, so no two builds share a value, not even the three in one `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the four, and there is no fifth — `script/check` runs the - two gate phases and `script/fmt-check`, and builds no image of its own. Every - stage of the `Dockerfile` declares `ARG CACHEBUST` as its first line after - `FROM`, and a new value reruns every `RUN` step after that line; `COPY` steps - are compared by content and need nothing. A stage without the `ARG` line is - served from cache on an unchanged tree, so a repository re-vendors the scripts - and adds the `ARG` lines in one commit. The scripts do not pass `--no-cache`: - it reruns every step too, but it also empties every cache mount, and Go's - build cache, which the canonical Go `Dockerfile` below mounts so that compiled - packages survive between check builds, would start empty every time. A bare + two gate phases and `script/fmt-check`, and builds no image of its own. In + every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps + that install system packages and download Go's modules, or right after `FROM` + in a stage with neither, and every step that lints, tests or compiles comes + after it. A step that installs anything else as well, such as a + `script/bootstrap` that also installs node packages, comes after it too. A new + value reruns every `RUN` step after that line; `COPY` steps are compared by + content and need nothing. Docker keeps the layers above the line until the + base image, their own lines or the files they copy change. A stage without the + `ARG` line is served from cache on an unchanged tree, so a repository + re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do + not pass `--no-cache`: it reruns every step too, but it also builds the layers + above the line again and empties every cache mount, so Go's build cache and + golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they + survive between check builds, would start empty every time. A bare `docker build .` is not evidence that anything ran: a sub-second build reporting success is a cache hit, not a result. Never invalidate by pruning — `docker builder prune` and friends destroy a build cache shared with every @@ -194,19 +203,25 @@ style conventions are in separate documents: `Dockerfile`: ```dockerfile - # Every stage declares ARG CACHEBUST first: script/ passes a new value - # on every build, which reruns every RUN step after it. The steps that - # compile mount Go's build cache, which survives from build to build. + # In every stage ARG CACHEBUST comes right after the steps that install + # system packages and download Go's modules, or right after FROM in a + # stage with neither, and every step that lints, tests or compiles + # comes after it. script/ passes a new value on every build, which + # reruns every RUN step after it. Docker keeps the layers above it + # until the base image, their own lines or go.mod and go.sum change. + # The steps that compile mount Go's build cache, and golangci-lint run + # also mounts golangci-lint's; both survive from build to build. # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint - ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/.cache/golangci-lint \ golangci-lint run --config .golangci.yml ./... # Test phase. -race needs cgo and so a C compiler, which the Debian Go @@ -214,10 +229,10 @@ style conventions are in separate documents: # cache also holds test results, which go test would otherwise replay. # golang:1.x, YYYY-MM-DD FROM golang@sha256:... AS test - ARG CACHEBUST WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST COPY . . RUN --mount=type=cache,target=/root/.cache/go-build \ go test -count=1 -timeout 90s -race -cover ./... || \ @@ -225,19 +240,19 @@ style conventions are in separate documents: go test -count=1 -timeout 90s -race -v ./...; exit 1; } # Build stage. Nothing is wanted from either phase above; the copies - # are what make BuildKit build them first, so this stage cannot run - # unless lint and test passed. + # are what make BuildKit build them first, so nothing after them can + # run unless lint and test passed. # golang:1.x-alpine, YYYY-MM-DD FROM golang@sha256:... AS builder - ARG CACHEBUST - COPY --from=lint /src/go.sum /dev/null - COPY --from=test /src/go.sum /dev/null RUN apk add --no-cache git - # A tar-stream context keeps the sender's file owners, which git refuses. - RUN git config --system --add safe.directory /src WORKDIR /src COPY go.mod go.sum ./ RUN go mod download + ARG CACHEBUST + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src COPY . . # The VERSION build arg when one is given, otherwise @@ -274,16 +289,26 @@ style conventions are in separate documents: - Keep the runtime stage last, and if you add a stage after it, give it the same two copies. A plain `docker build .` builds the last stage's chain and nothing else. - - Only Go's build cache is mounted (`/root/.cache/go-build`, where Go keeps - it for root), and only on the steps that compile: `golangci-lint run`, - both `go test` runs and `go build`. It keeps compiled packages between - check builds, and Go reuses one only when all its inputs are unchanged, so - it saves the compile without skipping a check. Nothing else is kept: - golangci-lint's own cache would replay stored findings in place of a lint - run, and Go builds from whatever files a kept module cache holds without - checking them again (only `go mod verify` does), so a file changed there - would go unnoticed. Go's build cache also holds test results, which is why - both `go test` runs carry `-count=1`. + - Check builds keep these, and none of them can hand back a stale result: + - The layers above `ARG CACHEBUST`: the system packages and Go's + modules. Docker installs the packages again when the base image or the + install line changes, and downloads the modules again when `go.mod` or + `go.sum` changes. Go checks every module against `go.sum` as it + downloads it, and no later build can change a kept layer. The modules + are kept in a layer rather than a cache mount because a cache mount is + shared by every build on the host, and Go builds from whatever files + it holds without checking them again (only `go mod verify` does), so a + file changed there would go unnoticed. + - Go's build cache (`/root/.cache/go-build`, where Go keeps it for + root), mounted on the steps that compile: `golangci-lint run`, both + `go test` runs and `go build`. Go reuses a compiled package only when + all its inputs are unchanged. The cache also holds test results, which + is why both `go test` runs carry `-count=1`. + - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned + image keeps it), mounted on `golangci-lint run`. golangci-lint reuses + a package's findings only when the package's files and those of every + package it imports, `go.mod`, the linters and their settings in + `.golangci.yml`, and the golangci-lint version are all unchanged. - If the project uses `//go:embed` directives that reference build artifacts (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: @@ -292,12 +317,20 @@ style conventions are in separate documents: in the lint phase. The `golangci/golangci-lint` image is Debian-based and has no `apk`, so install with `apt-get` under the Debian package name (`libvips-dev`, where alpine says `vips-dev`), and delete the package - lists in the same `RUN`, so the layer does not keep them: + lists in the same `RUN`, so the layer does not keep them. The test phase, + on the Debian Go image, installs the libraries its tests need the same + way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so + Docker keeps it until the base image or the install line changes: ```dockerfile + FROM golangci/golangci-lint@sha256:... AS lint RUN apt-get update \ && apt-get install -y --no-install-recommends libvips-dev \ && rm -rf /var/lib/apt/lists/* + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + ARG CACHEBUST ``` - `.dockerignore` lets `.git` into the build context. It keeps out every git diff --git a/TODO.md b/TODO.md index 284d19d..b8d828b 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,10 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-18 re-vendored from `sneak/prompts` commit `0653579` (closes #221): + each `Dockerfile` stage installs its system packages and downloads the Go + modules above `ARG CACHEBUST`, so check builds keep them, and the lint step + keeps golangci-lint's cache. - 2026-10-19 a `trusted_proxies` entry in the IPv4-mapped form trusted nothing; it is now read as the IPv4 range it covers by the same code that reads `blocked_networks`, and such an entry shorter than `/96` aborts startup diff --git a/script/bootstrap b/script/bootstrap index 8fd0cbe..e3fe598 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -19,9 +19,8 @@ # CGO image libraries (pkg-config, vips # with its JPEG XL support, libheif) for # the govips bindings instead of Node: to -# compile pixa, in the Dockerfile's test -# phase and build stage, which format -# nothing +# compile pixa outside Docker, as +# `make build` does set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/script/cibuild b/script/cibuild index 66fcca0..2656771 100755 --- a/script/cibuild +++ b/script/cibuild @@ -2,11 +2,12 @@ # script/cibuild: run the CI build. It bootstraps first: a CI runner # checks out and runs this and nothing else, and script/fmt-check runs # the formatter on the host, which a pristine checkout cannot do. -# A new CACHEBUST value reruns every RUN step, since a cached gate phase -# is a check that did not run; unlike --no-cache, it keeps Go's build cache. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. # This relies on every stage of the Dockerfile declaring ARG CACHEBUST -# first; a stage without that line is served from cache on an unchanged -# tree, with no warning. +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/docker b/script/docker index 360037a..cc00d9c 100755 --- a/script/docker +++ b/script/docker @@ -1,11 +1,12 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# A new CACHEBUST value reruns every RUN step, since a cached gate phase -# is a check that did not run; unlike --no-cache, it keeps Go's build cache. +# A new CACHEBUST value reruns every RUN step after the ARG CACHEBUST +# line, since a cached gate phase is a check that did not run; unlike +# --no-cache, it keeps the layers above that line and the cache mounts. # This relies on every stage of the Dockerfile declaring ARG CACHEBUST -# first; a stage without that line is served from cache on an unchanged -# tree, with no warning. +# before every step that lints, tests or compiles; a stage without that +# line is served from cache on an unchanged tree, with no warning. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/lint b/script/lint index c4c459e..451a067 100755 --- a/script/lint +++ b/script/lint @@ -5,11 +5,13 @@ # make its answer untrustworthy. # # The phase is not the last stage in the file, so it is built only when -# --target names it. A new CACHEBUST value reruns every RUN step, since -# a cached lint layer is a lint that did not run; unlike --no-cache, it -# keeps Go's build cache. This relies on every stage of the Dockerfile -# declaring ARG CACHEBUST first; a stage without that line is served -# from cache on an unchanged tree, with no warning. +# --target names it. A new CACHEBUST value reruns every RUN step after +# the ARG CACHEBUST line, since a cached lint layer is a lint that did +# not run; unlike --no-cache, it keeps the layers above that line and +# the cache mounts. This relies on every stage of the Dockerfile +# declaring ARG CACHEBUST before every step that lints, tests or +# compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. # --output type=cacheonly writes no image, since nothing uses one. set -eu diff --git a/script/test b/script/test index a56c4a1..1622a11 100755 --- a/script/test +++ b/script/test @@ -3,11 +3,12 @@ # and this builds that phase alone, on the same terms as script/lint: # --target because a phase that is not the last stage is built only when # named, and a new CACHEBUST value because a cached test layer is a test -# that did not run; unlike --no-cache, it keeps Go's build cache. This -# relies on every stage of the Dockerfile declaring ARG CACHEBUST first; -# a stage without that line is served from cache on an unchanged tree, -# with no warning. --output type=cacheonly writes no image, since -# nothing uses one. +# that did not run; unlike --no-cache, it keeps the layers above the ARG +# CACHEBUST line and the cache mounts. This relies on every stage of the +# Dockerfile declaring ARG CACHEBUST before every step that lints, tests +# or compiles; a stage without that line is served from cache on an +# unchanged tree, with no warning. --output type=cacheonly writes no +# image, since nothing uses one. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" From cde9c0d10df0ac7da35c16e29f1b96b3822f7f99 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 14:25:07 +0200 Subject: [PATCH 093/131] Let a bug fix change the test that pins the behavior the bug calls wrong (closes #280) AGENTS.md required the owner's approval for every change to an existing test. A bug fix may now change an existing test without asking when that test pins exactly the behavior the filed bug calls wrong, and the change moves it to the fixed behavior and nothing else. Any other change to an existing test still needs approval. The owner approved this on the issue. Model: opus-5-5 --- AGENTS.md | 6 +++++- TODO.md | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index af2e77c..3933bc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,11 @@ These rules MUST be followed at all times, it is very important. without asking first. In almost all cases, the code should be changed, NOT the tests. If you think the test needs to be changed, make your case for that and ask for permission to proceed, then stop. You need explicit user approval to - modify existing tests. (You do not need user approval for writing NEW tests.) + modify existing tests. A bug fix may change an existing test without asking + when that test pins exactly the behavior the filed bug calls wrong, and the + change moves it to the fixed behavior and nothing else; any other change to an + existing test still needs explicit user approval. (You do not need user + approval for writing NEW tests.) - When linting, assume the linter config is CORRECT, and that each item output by the linter is something that legitimately needs fixing in the code. diff --git a/TODO.md b/TODO.md index b8d828b..650e676 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,9 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-20 `AGENTS.md`: a bug fix may change an existing test without asking + when that test pins exactly the behavior the filed bug calls wrong, and the + change moves it to the fixed behavior and nothing else (closes #280). - 2026-10-18 re-vendored from `sneak/prompts` commit `0653579` (closes #221): each `Dockerfile` stage installs its system packages and downloads the Go modules above `ARG CACHEBUST`, so check builds keep them, and the lint step From 4fb6350aa113cf85c364b636b010ee427bc29aae Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 14:40:51 +0200 Subject: [PATCH 094/131] Answer 415 for a source whose first bytes do not match its Content-Type (closes #253) A body whose first bytes are not those of the format its Content-Type names answered 500 on both image routes, was logged as pixa's own error, and was fetched again on every request. Both routes now answer 415 for each error the first-bytes check returns, and the image service logs the refused source at warning level. The failure also gets a negative cache entry, as an upstream error status does, so the source URL is not fetched again for 5 minutes, and meanwhile an image that would need that fetch answers 502. README.md says so. The case of TestService_Get_ConcurrentMissesShareAFailure that expected such a source to be fetched again now expects the negative cache, as https://git.eeqj.de/sneak/pixa/issues/280#issuecomment-151029 allows. Model: opus-5-5 --- README.md | 17 +- TODO.md | 9 + .../handlers/first_bytes_internal_test.go | 180 ++++++++++++++++++ internal/handlers/image.go | 13 ++ internal/handlers/imageenc.go | 6 +- .../concurrent_misses_internal_test.go | 7 +- internal/imgcache/service.go | 35 +++- 7 files changed, 247 insertions(+), 20 deletions(-) create mode 100644 internal/handlers/first_bytes_internal_test.go diff --git a/README.md b/README.md index 2e0f3d2..697e071 100644 --- a/README.md +++ b/README.md @@ -247,9 +247,11 @@ path under `/v1/` answers 200, in maintenance mode too. redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an address in a blocked network (see `blocked_networks`); 429 over `image_requests_per_minute` (below); 415 when the upstream answers with a - `Content-Type` that is not one of the source formats (below), or with an image - whose first bytes libvips reads as another format; 502 when the upstream - answered with an error status, and for 5 minutes after that for an image of + `Content-Type` that is not one of the source formats (below), with a body + whose first bytes are not those of the format its `Content-Type` names, or + with an image whose first bytes libvips reads as another format; 502 when the + upstream answered with an error status, and for 5 minutes after that, or after + a body whose first bytes are not those of its `Content-Type`, for an image of the same source URL that pixa can neither serve from its cache nor make from the source image still in its cache; 503 when pixa is busy or in maintenance mode; 500 for any other failure. @@ -358,8 +360,13 @@ above 8192. The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's `Content-Type` must name its format, and the image's first bytes must match it. An upstream answering with any other `Content-Type`, SVG included, is refused -with 415, and so is an image whose first bytes libvips reads as another format, -such as an SVG whose first bytes pass the check for AVIF. +with 415. So is a body whose first bytes are not those of the format its +`Content-Type` names, such as a JPEG sent as `image/png`, text sent as an image, +or an empty body. pixa then does not fetch that source URL again for 5 minutes, +as after an upstream error status, and meanwhile answers 502 for an image that +would need that fetch. An image whose first bytes libvips reads as another +format, such as an SVG whose first bytes pass the check for AVIF, is refused +with 415 too. With the format `auto`, pixa chooses the format for each request from its `Accept` header, in this order: diff --git a/TODO.md b/TODO.md index 650e676..25d5537 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-20 a source whose first bytes do not match its `Content-Type` answers + 415 (closes #253): a body whose first bytes are not those of the format its + `Content-Type` names, such as a JPEG sent as `image/png`, text, or a body + under 12 bytes, answered 500 on both image routes and was logged as an error + of pixa's, and each request for it fetched the whole body again. Both routes + now answer 415, and the image service logs it at warning level as a refused + source. It also gets a negative cache entry, as an upstream error status does, + so the source URL is not fetched again for 5 minutes, and meanwhile an image + that would need that fetch answers 502. - 2026-10-20 `AGENTS.md`: a bug fix may change an existing test without asking when that test pins exactly the behavior the filed bug calls wrong, and the change moves it to the fixed behavior and nothing else (closes #280). diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go new file mode 100644 index 0000000..fb3ba30 --- /dev/null +++ b/internal/handlers/first_bytes_internal_test.go @@ -0,0 +1,180 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "image/color" + "log/slog" + "net/http" + "slices" + "testing" + "testing/fstest" + "time" + + "github.com/go-chi/chi/v5" + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// mismatchedSourcePath is the path of the source that +// TestFirstBytesNotMatchingContentType serves on allowlistedHost. The mock +// fetcher names a file's Content-Type from its extension, so the source is +// sent as image/png. +const mismatchedSourcePath = "/images/source.png" + +// newRoutesFetchingThrough returns both image routes, and the Handlers behind +// them, whose image service fetches through fetcher and has allowlistedHost on +// its allowlist. The handlers and the image service log to log. +func newRoutesFetchingThrough( + t *testing.T, fetcher httpfetcher.Fetcher, log *slog.Logger, +) (*Handlers, http.Handler) { + t.Helper() + + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + StateDir: t.TempDir(), + NegativeTTL: 5 * time.Minute, + }) + if err != nil { + t.Fatalf("imgcache.NewCache() error = %v", err) + } + + svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + Cache: cache, + Fetcher: fetcher, + SigningKey: testSigningKey, + Allowlist: []string{allowlistedHost}, + Logger: log, + }) + if err != nil { + t.Fatalf("imgcache.NewService() error = %v", err) + } + + encGen, err := encurl.NewGenerator(testSigningKey) + if err != nil { + t.Fatalf("encurl.NewGenerator() error = %v", err) + } + + h := &Handlers{log: log, imgSvc: svc, encGen: encGen} + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + r.Get("/v1/e/{token}/*", h.HandleImageEnc()) + + return h, r +} + +// mismatchedSourcePlainURL returns the /v1/image/ URL of mismatchedSourcePath +// on allowlistedHost at 50x50 as a JPEG. +func mismatchedSourcePlainURL(*testing.T, *Handlers) string { + return "/v1/image/" + allowlistedHost + mismatchedSourcePath + "/50x50.jpeg" +} + +// mismatchedSourceEncURL returns an encrypted /v1/e/ URL, made with h's +// generator, of mismatchedSourcePath on allowlistedHost at 50x50 as a JPEG. +func mismatchedSourceEncURL(t *testing.T, h *Handlers) string { + t.Helper() + + token, err := h.encGen.Generate(&encurl.Payload{ + SourceHost: allowlistedHost, + SourcePath: mismatchedSourcePath, + Width: 50, + Height: 50, + Format: imgcache.FormatJPEG, + }) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + return "/v1/e/" + token + "/img.jpg" +} + +// loggedLevels reads the JSON log lines in logs and returns the messages +// logged at each level. +func loggedLevels(t *testing.T, logs *bytes.Buffer) map[string][]string { + t.Helper() + + t.Logf("logged:\n%s", logs.String()) + + messages := make(map[string][]string) + + dec := json.NewDecoder(logs) + for dec.More() { + var line struct { + Level string `json:"level"` + Msg string `json:"msg"` + } + + err := dec.Decode(&line) + if err != nil { + t.Fatalf("decoding log line: %v", err) + } + + messages[line.Level] = append(messages[line.Level], line.Msg) + } + + return messages +} + +// TestFirstBytesNotMatchingContentType checks that both image routes answer +// 415 for an upstream body whose first bytes are not those of the format its +// Content-Type names, and log it as a refused source, not as an error of +// pixa's. The next request for that source must answer 502 without fetching +// it again, as after an upstream error status. +func TestFirstBytesNotMatchingContentType(t *testing.T) { + t.Parallel() + + bodies := map[string][]byte{ + "JPEG sent as image/png": generateTestJPEG(t, 10, 10, color.Black), + "text sent as image/png": []byte("This is just text, not a PNG"), + "body too short to name a format": []byte("short"), + } + + // The source's URL on each image route. + sourceURLs := map[string]func(*testing.T, *Handlers) string{ + "plain URL": mismatchedSourcePlainURL, + "encrypted URL": mismatchedSourceEncURL, + } + + for bodyName, body := range bodies { + for urlName, sourceURL := range sourceURLs { + t.Run(urlName+", "+bodyName, func(t *testing.T) { + t.Parallel() + + fetcher := &countingFetcher{ + Fetcher: httpfetcher.NewMock(fstest.MapFS{ + allowlistedHost + mismatchedSourcePath: &fstest.MapFile{ + Data: body, + }, + }), + } + + var logs bytes.Buffer + + h, routes := newRoutesFetchingThrough(t, fetcher, + slog.New(slog.NewJSONHandler(&logs, nil))) + target := sourceURL(t, h) + + checkErrorBody(t, sendGet(t, routes, target), + http.StatusUnsupportedMediaType, "unsupported source format") + + levels := loggedLevels(t, &logs) + if len(levels[slog.LevelError.String()]) != 0 { + t.Errorf("logged at error level: %q", + levels[slog.LevelError.String()]) + } + + if !slices.Contains(levels[slog.LevelWarn.String()], "source refused") { + t.Error(`no "source refused" line logged at warning level`) + } + + checkErrorBody(t, sendGet(t, routes, target), + http.StatusBadGateway, "upstream error") + + if n := fetcher.fetches.Load(); n != 1 { + t.Errorf("upstream fetched %d times, want 1", n) + } + }) + } + } +} diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 96dad32..ad7119d 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/magic" ) // HandleImage handles the main image proxy route: @@ -220,6 +221,18 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc { func (s *Handlers) respondImageError( w http.ResponseWriter, req *imgcache.ImageRequest, err error, ) { + // An upstream body whose first bytes are not those of the format its + // Content-Type names is the upstream's error, not pixa's: the image + // service has logged it as a refused source. + if errors.Is(err, magic.ErrMagicByteMismatch) || + errors.Is(err, magic.ErrUnknownFormat) || + errors.Is(err, magic.ErrNotEnoughData) { + s.respondError(w, "unsupported source format", + http.StatusUnsupportedMediaType) + + return + } + s.log.Error("failed to get image", "host", req.SourceHost, "path", req.SourcePath, diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 4da08eb..92df798 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -15,6 +15,7 @@ import ( "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/magic" ) // HandleImageEnc handles requests to /v1/e/{token}/* for encrypted @@ -163,7 +164,10 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { s.respondError(w, "server busy, try again later", http.StatusServiceUnavailable) case errors.Is(err, httpfetcher.ErrInvalidContentType), - errors.Is(err, imageprocessor.ErrUnsupportedInputFormat): + errors.Is(err, imageprocessor.ErrUnsupportedInputFormat), + errors.Is(err, magic.ErrMagicByteMismatch), + errors.Is(err, magic.ErrUnknownFormat), + errors.Is(err, magic.ErrNotEnoughData): s.respondError(w, "unsupported source format", http.StatusUnsupportedMediaType) case errors.Is(err, imageprocessor.ErrOutputTooLarge): diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index b81d63a..d763b99 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -331,8 +331,7 @@ func TestService_Get_FirstRequestLeavingKeepsTheWork(t *testing.T) { // TestService_Get_ConcurrentMissesShareAFailure has several requests for an // image that cannot be served arrive while its fetch is held: the one fetch // answers all of them with its error. The request after them is answered from -// the negative cache when the failure is kept there, and fetches again when it -// is not. +// the negative cache, where the failure is kept. func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { t.Parallel() @@ -346,8 +345,8 @@ func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { {"upstream answers 404, kept in the negative cache", "/images/missing.jpg", httpfetcher.ErrUpstreamError, ErrNegativeCached, 1}, - {"source fails the magic byte check, not kept", - "/images/text.png", magic.ErrUnknownFormat, magic.ErrUnknownFormat, 2}, + {"source fails the magic byte check, kept in the negative cache", + "/images/text.png", magic.ErrUnknownFormat, ErrNegativeCached, 1}, } for _, tc := range tests { diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 212e479..ced31b4 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -458,13 +458,7 @@ func (s *Service) fetchAndProcess( if err != nil { // Store negative cache for certain errors if isNegativeCacheable(err) { - statusCode := extractStatusCode(err) - - storeErr := s.cache.StoreNegative(ctx, req, statusCode, err.Error()) - if storeErr != nil { - s.log.Warn("failed to store negative cache entry", - "host", req.SourceHost, "path", req.SourcePath, "error", storeErr) - } + s.storeNegative(ctx, req, extractStatusCode(err), err) } return nil, fmt.Errorf("upstream fetch failed: %w", err) @@ -510,9 +504,16 @@ func (s *Service) fetchAndProcess( "cipher", fetchResult.TLSCipherSuite, ) - // Validate magic bytes match content type + // Validate magic bytes match content type. A body that does not is the + // upstream's error, so it gets a negative cache entry, as an upstream + // error status does. err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) if err != nil { + s.log.Warn("source refused", "request_id", middleware.GetReqID(ctx), + "host", req.SourceHost, "path", req.SourcePath, + "content_type", fetchResult.ContentType, "error", err) + s.storeNegative(ctx, req, httpStatusUnsupportedMediaType, err) + return nil, fmt.Errorf("content validation failed: %w", err) } @@ -528,6 +529,19 @@ func (s *Service) fetchAndProcess( ) } +// storeNegative stores a negative cache entry for req's source, whose fetch +// failed with err, so the source is not fetched again until the entry +// expires. A failure to store the entry is only logged. +func (s *Service) storeNegative( + ctx context.Context, req *ImageRequest, statusCode int, err error, +) { + storeErr := s.cache.StoreNegative(ctx, req, statusCode, err.Error()) + if storeErr != nil { + s.log.Warn("failed to store negative cache entry", + "host", req.SourceHost, "path", req.SourcePath, "error", storeErr) + } +} + // processedVariant is a variant as processAndStore made it. Each request that // shared its processing serves it through a reader of its own. type processedVariant struct { @@ -642,8 +656,9 @@ func signatureRequest(req *ImageRequest) *signature.Request { // HTTP status codes for error responses. const ( - httpStatusBadGateway = 502 - httpStatusInternalError = 500 + httpStatusBadGateway = 502 + httpStatusInternalError = 500 + httpStatusUnsupportedMediaType = 415 ) // isNegativeCacheable returns true if the error should be cached. From 989da786814390cb2625094587d28adebd579afa Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 15:19:07 +0200 Subject: [PATCH 095/131] Keep WebP, AVIF and JPEG XL encodes at the limits under 40 seconds (closes #272) WebP is saved at effort 2 and JPEG XL at effort 4, and an AVIF output of more than 16,777,216 pixels is refused with 400 before anything is decoded, as AVIF is already at the lowest effort govips can set. With one libvips thread, at any quality, 8192x8192 random pixels with an alpha channel take up to about 23 seconds as WebP and 27 as JPEG XL, and 4096x4096, 8192x2048 or 2048x8192 up to about 34 as AVIF. Resizing with fit=cover already decodes the input, so the output's dimensions are now worked out before resizing, in outputDimensions. Reading the input moves into readInput to keep Process within the linter's limits. TestImageProcessor_WebPAtDefaultEffort becomes TestImageProcessor_WebPAtEffort2, as https://git.eeqj.de/sneak/pixa/issues/280#issuecomment-151029 allows. Model: opus-5-5 --- README.md | 40 +-- TODO.md | 12 + .../handlers/avif_too_large_internal_test.go | 49 ++++ internal/handlers/image.go | 8 + internal/handlers/imageenc.go | 4 + .../avif_too_large_internal_test.go | 84 +++++++ .../encoder_settings_internal_test.go | 38 ++- internal/imageprocessor/imageprocessor.go | 227 +++++++++++------- 8 files changed, 360 insertions(+), 102 deletions(-) create mode 100644 internal/handlers/avif_too_large_internal_test.go create mode 100644 internal/imageprocessor/avif_too_large_internal_test.go diff --git a/README.md b/README.md index 697e071..9d1e2c4 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,12 @@ either side may be; so is a size with `fit=outside`, which scales the image to cover the size and so past it on one side, when that side would come out above 8192. +An AVIF image of more than 16,777,216 pixels, the number 4096x4096 and 8192x2048 +have, is refused with 400 before anything is decoded, as its encode could take +more than 40 seconds (see below). That includes `orig` of an AVIF source, and +`auto` when it chooses AVIF, so a client whose `Accept` names `image/avif` but +not `image/jxl` is refused such a size. + The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's `Content-Type` must name its format, and the image's first bytes must match it. An upstream answering with any other `Content-Type`, SVG included, is refused @@ -411,14 +417,18 @@ transcode that stops this way is answered 503 with the error ended. govips gives pixa no way to stop libvips once an encode has begun, so an encode keeps its slot until it finishes, which can be after `downstream_timeout`. The limit of 8192 on each side of the source and of the -output bounds how long that takes. With one libvips thread, as pixad runs it, -encoding 8192x8192 random pixels takes up to about 1 second as JPEG, 15 as PNG, -16 as GIF, 63 as WebP, 110 as AVIF and 96 as JPEG XL, the slowest of 8 or 16 -bits per sample, with or without an alpha channel. WebP, AVIF and JPEG XL are -that slow only with an alpha channel: 8-bit RGB takes about 44, 53 and 28 -seconds. Decoding such a source adds up to about 11 seconds. A WebP, AVIF or -JPEG XL encode can therefore hold its slot past the default `downstream_timeout` -of 60 seconds. +output, and for AVIF the limit of 16,777,216 pixels, bound how long that takes. +With one libvips thread, as pixad runs it, encoding 8192x8192 random pixels +takes up to about 1 second as JPEG, 15 as PNG and 16 as GIF, the slowest of 8 or +16 bits per sample, with or without an alpha channel. The WebP and JPEG XL +encoders run below libvips' default effort, and AVIF at the lowest effort govips +can set, so that no encode takes more than 40 seconds of the default +`downstream_timeout` of 60: at any quality and 8 or 16 bits per sample, +8192x8192 random pixels with an alpha channel take up to about 23 seconds as +WebP and 27 as JPEG XL, and 4096x4096, 8192x2048 or 2048x8192 up to about 34 as +AVIF, which at quality 85 would take about 96 at 8192x8192. Without an alpha +channel they take up to about 16, 13 and 22 seconds. Decoding such a source adds +up to about 11 seconds. The login form (`POST /`) is limited to 5 attempts per minute per client address, counting an IPv6 client by its /64; an attempt over the limit is @@ -470,13 +480,13 @@ nor change what it asks for. sent with an empty format, or none, also makes a JPEG XL URL. Width and height both empty or `0` keep the original size; if only one of them is empty or `0`, that side is scaled to keep the image's proportions. A URL whose - scaled side would come out above 8192 answers 400, and so does a source URL - whose path holds a `..` segment. So does a source URL that the image URL - would fetch written another way, and the page then shows the URL pixa would - fetch: one with a user name and password, which pixa does not send, or one - whose path holds `%2F`, fetched as `/`, or `(`, fetched as `%28`, as pixa - keeps the source path decoded and fetches it percent-encoded again (see - Routes). + scaled side would come out above 8192 answers 400, and so does an AVIF URL of + more than 16,777,216 pixels or a source URL whose path holds a `..` segment. + So does a source URL that the image URL would fetch written another way, and + the page then shows the URL pixa would fetch: one with a user name and + password, which pixa does not send, or one whose path holds `%2F`, fetched as + `/`, or `(`, fetched as `%28`, as pixa keeps the source path decoded and + fetches it percent-encoded again (see Routes). 3. The page shows the URL, `https:///v1/e//img.`, and when it expires. `` is the host the page was opened on, and the URL starts with `http` instead while `debug` is on. The name after the token is ignored diff --git a/TODO.md b/TODO.md index 25d5537..fb2cde2 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,18 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-20 a WebP, AVIF or JPEG XL encode at the limits takes at most about 34 + seconds (closes #272): with one libvips thread, 8192x8192 random pixels with + an alpha channel took about 61 seconds to encode as WebP, 81 as JPEG XL and, + at quality 85, 96 as AVIF, so with the default `downstream_timeout` of 60 + seconds such an image was never delivered, yet its encode held a processing + slot to the end. WebP is now saved at effort 2 and JPEG XL at effort 4, which + take up to about 23 and 27 seconds at any quality. AVIF was already at the + lowest effort govips can set, so an AVIF output of more than 16,777,216 pixels + is refused with 400 before anything is decoded; 4096x4096, 8192x2048 or + 2048x8192 takes up to about 34 seconds at any quality. Resizing with + `fit=cover` already decodes the input, so the output's dimensions are worked + out before resizing. `README.md` gives the limit and the new times. - 2026-10-20 a source whose first bytes do not match its `Content-Type` answers 415 (closes #253): a body whose first bytes are not those of the format its `Content-Type` names, such as a JPEG sent as `image/png`, text, or a body diff --git a/internal/handlers/avif_too_large_internal_test.go b/internal/handlers/avif_too_large_internal_test.go new file mode 100644 index 0000000..c634a03 --- /dev/null +++ b/internal/handlers/avif_too_large_internal_test.go @@ -0,0 +1,49 @@ +package handlers + +import ( + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestAVIFOutputTooLargeAnswers400 checks that respondImageError and +// handleImageError, which answer the errors of the two image routes, answer +// 400 naming the AVIF pixel limit when the image processor refuses an AVIF +// output above it, wrapped as the service wraps it. The request's source only +// goes into a log line, so it is left empty. +func TestAVIFOutputTooLargeAnswers400(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + err := fmt.Errorf("image processing failed: %w: 4097x4096", + imageprocessor.ErrAVIFOutputTooLarge) + wantBody := fmt.Sprintf("above %d pixels", imageprocessor.MaxAVIFOutputPixels) + + plain := httptest.NewRecorder() + h.respondImageError(plain, &imgcache.ImageRequest{}, err) + + encrypted := httptest.NewRecorder() + h.handleImageError(encrypted, err) + + for name, rec := range map[string]*httptest.ResponseRecorder{ + "respondImageError": plain, "handleImageError": encrypted, + } { + t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String()) + + if rec.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want %d", + name, rec.Code, http.StatusBadRequest) + } + + if !strings.Contains(rec.Body.String(), wantBody) { + t.Errorf("%s: body = %q, want it to say %q", + name, rec.Body.String(), wantBody) + } + } +} diff --git a/internal/handlers/image.go b/internal/handlers/image.go index ad7119d..f1a6ff0 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -277,6 +277,14 @@ func (s *Handlers) respondImageError( return } + if errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge) { + s.respondError(w, fmt.Sprintf("size too large for AVIF: the output "+ + "would be above %d pixels", imageprocessor.MaxAVIFOutputPixels), + http.StatusBadRequest) + + return + } + s.respondError(w, "internal error", http.StatusInternalServerError) } diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index 92df798..eefb74f 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -174,6 +174,10 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { s.respondError(w, fmt.Sprintf("size too large for the source: the side "+ "computed from its proportions is above %d", imageprocessor.MaxOutputDimension), http.StatusBadRequest) + case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge): + s.respondError(w, fmt.Sprintf("size too large for AVIF: the output "+ + "would be above %d pixels", imageprocessor.MaxAVIFOutputPixels), + http.StatusBadRequest) default: s.log.Error("image request failed", "error", err) s.respondError(w, "internal error", http.StatusInternalServerError) diff --git a/internal/imageprocessor/avif_too_large_internal_test.go b/internal/imageprocessor/avif_too_large_internal_test.go new file mode 100644 index 0000000..2770dd4 --- /dev/null +++ b/internal/imageprocessor/avif_too_large_internal_test.go @@ -0,0 +1,84 @@ +package imageprocessor + +import ( + "bytes" + "encoding/binary" + "errors" + "os" + "testing" +) + +// TestImageProcessor_RefusesAVIFAboveMaxPixels verifies that an AVIF output +// of more than MaxAVIFOutputPixels is refused with ErrAVIFOutputTooLarge +// before anything is decoded, whatever size and fit give it, the source's own +// size included, and that an AVIF output of at most that many pixels, or one +// in another format, is not. One input is a 64x32 PNG cut off in its pixel +// data. The other is testdata/red.avif, a 4x4 AVIF, with the size in its +// header changed to 4097x4096. libvips reads each input's size from its +// header but fails when it decodes its pixel data, so a request that gets +// that failure was not refused, and a refusal came before any decoding. +func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) { + t.Parallel() + + whole := createTestPNG(t, 64, 32) + cutOff := whole[:len(whole)/2] + + largeAVIF, err := os.ReadFile("testdata/red.avif") + if err != nil { + t.Fatalf("failed to read test AVIF: %v", err) + } + + // The ispe box gives the width and then the height, 4 bytes each, after + // the box's type and its 4 bytes of version and flags + ispe := bytes.Index(largeAVIF, []byte("ispe")) + binary.BigEndian.PutUint32(largeAVIF[ispe+8:], 4097) + binary.BigEndian.PutUint32(largeAVIF[ispe+12:], 4096) + + tests := []struct { + name string + input []byte + format Format + width, height int + fit FitMode + refused bool + }{ + {"AVIF at the limit", cutOff, FormatAVIF, 4096, 4096, FitCover, false}, + {"AVIF above the limit", cutOff, FormatAVIF, 4096, 4097, FitCover, true}, + {"WebP above the AVIF limit", cutOff, FormatWebP, 4096, 4097, FitCover, false}, + // FitInside does not enlarge, so the output stays 64x32 + {"AVIF inside a larger size", cutOff, FormatAVIF, 8192, 8192, FitInside, false}, + // FitOutside scales 64x32 by 128 to cover 8192x1: 8192x4096 + {"AVIF outside a smaller size", cutOff, FormatAVIF, 8192, 1, FitOutside, true}, + // With no size, the output keeps the source's 4097x4096 + {"AVIF of the source's size", largeAVIF, FormatAVIF, 0, 0, FitCover, true}, + {"orig of an AVIF source", largeAVIF, FormatOriginal, 0, 0, FitCover, true}, + {"JPEG of the source's size", largeAVIF, FormatJPEG, 0, 0, FitCover, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := New(Params{}).Process(t.Context(), bytes.NewReader(tt.input), + &Request{ + Size: Size{Width: tt.width, Height: tt.height}, + Format: tt.format, + FitMode: tt.fit, + }) + t.Logf("Process() error = %v", err) + + if tt.refused { + if !errors.Is(err, ErrAVIFOutputTooLarge) { + t.Errorf("Process() error = %v, want %v", err, ErrAVIFOutputTooLarge) + } + + return + } + + if err == nil || errors.Is(err, ErrAVIFOutputTooLarge) { + t.Errorf("Process() error = %v, want the failure to decode "+ + "the input", err) + } + }) + } +} diff --git a/internal/imageprocessor/encoder_settings_internal_test.go b/internal/imageprocessor/encoder_settings_internal_test.go index 90634ce..af1b31d 100644 --- a/internal/imageprocessor/encoder_settings_internal_test.go +++ b/internal/imageprocessor/encoder_settings_internal_test.go @@ -77,10 +77,10 @@ func TestImageProcessor_PNGIsCompressed(t *testing.T) { } } -// TestImageProcessor_WebPAtDefaultEffort verifies that WebP is saved at -// libvips' default effort, 4: the output is the size of the same image saved -// at that effort. -func TestImageProcessor_WebPAtDefaultEffort(t *testing.T) { +// TestImageProcessor_WebPAtEffort2 verifies that WebP is saved at effort 2, +// not libvips' default of 4: the output is the size of the same image saved +// at effort 2. +func TestImageProcessor_WebPAtEffort2(t *testing.T) { t.Parallel() input := createTestJPEG(t, 200, 150) @@ -90,14 +90,14 @@ func TestImageProcessor_WebPAtDefaultEffort(t *testing.T) { want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{ StripMetadata: true, Quality: 85, - ReductionEffort: 4, + ReductionEffort: 2, }) if err != nil { t.Fatalf("ExportWebp() error = %v", err) } if size != int64(len(want)) { - t.Errorf("WebP output is %d bytes, want %d, the size at effort 4", + t.Errorf("WebP output is %d bytes, want %d, the size at effort 2", size, len(want)) } } @@ -143,3 +143,29 @@ func TestImageProcessor_AVIFAtEffort1(t *testing.T) { "and 8 bits", size, len(want)) } } + +// TestImageProcessor_JPEGXLAtEffort4 verifies that JPEG XL is saved at +// effort 4, not libvips' default of 7: the output is the size of the same +// image saved at effort 4, with the distance exportJXL gives quality 85. +func TestImageProcessor_JPEGXLAtEffort4(t *testing.T) { + t.Parallel() + + input := createTestJPEG(t, 200, 150) + + size := processedSize(t, input, &Request{Format: FormatJXL, Quality: 85}) + + params := vips.NewJxlExportParams() + params.Quality = 85 + params.Distance = jxlDistance(85) + params.Effort = 4 + + want, _, err := decode(t, input).ExportJxl(params) + if err != nil { + t.Fatalf("ExportJxl() error = %v", err) + } + + if size != int64(len(want)) { + t.Errorf("JPEG XL output is %d bytes, want %d, the size at effort 4", + size, len(want)) + } +} diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 7b83a41..c50452b 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -142,6 +142,19 @@ const MaxOutputDimension = 8192 // requested size. var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum") +// MaxAVIFOutputPixels is the most pixels an AVIF output may have, so that an +// encode takes at most 40 seconds of the default downstream_timeout of 60 +// seconds; AVIF is already at the lowest effort govips can set (avifEffort). +// With one thread, as pixad runs libvips, saving 4096x4096, 8192x2048 or +// 2048x8192 random pixels with an alpha channel takes up to about 34 +// seconds, the slowest at any quality and at 8 or 16 bits per sample, and +// 5120x4096 at 8 bits already takes about 36. +const MaxAVIFOutputPixels = 4096 * 4096 + +// ErrAVIFOutputTooLarge is returned, before anything is decoded, when an AVIF +// output would have more than MaxAVIFOutputPixels pixels. +var ErrAVIFOutputTooLarge = errors.New("AVIF output has too many pixels") + // ErrUnsupportedOutputFormat is returned when the requested output format is // not supported. var ErrUnsupportedOutputFormat = errors.New("unsupported output format") @@ -210,7 +223,8 @@ func New(params Params) *ImageProcessor { // ErrTooManyImages. When ctx has ended once Process has a slot, or before it // encodes, it returns ctx's error and gives the slot back. govips gives pixa // no way to stop libvips once an encode has begun, so an encode keeps its -// slot until it finishes. +// slot until it finishes. An AVIF output of more than MaxAVIFOutputPixels is +// refused with ErrAVIFOutputTooLarge before anything is decoded. func (p *ImageProcessor) Process( ctx context.Context, input io.Reader, @@ -223,18 +237,9 @@ func (p *ImageProcessor) Process( defer release() - // Read input with a size limit to prevent unbounded memory consumption. - // We read at most maxInputBytes+1 so we can detect if the input exceeds - // the limit without consuming additional memory. - limited := io.LimitReader(input, p.maxInputBytes+1) - - data, err := io.ReadAll(limited) + data, err := p.readInput(input) if err != nil { - return nil, fmt.Errorf("failed to read input: %w", err) - } - - if int64(len(data)) > p.maxInputBytes { - return nil, ErrInputDataTooLarge + return nil, err } // govips picks the libvips loader from the input's first bytes with @@ -277,20 +282,28 @@ func (p *ImageProcessor) Process( return nil, err } - // Resize if needed - if targetWidth != origWidth || targetHeight != origHeight { - err := p.resize(img, targetWidth, targetHeight, req.FitMode) - if err != nil { - return nil, fmt.Errorf("failed to resize: %w", err) - } - } - // orig is the source's own format; encode refuses an empty format outputFormat := req.Format if outputFormat == FormatOriginal { outputFormat = sourceFormat } + // Determine output dimensions; too large ones are refused before + // resizing, which with FitCover already decodes the input + outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight, + targetWidth, targetHeight, req.FitMode, outputFormat) + if err != nil { + return nil, err + } + + // Resize if needed + if targetWidth != origWidth || targetHeight != origHeight { + err := p.resize(img, outputWidth, outputHeight, req.FitMode) + if err != nil { + return nil, fmt.Errorf("failed to resize: %w", err) + } + } + // Encode to target format output, err := p.encode(ctx, img, outputFormat, req.Quality) if err != nil { @@ -445,6 +458,23 @@ func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) { return release, nil } +// readInput reads input with a size limit to prevent unbounded memory +// consumption, refusing more than maxInputBytes with ErrInputDataTooLarge. It +// reads at most maxInputBytes+1 so it can detect if the input exceeds the +// limit without consuming additional memory. +func (p *ImageProcessor) readInput(input io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(input, p.maxInputBytes+1)) + if err != nil { + return nil, fmt.Errorf("failed to read input: %w", err) + } + + if int64(len(data)) > p.maxInputBytes { + return nil, ErrInputDataTooLarge + } + + return data, nil +} + // detectFormat returns the format string for a govips image type. func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { switch imageType { @@ -469,9 +499,67 @@ func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { } } -// resize resizes the image according to the fit mode. With FitOutside one -// side can come out larger than requested; when it would be above -// MaxOutputDimension, resize returns ErrOutputTooLarge without resizing. +// outputDimensions returns the dimensions resize gives an image of imgWidth x +// imgHeight for the target dimensions and fit mode, without resizing it, and +// refuses an output too large for format. With FitOutside one side can come +// out larger than the target; when it would be above MaxOutputDimension, +// outputDimensions returns ErrOutputTooLarge. An AVIF output of more than +// MaxAVIFOutputPixels is ErrAVIFOutputTooLarge. +func outputDimensions( + imgWidth, imgHeight, width, height int, fit FitMode, format Format, +) (int, int, error) { + scaleW := float64(width) / float64(imgWidth) + scaleH := float64(height) / float64(imgHeight) + outputWidth, outputHeight := width, height + + switch fit { + case FitCover, FitFill, "": + // Cover crops, and fill stretches, to exactly the target dimensions + + case FitContain: + // Fit within the target dimensions, maintaining aspect ratio. Scaled + // by the smaller scale, the short side of a thin image can round + // down to 0, which libvips refuses as a width, so each side is at + // least 1. + scale := min(scaleW, scaleH) + outputWidth = max(1, int(float64(imgWidth)*scale)) + outputHeight = max(1, int(float64(imgHeight)*scale)) + + case FitInside: + // Same as contain, but only shrink + scale := min(scaleW, scaleH, 1) + outputWidth = max(1, int(float64(imgWidth)*scale)) + outputHeight = max(1, int(float64(imgHeight)*scale)) + + case FitOutside: + // The smallest dimension fits, the other may exceed the target. A + // side scaled to 1 can come out just under 1 in floating point, as + // 49 times 1/49 does, and round down to 0, which libvips refuses as + // a width, so each side is at least 1. + scale := max(scaleW, scaleH) + outputWidth = max(1, int(float64(imgWidth)*scale)) + outputHeight = max(1, int(float64(imgHeight)*scale)) + + default: + return 0, 0, fmt.Errorf("%w: %s", ErrInvalidFitMode, fit) + } + + // The side FitOutside scales past the target is held to the maximum too + if max(outputWidth, outputHeight) > MaxOutputDimension { + return 0, 0, fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge, + outputWidth, outputHeight, MaxOutputDimension) + } + + if format == FormatAVIF && outputWidth*outputHeight > MaxAVIFOutputPixels { + return 0, 0, fmt.Errorf("%w: %dx%d is above %d pixels", + ErrAVIFOutputTooLarge, outputWidth, outputHeight, MaxAVIFOutputPixels) + } + + return outputWidth, outputHeight, nil +} + +// resize resizes the image to width x height, the dimensions +// outputDimensions gives for the fit mode. func (p *ImageProcessor) resize( img *vips.ImageRef, width, height int, fit FitMode, ) error { @@ -480,58 +568,20 @@ func (p *ImageProcessor) resize( // Crop the centre and resize to fill exact dimensions (default) return cropCentreAndResize(img, width, height) - case FitContain: - // Resize to fit within dimensions, maintaining aspect ratio. Scaled - // by the smaller scale, the short side of a thin image can round - // down to 0, which libvips refuses as a width, so each side is at - // least 1. - imgW, imgH := img.Width(), img.Height() - scaleW := float64(width) / float64(imgW) - scaleH := float64(height) / float64(imgH) - scale := min(scaleW, scaleH) - newW := max(1, int(float64(imgW)*scale)) - newH := max(1, int(float64(imgH)*scale)) - - return img.Thumbnail(newW, newH, vips.InterestingNone) - case FitFill: // Resize to exact dimensions (may distort) return img.ThumbnailWithSize(width, height, vips.InterestingNone, vips.SizeForce) case FitInside: - // Same as contain, but only shrink - if img.Width() <= width && img.Height() <= height { - return nil // Already fits + // outputDimensions keeps the dimensions of an image that already fits + if img.Width() == width && img.Height() == height { + return nil } - imgW, imgH := img.Width(), img.Height() - scaleW := float64(width) / float64(imgW) - scaleH := float64(height) / float64(imgH) - scale := min(scaleW, scaleH) - newW := max(1, int(float64(imgW)*scale)) - newH := max(1, int(float64(imgH)*scale)) + return img.Thumbnail(width, height, vips.InterestingNone) - return img.Thumbnail(newW, newH, vips.InterestingNone) - - case FitOutside: - // Resize so smallest dimension fits, may exceed target on other - // dimension. A side scaled to 1 can come out just under 1 in - // floating point, as 49 times 1/49 does, and round down to 0, which - // libvips refuses as a width, so each side is at least 1. - imgW, imgH := img.Width(), img.Height() - scaleW := float64(width) / float64(imgW) - scaleH := float64(height) / float64(imgH) - scale := max(scaleW, scaleH) - newW := max(1, int(float64(imgW)*scale)) - newH := max(1, int(float64(imgH)*scale)) - - // The dimension that exceeds the target is held to the maximum too - if newW > MaxOutputDimension || newH > MaxOutputDimension { - return fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge, - newW, newH, MaxOutputDimension) - } - - return img.Thumbnail(newW, newH, vips.InterestingNone) + case FitContain, FitOutside: + return img.Thumbnail(width, height, vips.InterestingNone) default: return fmt.Errorf("%w: %s", ErrInvalidFitMode, fit) @@ -667,11 +717,16 @@ func exportGIF(img *vips.ImageRef) ([]byte, error) { return output, err } -// webpEffort is libvips' default WebP effort, from 0 (fastest) to 6. -const webpEffort = 4 +// webpEffort is the WebP effort, from 0 (fastest) to 6. It is below libvips' +// default of 4 so that an encode takes at most 40 seconds of the default +// downstream_timeout of 60 seconds. With one thread, as pixad runs libvips, +// saving an 8192x8192 image of random pixels with an alpha channel takes up +// to about 23 seconds at 2, and up to about 16 without one, at any quality; +// with an alpha channel, 3 takes about 50 seconds and 4 about 61. +const webpEffort = 2 -// exportWebP encodes img as lossy WebP at quality and libvips' default -// effort, without metadata. +// exportWebP encodes img as lossy WebP at quality and webpEffort, without +// metadata. func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) { output, _, err := img.ExportWebp(&vips.WebpExportParams{ StripMetadata: true, @@ -683,12 +738,13 @@ func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) { } // avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest -// govips can set. With one thread, as pixad runs libvips, 1 takes about 51 -// seconds to save an 8192x8192 RGB image of random pixels, and about 110 -// with an alpha channel, the worst case, against the default -// downstream_timeout of 60 seconds. On an image of milder noise, which 1 -// saves in about 12 seconds, 2 takes nearly a minute and libvips' default, -// 4, takes minutes. +// govips can set. With one thread, as pixad runs libvips, 1 takes about 52 +// seconds to save an 8192x8192 RGB image of random pixels at quality 85, +// and about 96 with an alpha channel, against the default +// downstream_timeout of 60 seconds, so an AVIF output is held to +// MaxAVIFOutputPixels. On an image of milder noise, which 1 saves in about +// 12 seconds, 2 takes nearly a minute and libvips' default, 4, takes +// minutes. const avifEffort = 1 // avifBitdepth is the AVIF bit depth, 8 bits per sample for every image. @@ -715,10 +771,18 @@ func exportAVIF(img *vips.ImageRef, quality int) ([]byte, error) { // JPEG that names none. const jxlResolution = 72 / 25.4 -// exportJXL encodes img as JPEG XL at quality, with libvips' default effort -// and without metadata. govips sends libvips a distance, the JPEG XL -// encoder's own measure of quality, along with the quality, and libvips then -// uses the distance alone, so the quality is also given as a distance. +// jxlEffort is the JPEG XL effort, from 1 (fastest) to 9. It is below +// libvips' default of 7 so that an encode takes at most 40 seconds of the +// default downstream_timeout of 60 seconds. With one thread, as pixad runs +// libvips, saving an 8192x8192 image of random pixels with an alpha channel +// takes up to about 27 seconds at 4, and up to about 13 without one, at any +// quality; with an alpha channel, 5 takes about 43 seconds and 7 about 81. +const jxlEffort = 4 + +// exportJXL encodes img as JPEG XL at quality and jxlEffort, without +// metadata. govips sends libvips a distance, the JPEG XL encoder's own +// measure of quality, along with the quality, and libvips then uses the +// distance alone, so the quality is also given as a distance. func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) { // libvips converts a CMYK image to sRGB before it saves WebP, AVIF or // PNG, but cannot save one as JPEG XL. encode has already converted @@ -757,6 +821,7 @@ func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) { params := vips.NewJxlExportParams() params.Quality = quality params.Distance = jxlDistance(quality) + params.Effort = jxlEffort output, _, err := toSave.ExportJxl(params) if err != nil { From 385c0ca76f35e4229b1f34d23a4323fca5ea5918 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 21 Oct 2026 13:22:59 +0200 Subject: [PATCH 096/131] Run only script/cibuild in the Gitea workflow (closes #203) The workflow's script/docker-smoke step is removed, so the workflow checks out the repo and runs script/cibuild and nothing else, as the canonical REPO_POLICIES.md says. The owner ruled on the issue that make check covers what needs covering. CI no longer starts the image and waits for its healthcheck. script/docker-smoke and make docker-smoke stay, to run by hand, and the script's header comment no longer says the workflow runs it. The dated TODO.md entries that say the workflow runs it are history and stay as written. Model: opus-5-5 --- .gitea/workflows/check.yml | 1 - TODO.md | 5 +++++ script/docker-smoke | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 3556efe..2e96965 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -18,4 +18,3 @@ jobs: # All history and tags, so git describe finds the version tag. fetch-depth: 0 - run: script/cibuild - - run: script/docker-smoke diff --git a/TODO.md b/TODO.md index fb2cde2..e75260a 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,11 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-21 the Gitea workflow checks out the repo and runs `script/cibuild`, + nothing else (closes #203): its `script/docker-smoke` step is removed, as the + canonical `REPO_POLICIES.md` says, since `make check` covers what CI needs to + check. CI no longer starts the image and waits for its healthcheck. + `script/docker-smoke` and `make docker-smoke` stay, to run by hand. - 2026-10-20 a WebP, AVIF or JPEG XL encode at the limits takes at most about 34 seconds (closes #272): with one libvips thread, 8192x8192 random pixels with an alpha channel took about 61 seconds to encode as WebP, 81 as JPEG XL and, diff --git a/script/docker-smoke b/script/docker-smoke index 05ed46a..3bd7862 100755 --- a/script/docker-smoke +++ b/script/docker-smoke @@ -1,8 +1,8 @@ #!/bin/sh # script/docker-smoke: build the Docker image, start it, and wait up to -# 30 seconds for its HEALTHCHECK to report healthy. Needs a Docker -# daemon, so it is not part of script/check; the Gitea workflow runs it -# after script/cibuild. +# 30 seconds for its HEALTHCHECK to report healthy. It is not part of +# script/check, which never starts the image, and the Gitea workflow +# does not run it. Run it by hand. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" From 00d8ba26c5bd90483b59238e66e65f948b30c984 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 21 Oct 2026 14:41:07 +0200 Subject: [PATCH 097/131] One image request type instead of five (closes #307) The new package internal/imagerequest holds the request type, its two parts Source and Transform, the Format and Fit types, the dimension and fit checks, the /v1/image/ path parsing and the variant key; it imports no other package of pixa. imgcache.ParsedURL, imgcache.ImageRequest, signature.Request, imageprocessor.Request, the second Size, format and fit types and the conversions between them are gone. The image processor takes a Transform only. encurl keeps a private struct with the token's short CBOR field names. VariantKey moves with the function that makes it. Tests follow the moved and renamed code; the tests of the removed conversions go with them. Model: opus-5-5 --- TODO.md | 11 + internal/encurl/encurl.go | 117 +++--- internal/encurl/encurl_test.go | 265 +++++------- internal/handlers/auth.go | 68 ++-- .../handlers/auth_generate_internal_test.go | 18 +- .../handlers/auth_session_internal_test.go | 6 +- .../handlers/avif_too_large_internal_test.go | 4 +- .../handlers/first_bytes_internal_test.go | 16 +- internal/handlers/format_auto.go | 20 +- .../handlers/format_auto_internal_test.go | 70 ++-- internal/handlers/image.go | 40 +- .../image_cache_control_internal_test.go | 40 +- .../handlers/image_errors_internal_test.go | 43 +- .../handlers/image_signature_internal_test.go | 123 +++--- internal/handlers/imageenc.go | 29 +- internal/handlers/imageenc_internal_test.go | 30 +- .../handlers/imageenc_token_internal_test.go | 16 +- .../handlers/jpegxl_default_internal_test.go | 36 +- internal/handlers/jpegxl_internal_test.go | 12 +- internal/handlers/request_id_internal_test.go | 32 +- .../handlers/server_busy_internal_test.go | 7 +- .../unsupported_source_internal_test.go | 6 +- .../avif_too_large_internal_test.go | 34 +- .../computed_side_internal_test.go | 51 ++- .../imageprocessor/deadline_internal_test.go | 2 +- .../empty_format_internal_test.go | 5 +- .../encoder_settings_internal_test.go | 19 +- .../imageprocessor/fit_cover_internal_test.go | 46 ++- .../fit_outside_internal_test.go | 37 +- internal/imageprocessor/imageprocessor.go | 170 ++++---- .../imageprocessor_internal_test.go | 152 +++---- .../jpegxl_output_internal_test.go | 26 +- ...max_concurrent_processing_internal_test.go | 60 +-- .../orig_format_internal_test.go | 4 +- .../source_formats_internal_test.go | 12 +- internal/imagerequest/imagerequest.go | 147 +++++++ .../sourceurl_internal_test.go | 38 +- .../{imgcache => imagerequest}/urlparser.go | 54 +-- .../urlparser_internal_test.go | 182 ++++----- .../validate_internal_test.go | 30 +- internal/imagerequest/variantkey.go | 39 ++ .../imagerequest/variantkey_internal_test.go | 142 +++++++ internal/imgcache/cache.go | 62 +-- internal/imgcache/cache_internal_test.go | 127 +++--- .../imgcache/cache_usage_internal_test.go | 5 +- .../concurrent_misses_internal_test.go | 61 +-- internal/imgcache/eviction.go | 24 +- internal/imgcache/eviction_internal_test.go | 83 ++-- internal/imgcache/imgcache.go | 130 +----- ...max_concurrent_processing_internal_test.go | 23 +- internal/imgcache/metacache_internal_test.go | 46 ++- .../imgcache/negative_cache_internal_test.go | 37 +- .../partial_cache_files_internal_test.go | 34 +- .../imgcache/pendingcounts_internal_test.go | 4 +- internal/imgcache/service.go | 102 ++--- internal/imgcache/service_internal_test.go | 383 ++++++++++-------- internal/imgcache/stats_internal_test.go | 55 ++- internal/imgcache/storage.go | 50 +-- internal/imgcache/storage_internal_test.go | 153 +------ .../server/computed_side_internal_test.go | 11 +- internal/server/fit_outside_internal_test.go | 13 +- internal/server/source_url_internal_test.go | 23 +- internal/server/svg_as_avif_internal_test.go | 17 +- internal/signature/escaping_test.go | 55 +-- internal/signature/golden_test.go | 87 ++-- internal/signature/quality_fit_test.go | 30 +- internal/signature/signature.go | 75 ++-- internal/signature/signature_test.go | 246 ++++++----- 68 files changed, 2217 insertions(+), 1978 deletions(-) create mode 100644 internal/imagerequest/imagerequest.go rename internal/{imgcache => imagerequest}/sourceurl_internal_test.go (50%) rename internal/{imgcache => imagerequest}/urlparser.go (88%) rename internal/{imgcache => imagerequest}/urlparser_internal_test.go (72%) rename internal/{imgcache => imagerequest}/validate_internal_test.go (54%) create mode 100644 internal/imagerequest/variantkey.go create mode 100644 internal/imagerequest/variantkey_internal_test.go diff --git a/TODO.md b/TODO.md index e75260a..2290071 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-21 one image request type instead of five (closes #307): the new + package `internal/imagerequest`, which imports no other package of pixa, holds + `imagerequest.Request` with its two parts, `Source` (host, path and query) and + `Transform` (size, format, quality and fit), the `Format` and `Fit` types, the + checks, the parsing of the `/v1/image/` path and the variant key. + `imgcache.ParsedURL`, `imgcache.ImageRequest`, `signature.Request`, + `imageprocessor.Request`, the second `Size`, format and fit types and the + conversions between them are gone. The image processor takes a `Transform` + only, and `encurl` keeps a private struct with the token's short CBOR field + names, so tokens made before still decode. Every URL of both image routes + answers as before. - 2026-10-21 the Gitea workflow checks out the repo and runs `script/cibuild`, nothing else (closes #203): its `script/docker-smoke` step is removed, as the canonical `REPO_POLICIES.md` says, since `make check` covers what CI needs to diff --git a/internal/encurl/encurl.go b/internal/encurl/encurl.go index 13a5772..e754ec9 100644 --- a/internal/encurl/encurl.go +++ b/internal/encurl/encurl.go @@ -7,15 +7,15 @@ import ( "github.com/fxamacker/cbor/v2" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/seal" ) // Default values for optional fields. const ( DefaultQuality = 85 - DefaultFormat = imgcache.FormatJXL - DefaultFitMode = imgcache.FitCover + DefaultFormat = imagerequest.FormatJXL + DefaultFitMode = imagerequest.FitCover // HKDF salt for URL encryption key derivation urlKeySalt = "pixa-urlenc-v1" @@ -28,19 +28,19 @@ var ( ErrDecryptFailed = errors.New("failed to decrypt URL") ) -// Payload contains all image request parameters in a compact format. +// payload contains all image request parameters in a compact format. // Field names are short to minimize encoded size. // Fields with omitempty use sane defaults when absent. -type Payload struct { - SourceHost string `cbor:"h"` // required - SourcePath string `cbor:"p"` // required - SourceQuery string `cbor:"q,omitempty"` // optional - Width int `cbor:"w,omitempty"` // 0 = original - Height int `cbor:"ht,omitempty"` // 0 = original - Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: jxl - Quality int `cbor:"ql,omitempty"` // default: 85 - FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover - ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires +type payload struct { + SourceHost string `cbor:"h"` // required + SourcePath string `cbor:"p"` // required + SourceQuery string `cbor:"q,omitempty"` // optional + Width int `cbor:"w,omitempty"` // 0 = original + Height int `cbor:"ht,omitempty"` // 0 = original + Format imagerequest.Format `cbor:"f,omitempty"` // default: jxl + Quality int `cbor:"ql,omitempty"` // default: 85 + FitMode imagerequest.Fit `cbor:"fm,omitempty"` // default: cover + ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires } // Generator creates and parses encrypted URL tokens. @@ -59,9 +59,27 @@ func NewGenerator(signingKey string) (*Generator, error) { return &Generator{key: key}, nil } -// Generate creates an encrypted URL token from the payload. -// The token is CBOR-encoded, encrypted with NaCl secretbox, and base64url-encoded. -func (g *Generator) Generate(p *Payload) (string, error) { +// Generate creates an encrypted URL token from the request's source, +// transform and expiry, each as the request holds it; an empty format, a +// quality of 0 or an empty fit mode is left out of the token, and a zero +// Expires makes a token that never expires. The token is CBOR-encoded, +// encrypted with NaCl secretbox, and base64url-encoded. +func (g *Generator) Generate(req *imagerequest.Request) (string, error) { + p := &payload{ + SourceHost: req.Source.Host, + SourcePath: req.Source.Path, + SourceQuery: req.Source.Query, + Width: req.Transform.Size.Width, + Height: req.Transform.Size.Height, + Format: req.Transform.Format, + Quality: req.Transform.Quality, + FitMode: req.Transform.FitMode, + } + + if !req.Expires.IsZero() { + p.ExpiresAt = req.Expires.Unix() + } + // CBOR encode the payload data, err := cbor.Marshal(p) if err != nil { @@ -72,9 +90,11 @@ func (g *Generator) Generate(p *Payload) (string, error) { return seal.Encrypt(g.key, data) } -// Parse decrypts and validates an encrypted URL token. +// Parse decrypts and validates an encrypted URL token and returns the +// request it holds, with default values for omitted optional fields. A token +// that never expires gives the zero Expires. // Returns ErrExpired if the token has expired, or other errors for invalid tokens. -func (g *Generator) Parse(token string) (*Payload, error) { +func (g *Generator) Parse(token string) (*imagerequest.Request, error) { // Decrypt data, err := seal.Decrypt(g.key, token) if err != nil { @@ -87,7 +107,7 @@ func (g *Generator) Parse(token string) (*Payload, error) { } // CBOR decode - var p Payload + var p payload err = cbor.Unmarshal(data, &p) if err != nil { @@ -99,13 +119,6 @@ func (g *Generator) Parse(token string) (*Payload, error) { return nil, ErrExpired } - return &p, nil -} - -// ToImageRequest converts the payload to an ImageRequest. -// Applies default values for omitted optional fields. An ExpiresAt of 0, a URL -// that never expires, gives the zero Expires. -func (p *Payload) ToImageRequest() *imgcache.ImageRequest { format := p.Format if format == "" { format = DefaultFormat @@ -126,44 +139,18 @@ func (p *Payload) ToImageRequest() *imgcache.ImageRequest { expires = time.Unix(p.ExpiresAt, 0) } - return &imgcache.ImageRequest{ - SourceHost: p.SourceHost, - SourcePath: p.SourcePath, - SourceQuery: p.SourceQuery, - Size: imgcache.Size{ - Width: p.Width, - Height: p.Height, + return &imagerequest.Request{ + Source: imagerequest.Source{ + Host: p.SourceHost, + Path: p.SourcePath, + Query: p.SourceQuery, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: p.Width, Height: p.Height}, + Format: format, + Quality: quality, + FitMode: fitMode, }, - Format: format, - Quality: quality, - FitMode: fitMode, Expires: expires, - } -} - -// FromImageRequest creates a Payload from an ImageRequest with the given expiration. -func FromImageRequest(req *imgcache.ImageRequest, expiresAt time.Time) *Payload { - p := &Payload{ - SourceHost: req.SourceHost, - SourcePath: req.SourcePath, - SourceQuery: req.SourceQuery, - Width: req.Size.Width, - Height: req.Size.Height, - ExpiresAt: expiresAt.Unix(), - } - - // Only set non-default values to benefit from omitempty - if req.Format != "" && req.Format != DefaultFormat { - p.Format = req.Format - } - - if req.Quality != 0 && req.Quality != DefaultQuality { - p.Quality = req.Quality - } - - if req.FitMode != "" && req.FitMode != DefaultFitMode { - p.FitMode = req.FitMode - } - - return p + }, nil } diff --git a/internal/encurl/encurl_test.go b/internal/encurl/encurl_test.go index 48a1dd8..338f678 100644 --- a/internal/encurl/encurl_test.go +++ b/internal/encurl/encurl_test.go @@ -6,7 +6,7 @@ import ( "time" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // Shared test fixture strings. @@ -24,19 +24,22 @@ func TestGenerator_GenerateAndParse(t *testing.T) { t.Fatalf("NewGenerator() error = %v", err) } - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - SourceQuery: testSourceQuery, - Width: 800, - Height: 600, - Format: imgcache.FormatWebP, - Quality: 90, - FitMode: imgcache.FitContain, - ExpiresAt: time.Now().Add(time.Hour).Unix(), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testSourceHost, + Path: testSourcePath, + Query: testSourceQuery, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + Quality: 90, + FitMode: imagerequest.FitContain, + }, + Expires: time.Now().Add(time.Hour), } - token, err := gen.Generate(payload) + token, err := gen.Generate(req) if err != nil { t.Fatalf("Generate() error = %v", err) } @@ -51,40 +54,45 @@ func TestGenerator_GenerateAndParse(t *testing.T) { } // Verify all fields - if parsed.SourceHost != payload.SourceHost { - t.Errorf("SourceHost = %q, want %q", parsed.SourceHost, payload.SourceHost) + if parsed.Source.Host != req.Source.Host { + t.Errorf("SourceHost = %q, want %q", parsed.Source.Host, req.Source.Host) } - if parsed.SourcePath != payload.SourcePath { - t.Errorf("SourcePath = %q, want %q", parsed.SourcePath, payload.SourcePath) + if parsed.Source.Path != req.Source.Path { + t.Errorf("SourcePath = %q, want %q", parsed.Source.Path, req.Source.Path) } - if parsed.SourceQuery != payload.SourceQuery { - t.Errorf("SourceQuery = %q, want %q", parsed.SourceQuery, payload.SourceQuery) + if parsed.Source.Query != req.Source.Query { + t.Errorf("SourceQuery = %q, want %q", parsed.Source.Query, req.Source.Query) } - if parsed.Width != payload.Width { - t.Errorf("Width = %d, want %d", parsed.Width, payload.Width) + if parsed.Transform.Size.Width != req.Transform.Size.Width { + t.Errorf("Width = %d, want %d", + parsed.Transform.Size.Width, req.Transform.Size.Width) } - if parsed.Height != payload.Height { - t.Errorf("Height = %d, want %d", parsed.Height, payload.Height) + if parsed.Transform.Size.Height != req.Transform.Size.Height { + t.Errorf("Height = %d, want %d", + parsed.Transform.Size.Height, req.Transform.Size.Height) } - if parsed.Format != payload.Format { - t.Errorf("Format = %q, want %q", parsed.Format, payload.Format) + if parsed.Transform.Format != req.Transform.Format { + t.Errorf("Format = %q, want %q", + parsed.Transform.Format, req.Transform.Format) } - if parsed.Quality != payload.Quality { - t.Errorf("Quality = %d, want %d", parsed.Quality, payload.Quality) + if parsed.Transform.Quality != req.Transform.Quality { + t.Errorf("Quality = %d, want %d", + parsed.Transform.Quality, req.Transform.Quality) } - if parsed.FitMode != payload.FitMode { - t.Errorf("FitMode = %q, want %q", parsed.FitMode, payload.FitMode) + if parsed.Transform.FitMode != req.Transform.FitMode { + t.Errorf("FitMode = %q, want %q", + parsed.Transform.FitMode, req.Transform.FitMode) } - if parsed.ExpiresAt != payload.ExpiresAt { - t.Errorf("ExpiresAt = %d, want %d", parsed.ExpiresAt, payload.ExpiresAt) + if parsed.Expires.Unix() != req.Expires.Unix() { + t.Errorf("ExpiresAt = %d, want %d", parsed.Expires.Unix(), req.Expires.Unix()) } } @@ -93,13 +101,15 @@ func TestGenerator_Parse_Expired(t *testing.T) { gen, _ := encurl.NewGenerator("test-signing-key-12345") - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - ExpiresAt: time.Now().Add(-time.Hour).Unix(), // Already expired + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testSourceHost, + Path: testSourcePath, + }, + Expires: time.Now().Add(-time.Hour), // Already expired } - token, err := gen.Generate(payload) + token, err := gen.Generate(req) if err != nil { t.Fatalf("Generate() error = %v", err) } @@ -130,13 +140,15 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) { gen, _ := encurl.NewGenerator("test-signing-key-12345") - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - ExpiresAt: time.Now().Add(time.Hour).Unix(), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testSourceHost, + Path: testSourcePath, + }, + Expires: time.Now().Add(time.Hour), } - token, _ := gen.Generate(payload) + token, _ := gen.Generate(req) // Tamper with the token tampered := []byte(token) @@ -156,13 +168,15 @@ func TestGenerator_Parse_WrongKey(t *testing.T) { gen1, _ := encurl.NewGenerator("signing-key-1") gen2, _ := encurl.NewGenerator("signing-key-2") - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - ExpiresAt: time.Now().Add(time.Hour).Unix(), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testSourceHost, + Path: testSourcePath, + }, + Expires: time.Now().Add(time.Hour), } - token, _ := gen1.Generate(payload) + token, _ := gen1.Generate(req) _, err := gen2.Parse(token) if err == nil { @@ -170,140 +184,41 @@ func TestGenerator_Parse_WrongKey(t *testing.T) { } } -func TestPayload_ToImageRequest(t *testing.T) { +func TestGenerator_Parse_Defaults(t *testing.T) { t.Parallel() - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - SourceQuery: testSourceQuery, - Width: 800, - Height: 600, - Format: imgcache.FormatWebP, - Quality: 90, - FitMode: imgcache.FitContain, - ExpiresAt: time.Now().Add(time.Hour).Unix(), + gen, _ := encurl.NewGenerator("test-signing-key-12345") + + // Request with only required fields - should get defaults + token, err := gen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: testSourceHost, + Path: testSourcePath, + }, + Expires: time.Now().Add(time.Hour), + }) + if err != nil { + t.Fatalf("Generate() error = %v", err) } - req := payload.ToImageRequest() - - if req.SourceHost != payload.SourceHost { - t.Errorf("SourceHost = %q, want %q", req.SourceHost, payload.SourceHost) + req, err := gen.Parse(token) + if err != nil { + t.Fatalf("Parse() error = %v", err) } - if req.SourcePath != payload.SourcePath { - t.Errorf("SourcePath = %q, want %q", req.SourcePath, payload.SourcePath) + if req.Transform.Format != encurl.DefaultFormat { + t.Errorf("Format = %q, want default %q", + req.Transform.Format, encurl.DefaultFormat) } - if req.SourceQuery != payload.SourceQuery { - t.Errorf("SourceQuery = %q, want %q", req.SourceQuery, payload.SourceQuery) + if req.Transform.Quality != encurl.DefaultQuality { + t.Errorf("Quality = %d, want default %d", + req.Transform.Quality, encurl.DefaultQuality) } - if req.Size.Width != payload.Width { - t.Errorf("Width = %d, want %d", req.Size.Width, payload.Width) - } - - if req.Size.Height != payload.Height { - t.Errorf("Height = %d, want %d", req.Size.Height, payload.Height) - } - - if req.Format != payload.Format { - t.Errorf("Format = %q, want %q", req.Format, payload.Format) - } - - if req.Quality != payload.Quality { - t.Errorf("Quality = %d, want %d", req.Quality, payload.Quality) - } - - if req.FitMode != payload.FitMode { - t.Errorf("FitMode = %q, want %q", req.FitMode, payload.FitMode) - } -} - -func TestPayload_ToImageRequest_Defaults(t *testing.T) { - t.Parallel() - - // Payload with only required fields - should get defaults - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - ExpiresAt: time.Now().Add(time.Hour).Unix(), - } - - req := payload.ToImageRequest() - - if req.Format != encurl.DefaultFormat { - t.Errorf("Format = %q, want default %q", req.Format, encurl.DefaultFormat) - } - - if req.Quality != encurl.DefaultQuality { - t.Errorf("Quality = %d, want default %d", req.Quality, encurl.DefaultQuality) - } - - if req.FitMode != encurl.DefaultFitMode { - t.Errorf("FitMode = %q, want default %q", req.FitMode, encurl.DefaultFitMode) - } -} - -func TestFromImageRequest(t *testing.T) { - t.Parallel() - - req := &imgcache.ImageRequest{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - SourceQuery: testSourceQuery, - Size: imgcache.Size{Width: 800, Height: 600}, - Format: imgcache.FormatWebP, - Quality: 90, - FitMode: imgcache.FitContain, - } - - expiresAt := time.Now().Add(time.Hour) - payload := encurl.FromImageRequest(req, expiresAt) - - if payload.SourceHost != req.SourceHost { - t.Errorf("SourceHost = %q, want %q", payload.SourceHost, req.SourceHost) - } - - if payload.SourcePath != req.SourcePath { - t.Errorf("SourcePath = %q, want %q", payload.SourcePath, req.SourcePath) - } - - if payload.Width != req.Size.Width { - t.Errorf("Width = %d, want %d", payload.Width, req.Size.Width) - } - - if payload.ExpiresAt != expiresAt.Unix() { - t.Errorf("ExpiresAt = %d, want %d", payload.ExpiresAt, expiresAt.Unix()) - } -} - -func TestFromImageRequest_OmitsDefaults(t *testing.T) { - t.Parallel() - - // Request with default values - payload should omit them for - // smaller encoding - req := &imgcache.ImageRequest{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - Format: encurl.DefaultFormat, - Quality: encurl.DefaultQuality, - FitMode: encurl.DefaultFitMode, - } - - payload := encurl.FromImageRequest(req, time.Now().Add(time.Hour)) - - // These should be zero/empty because they match defaults - if payload.Format != "" { - t.Errorf("Format should be empty for default, got %q", payload.Format) - } - - if payload.Quality != 0 { - t.Errorf("Quality should be 0 for default, got %d", payload.Quality) - } - - if payload.FitMode != "" { - t.Errorf("FitMode should be empty for default, got %q", payload.FitMode) + if req.Transform.FitMode != encurl.DefaultFitMode { + t.Errorf("FitMode = %q, want default %q", + req.Transform.FitMode, encurl.DefaultFitMode) } } @@ -312,13 +227,15 @@ func TestGenerator_TokenIsURLSafe(t *testing.T) { gen, _ := encurl.NewGenerator("test-signing-key-12345") - payload := &encurl.Payload{ - SourceHost: testSourceHost, - SourcePath: testSourcePath, - ExpiresAt: time.Now().Add(time.Hour).Unix(), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testSourceHost, + Path: testSourcePath, + }, + Expires: time.Now().Add(time.Hour), } - token, _ := gen.Generate(payload) + token, _ := gen.Generate(req) // Check that token only contains URL-safe characters for _, c := range token { diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index 27105f2..a5a6702 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -16,7 +16,7 @@ import ( "sneak.berlin/go/pixa/internal/clientip" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/templates" ) @@ -145,7 +145,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { return } - payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form) + req, expiresAt, ttl, err := buildGenerateRequest(parsed, r.Form) if err != nil { s.renderGeneratorWithForm(w, r, err.Error(), r.Form, http.StatusBadRequest) @@ -154,9 +154,8 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { } // The image URL must fetch the source URL as it was written - sourceRequest := payload.ToImageRequest() - sourceRequest.AllowHTTP = s.config.AllowHTTP - fetchURL := sourceRequest.SourceURL() + req.AllowHTTP = s.config.AllowHTTP + fetchURL := req.SourceURL() if !fetchedAsWritten(parsed, fetchURL) { s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+ @@ -167,7 +166,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { } // Generate encrypted token - token, err := s.encGen.Generate(payload) + token, err := s.encGen.Generate(req) if err != nil { s.log.Error("failed to generate encrypted URL", "error", err) s.renderGeneratorWithForm(w, r, "Failed to generate URL", r.Form, @@ -240,14 +239,15 @@ func fetchedAsWritten(source *url.URL, fetchURL string) bool { fetched.RawQuery == source.RawQuery } -// buildGeneratePayload parses the numeric form fields and assembles the -// encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A -// non-numeric or out-of-range width, height, quality or ttl, or an -// unrecognized fit mode, is a client error naming the offending field. The -// format field is passed through unchecked. -func buildGeneratePayload( +// buildGenerateRequest parses the numeric form fields and assembles the +// image request the encrypted URL's token is made from. ttl=0 means never +// expires (Expires stays the zero time). A non-numeric or out-of-range +// width, height, quality or ttl, or an unrecognized fit mode, is a client +// error naming the offending field. The format field is passed through +// unchecked. +func buildGenerateRequest( parsed *url.URL, form url.Values, -) (*encurl.Payload, time.Time, int, error) { +) (*imagerequest.Request, time.Time, int, error) { width, err := parseFormDimension(form, "width") if err != nil { return nil, time.Time{}, 0, err @@ -269,42 +269,40 @@ func buildGeneratePayload( return nil, time.Time{}, 0, err } - fitMode := imgcache.FitMode(form.Get("fit")) + fitMode := imagerequest.Fit(form.Get("fit")) - err = imgcache.ValidateFitMode(fitMode) + err = imagerequest.ValidateFitMode(fitMode) if err != nil { return nil, time.Time{}, 0, - fmt.Errorf("%w: %s", imgcache.ErrInvalidFitMode, form.Get("fit")) + fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, form.Get("fit")) } - var ( - expiresAt time.Time - expiresAtUnix int64 - ) - + var expiresAt time.Time if ttl > 0 { expiresAt = time.Now().Add(time.Duration(ttl) * time.Second) - expiresAtUnix = expiresAt.Unix() } - payload := &encurl.Payload{ - SourceHost: parsed.Host, - SourcePath: parsed.Path, - SourceQuery: parsed.RawQuery, - Width: width, - Height: height, - Format: imgcache.ImageFormat(form.Get("format")), - Quality: quality, - FitMode: fitMode, - ExpiresAt: expiresAtUnix, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: parsed.Host, + Path: parsed.Path, + Query: parsed.RawQuery, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: width, Height: height}, + Format: imagerequest.Format(form.Get("format")), + Quality: quality, + FitMode: fitMode, + }, + Expires: expiresAt, } - return payload, expiresAt, ttl, nil + return req, expiresAt, ttl, nil } // parseFormDimension reads an optional width or height form field. An empty // value means "original size" (0). A non-numeric value, or one -// imgcache.ValidateDimension rejects, is an error naming the field. +// imagerequest.ValidateDimension rejects, is an error naming the field. func parseFormDimension(form url.Values, field string) (int, error) { raw := form.Get(field) if raw == "" { @@ -316,7 +314,7 @@ func parseFormDimension(form url.Values, field string) (int, error) { return 0, fmt.Errorf("%w %s: not a number", errInvalidFormField, field) } - err = imgcache.ValidateDimension(field, value) + err = imagerequest.ValidateDimension(field, value) if err != nil { return 0, err } diff --git a/internal/handlers/auth_generate_internal_test.go b/internal/handlers/auth_generate_internal_test.go index bc26990..1da7c1b 100644 --- a/internal/handlers/auth_generate_internal_test.go +++ b/internal/handlers/auth_generate_internal_test.go @@ -11,7 +11,7 @@ import ( "time" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // Generator form field names, and a value that is not a number. @@ -125,10 +125,10 @@ func TestGeneratePostRejectsBadField(t *testing.T) { } } -// TestBuildGeneratePayloadDefaultAndLimits verifies that an empty quality +// TestBuildGenerateRequestDefaultAndLimits verifies that an empty quality // takes the default, and that the largest accepted width, height, quality and // ttl are accepted with an expiry still in the future. -func TestBuildGeneratePayloadDefaultAndLimits(t *testing.T) { +func TestBuildGenerateRequestDefaultAndLimits(t *testing.T) { t.Parallel() parsed, err := url.Parse(testSourceURL) @@ -136,19 +136,19 @@ func TestBuildGeneratePayloadDefaultAndLimits(t *testing.T) { t.Fatalf("url.Parse() error = %v", err) } - payload, _, _, err := buildGeneratePayload(parsed, url.Values{}) + req, _, _, err := buildGenerateRequest(parsed, url.Values{}) if err != nil { t.Fatalf("empty form: error = %v", err) } - if payload.Quality != encurl.DefaultQuality { + if req.Transform.Quality != encurl.DefaultQuality { t.Errorf("empty quality gave %d, want %d", - payload.Quality, encurl.DefaultQuality) + req.Transform.Quality, encurl.DefaultQuality) } - _, expiresAt, _, err := buildGeneratePayload(parsed, url.Values{ - widthField: {strconv.Itoa(imgcache.MaxDimension)}, - heightField: {strconv.Itoa(imgcache.MaxDimension)}, + _, expiresAt, _, err := buildGenerateRequest(parsed, url.Values{ + widthField: {strconv.Itoa(imagerequest.MaxDimension)}, + heightField: {strconv.Itoa(imagerequest.MaxDimension)}, qualityField: {strconv.Itoa(maxQuality)}, ttlField: {strconv.Itoa(maxTTL)}, }) diff --git a/internal/handlers/auth_session_internal_test.go b/internal/handlers/auth_session_internal_test.go index 250d627..d99ca86 100644 --- a/internal/handlers/auth_session_internal_test.go +++ b/internal/handlers/auth_session_internal_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/session" ) @@ -314,7 +314,7 @@ func TestGeneratePost_URLServesImage(t *testing.T) { sourceURLField: {"https://" + signedHost + photoPath}, widthField: {"50"}, heightField: {"50"}, - formatField: {string(imgcache.FormatJPEG)}, + formatField: {string(imagerequest.FormatJPEG)}, }) if rec.Code != http.StatusOK { @@ -347,7 +347,7 @@ func TestGeneratePost_URLWithTTLExpires(t *testing.T) { sourceURLField: {"https://" + signedHost + photoPath}, widthField: {"50"}, heightField: {"50"}, - formatField: {string(imgcache.FormatJPEG)}, + formatField: {string(imagerequest.FormatJPEG)}, ttlField: {"1"}, }) diff --git a/internal/handlers/avif_too_large_internal_test.go b/internal/handlers/avif_too_large_internal_test.go index c634a03..1efad83 100644 --- a/internal/handlers/avif_too_large_internal_test.go +++ b/internal/handlers/avif_too_large_internal_test.go @@ -9,7 +9,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/imageprocessor" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestAVIFOutputTooLargeAnswers400 checks that respondImageError and @@ -26,7 +26,7 @@ func TestAVIFOutputTooLargeAnswers400(t *testing.T) { wantBody := fmt.Sprintf("above %d pixels", imageprocessor.MaxAVIFOutputPixels) plain := httptest.NewRecorder() - h.respondImageError(plain, &imgcache.ImageRequest{}, err) + h.respondImageError(plain, &imagerequest.Request{}, err) encrypted := httptest.NewRecorder() h.handleImageError(encrypted, err) diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go index fb3ba30..63e7aad 100644 --- a/internal/handlers/first_bytes_internal_test.go +++ b/internal/handlers/first_bytes_internal_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/v5" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -75,12 +76,15 @@ func mismatchedSourcePlainURL(*testing.T, *Handlers) string { func mismatchedSourceEncURL(t *testing.T, h *Handlers) string { t.Helper() - token, err := h.encGen.Generate(&encurl.Payload{ - SourceHost: allowlistedHost, - SourcePath: mismatchedSourcePath, - Width: 50, - Height: 50, - Format: imgcache.FormatJPEG, + token, err := h.encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: allowlistedHost, + Path: mismatchedSourcePath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, }) if err != nil { t.Fatalf("Generate() error = %v", err) diff --git a/internal/handlers/format_auto.go b/internal/handlers/format_auto.go index bca3bde..0dda8f8 100644 --- a/internal/handlers/format_auto.go +++ b/internal/handlers/format_auto.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // Errors for an Accept header that an auto URL cannot be served for. @@ -24,9 +24,9 @@ var ( // header that is not valid and 406 for one that allows none of the formats, // and reports whether req can be served. Any other format is left as it is. func (s *Handlers) chooseAutoFormat( - w http.ResponseWriter, r *http.Request, req *imgcache.ImageRequest, + w http.ResponseWriter, r *http.Request, req *imagerequest.Request, ) bool { - if req.Format != imgcache.FormatAuto { + if req.Transform.Format != imagerequest.FormatAuto { return true } @@ -45,7 +45,7 @@ func (s *Handlers) chooseAutoFormat( return false } - req.Format = format + req.Transform.Format = format return true } @@ -56,26 +56,26 @@ func (s *Handlers) chooseAutoFormat( // specific entry of image/jpeg, image/* and */* allows it, or when it names // nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as // clients that cannot show them send image/* and */* too. -func formatForAccept(accept string) (imgcache.ImageFormat, error) { +func formatForAccept(accept string) (imagerequest.Format, error) { qualities, err := parseAccept(accept) if err != nil { return "", err } if len(qualities) == 0 { - return imgcache.FormatJPEG, nil + return imagerequest.FormatJPEG, nil } if qualities["image/jxl"] > 0 { - return imgcache.FormatJXL, nil + return imagerequest.FormatJXL, nil } if qualities["image/avif"] > 0 { - return imgcache.FormatAVIF, nil + return imagerequest.FormatAVIF, nil } if qualities["image/webp"] > 0 { - return imgcache.FormatWebP, nil + return imagerequest.FormatWebP, nil } // For JPEG, the most specific entry the header has decides @@ -89,7 +89,7 @@ func formatForAccept(accept string) (imgcache.ImageFormat, error) { } if quality > 0 { - return imgcache.FormatJPEG, nil + return imagerequest.FormatJPEG, nil } return "", errNotAcceptable diff --git a/internal/handlers/format_auto_internal_test.go b/internal/handlers/format_auto_internal_test.go index 9b28d13..d899b36 100644 --- a/internal/handlers/format_auto_internal_test.go +++ b/internal/handlers/format_auto_internal_test.go @@ -12,8 +12,7 @@ import ( "github.com/go-chi/chi/v5" - "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // The content types the tests below expect. @@ -33,24 +32,24 @@ func TestFormatForAccept(t *testing.T) { tests := []struct { name string accept string - want imgcache.ImageFormat + want imagerequest.Format wantErr error }{ {"AVIF-capable browser", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", - imgcache.FormatAVIF, nil}, + imagerequest.FormatAVIF, nil}, {"WebP-capable browser", "image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5", - imgcache.FormatWebP, nil}, - {"WebP only", webpType, imgcache.FormatWebP, nil}, - {"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imgcache.FormatJPEG, nil}, - {"wildcard only", "*/*", imgcache.FormatJPEG, nil}, - {"image wildcard only", "image/*", imgcache.FormatJPEG, nil}, - {"absent", "", imgcache.FormatJPEG, nil}, - {"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imgcache.FormatWebP, nil}, + imagerequest.FormatWebP, nil}, + {"WebP only", webpType, imagerequest.FormatWebP, nil}, + {"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imagerequest.FormatJPEG, nil}, + {"wildcard only", "*/*", imagerequest.FormatJPEG, nil}, + {"image wildcard only", "image/*", imagerequest.FormatJPEG, nil}, + {"absent", "", imagerequest.FormatJPEG, nil}, + {"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imagerequest.FormatWebP, nil}, {"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*", - imgcache.FormatJPEG, nil}, - {"upper case and spaces", " Image/AVIF ; Q=0.5 ", imgcache.FormatAVIF, nil}, + imagerequest.FormatJPEG, nil}, + {"upper case and spaces", " Image/AVIF ; Q=0.5 ", imagerequest.FormatAVIF, nil}, {"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable}, {"q=0 on everything", "*/*;q=0", "", errNotAcceptable}, {"PNG only", "image/png", "", errNotAcceptable}, @@ -80,23 +79,30 @@ func TestFormatForAccept(t *testing.T) { func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { t.Helper() - signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: photoPath, - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatAuto, + signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatAuto, + }, }, time.Minute) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) } - token, err := h.encGen.Generate(&encurl.Payload{ - SourceHost: signedHost, - SourcePath: photoPath, - Width: 50, - Height: 50, - Format: imgcache.FormatAuto, - ExpiresAt: time.Now().Add(time.Minute).Unix(), + token, err := h.encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatAuto, + }, + Expires: time.Now().Add(time.Minute), }) if err != nil { t.Fatalf("Generate() error = %v", err) @@ -184,11 +190,15 @@ func TestFormatAuto_SignatureCoversAuto(t *testing.T) { h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: photoPath, - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatAVIF, + signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatAVIF, + }, }, time.Minute) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) diff --git a/internal/handlers/image.go b/internal/handlers/image.go index f1a6ff0..b11b0f5 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -13,6 +13,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/magic" ) @@ -35,8 +36,8 @@ func (s *Handlers) HandleImage() http.HandlerFunc { err := s.imgSvc.ValidateRequest(req) if err != nil { s.log.Warn("signature validation failed", - "host", req.SourceHost, - "path", req.SourcePath, + "host", req.Source.Host, + "path", req.Source.Path, "error", err, ) s.respondError(w, "unauthorized", http.StatusUnauthorized) @@ -50,7 +51,7 @@ func (s *Handlers) HandleImage() http.HandlerFunc { } // Get cache key for logging - cacheKey := imgcache.CacheKey(req) + cacheKey := imagerequest.CacheKey(req) // Get the image (from cache or fetch/process) startTime := time.Now() @@ -81,17 +82,17 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc { } // parseImageRequest parses the image URL's path and query parameters into -// an ImageRequest. On invalid input it writes an error response and +// an image request. On invalid input it writes an error response and // returns false. func (s *Handlers) parseImageRequest( w http.ResponseWriter, r *http.Request, -) (*imgcache.ImageRequest, bool) { +) (*imagerequest.Request, bool) { // The path is parsed decoded once, as r.URL.Path holds it, and not from // chi's wildcard: chi routes on the path as the client wrote it whenever // that is not Go's own encoding of it, so its wildcard would still hold // the %3F before a source query when, say, a "/" in the query is written // as %2F - parsed, err := imgcache.ParseImageURL(r.URL.Path) + req, err := imagerequest.ParseImageURL(r.URL.Path) if err != nil { s.log.Warn("failed to parse image URL", "path", r.URL.Path, @@ -102,9 +103,6 @@ func (s *Handlers) parseImageRequest( return nil, false } - // Convert to ImageRequest - req := parsed.ToImageRequest() - // Parse signature params from query string. r.URL.Query() would silently // drop a pair it cannot decode, such as q=80%, so that q would be served // at 85; a query string that cannot be decoded is refused instead. A @@ -148,7 +146,7 @@ func (s *Handlers) parseImageRequest( return nil, false } - req.Quality, err = parseFormInt(query, "q", + req.Transform.Quality, err = parseFormInt(query, "q", encurl.DefaultQuality, minQuality, maxQuality) if err != nil { s.respondError(w, fmt.Sprintf("%v, got %q", err, qStr), @@ -160,23 +158,23 @@ func (s *Handlers) parseImageRequest( // Only a fit missing from the URL is cover. A fit in the URL that is not a // fit mode is refused by the fit-mode check below; that check would take an // empty fit as missing, so an empty one is refused here. - req.FitMode = imgcache.FitMode(query.Get("fit")) - if query.Has("fit") && req.FitMode == "" { + req.Transform.FitMode = imagerequest.Fit(query.Get("fit")) + if query.Has("fit") && req.Transform.FitMode == "" { s.respondError(w, `invalid fit: not a fit mode, got ""`, http.StatusBadRequest) return nil, false } // Default fit mode if not set - if req.FitMode == "" { - req.FitMode = imgcache.FitCover + if req.Transform.FitMode == "" { + req.Transform.FitMode = imagerequest.FitCover } // Enforce dimension and fit-mode bounds, shared with the encrypted-URL // route. Dimensions are already bounded by the path parser above; this // also rejects an unrecognized fit mode with 400 instead of letting it // reach the processor as a 500. - err = imgcache.ValidateImageRequest(req) + err = imagerequest.ValidateImageRequest(req) if err != nil { s.respondError(w, "invalid image request: "+err.Error(), http.StatusBadRequest) @@ -219,7 +217,7 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc { // respondImageError maps image retrieval errors to HTTP responses. func (s *Handlers) respondImageError( - w http.ResponseWriter, req *imgcache.ImageRequest, err error, + w http.ResponseWriter, req *imagerequest.Request, err error, ) { // An upstream body whose first bytes are not those of the format its // Content-Type names is the upstream's error, not pixa's: the image @@ -234,8 +232,8 @@ func (s *Handlers) respondImageError( } s.log.Error("failed to get image", - "host", req.SourceHost, - "path", req.SourcePath, + "host", req.Source.Host, + "path", req.Source.Path, "error", err, ) @@ -346,8 +344,8 @@ func notModified(w http.ResponseWriter, r *http.Request, etag string) bool { // handling conditional and HEAD requests. func (s *Handlers) writeImageResponse( w http.ResponseWriter, r *http.Request, - req *imgcache.ImageRequest, resp *imgcache.ImageResponse, - cacheKey imgcache.VariantKey, startTime time.Time, + req *imagerequest.Request, resp *imgcache.ImageResponse, + cacheKey imagerequest.VariantKey, startTime time.Time, ) { // Set response headers w.Header().Set("Content-Type", resp.ContentType) @@ -388,7 +386,7 @@ func (s *Handlers) writeImageResponse( "cache_key", cacheKey, "cache_status", resp.CacheStatus, "duration_ms", duration.Milliseconds(), - "format", req.Format, + "format", req.Transform.Format, "served_bytes", servedBytes, "fetched_bytes", resp.FetchedBytes, ) diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index f4e65a5..e94c30f 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/v5" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -107,11 +108,15 @@ func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) { h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: photoPath, - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatJPEG, + signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, }, time.Minute) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) @@ -168,13 +173,13 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { tests := []struct { name string - expiresAt int64 + expires time.Time wantAtLeast int wantAtMost int }{ - {"60 second TTL", time.Now().Add(time.Minute).Unix(), 50, 60}, - {"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour).Unix(), 31536000, 31536000}, - {"no TTL", 0, 31536000, 31536000}, + {"60 second TTL", time.Now().Add(time.Minute), 50, 60}, + {"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour), 31536000, 31536000}, + {"no TTL", time.Time{}, 31536000, 31536000}, } for _, tt := range tests { @@ -183,13 +188,16 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - token, err := h.encGen.Generate(&encurl.Payload{ - SourceHost: signedHost, - SourcePath: photoPath, - Width: 50, - Height: 50, - Format: imgcache.FormatJPEG, - ExpiresAt: tt.expiresAt, + token, err := h.encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, + Expires: tt.expires, }) if err != nil { t.Fatalf("Generate() error = %v", err) diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index 25fe061..72acfd8 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/v5" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/signature" ) @@ -84,15 +85,18 @@ func photoURLWithSig(host, sig string, expires time.Time) string { // photoSignature returns the signature of photoURL(host) at the default // quality and fit, made with key and expiring at expires. func photoSignature(key, host string, expires time.Time) string { - return signature.New(key).Sign(&signature.Request{ - SourceHost: host, - SourcePath: photoPath, - Width: 50, - Height: 50, - Format: string(imgcache.FormatJPEG), - Quality: 85, - FitMode: string(imgcache.FitCover), - Expires: expires, + return signature.New(key).Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: host, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: expires, }) } @@ -353,15 +357,18 @@ func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) { photos := newPhotoFetcher(t, allowlistedHost, signedHost) exp := time.Now().Add(time.Hour) - sig := signature.New(testSigningKey).Sign(&signature.Request{ - SourceHost: signedHost, - SourcePath: photoPath + "#x", - Width: 50, - Height: 50, - Format: string(imgcache.FormatJPEG), - Quality: 85, - FitMode: string(imgcache.FitCover), - Expires: exp, + sig := signature.New(testSigningKey).Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath + "#x", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: exp, }) tests := []struct { diff --git a/internal/handlers/image_signature_internal_test.go b/internal/handlers/image_signature_internal_test.go index babdb1e..2849565 100644 --- a/internal/handlers/image_signature_internal_test.go +++ b/internal/handlers/image_signature_internal_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/go-chi/chi/v5" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/signature" ) @@ -44,15 +44,18 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) { expires := time.Now().Add(time.Hour) signer := signature.New("test-signing-key-must-be-32-chars") - sig := signer.Sign(&signature.Request{ - SourceHost: signedHost, - SourcePath: "/images/photo.jpg", - Width: 50, - Height: 50, - Format: string(imgcache.FormatJPEG), - Quality: 85, - FitMode: string(imgcache.FitCover), - Expires: expires, + sig := signer.Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: "/images/photo.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: expires, }) signedURL := fmt.Sprintf("/v1/image/%s/images/photo.jpg/50x50.jpeg?sig=%s&exp=%d", signedHost, sig, expires.Unix()) @@ -88,9 +91,9 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) { tests := []struct { name string quality int - fitMode imgcache.FitMode + fitMode imagerequest.Fit }{ - {"quality 40 and fit contain", 40, imgcache.FitContain}, + {"quality 40 and fit contain", 40, imagerequest.FitContain}, {"quality and fit unset", 0, ""}, } @@ -100,13 +103,17 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) { fix := setupTestHandler(t) - signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: "/images/photo.jpg", - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatJPEG, - Quality: tt.quality, - FitMode: tt.fitMode, + signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: "/images/photo.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: tt.quality, + FitMode: tt.fitMode, + }, }, time.Hour) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) @@ -196,12 +203,16 @@ func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) { fix := setupTestHandler(t) - signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: photoPath, - SourceQuery: tt.query, - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatJPEG, + signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + Query: tt.query, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, }, time.Hour) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) @@ -224,16 +235,19 @@ func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) { expires := time.Now().Add(time.Hour) signer := signature.New("test-signing-key-must-be-32-chars") - sig := signer.Sign(&signature.Request{ - SourceHost: signedHost, - SourcePath: photoPath, - SourceQuery: "url=https://origin.example.com/my%20cat.jpg", - Width: 50, - Height: 50, - Format: string(imgcache.FormatJPEG), - Quality: 85, - FitMode: string(imgcache.FitCover), - Expires: expires, + sig := signer.Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + Query: "url=https://origin.example.com/my%20cat.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: expires, }) tests := []struct { @@ -288,16 +302,19 @@ func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) { expires := time.Now().Add(time.Hour) signer := signature.New("test-signing-key-must-be-32-chars") - sig := signer.Sign(&signature.Request{ - SourceHost: signedHost, - SourcePath: tt.sourcePath, - SourceQuery: tt.sourceQuery, - Width: 50, - Height: 50, - Format: string(imgcache.FormatJPEG), - Quality: 85, - FitMode: string(imgcache.FitCover), - Expires: expires, + sig := signer.Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: tt.sourcePath, + Query: tt.sourceQuery, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: expires, }) target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d", @@ -314,12 +331,16 @@ func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) { fix := setupTestHandler(t) - signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: tt.sourcePath, - SourceQuery: tt.sourceQuery, - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatJPEG, + signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: tt.sourcePath, + Query: tt.sourceQuery, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, }, time.Hour) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index eefb74f..e5a94d4 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/magic" ) @@ -37,10 +38,11 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { // Log the request s.log.Debug("encrypted image request", - "host", req.SourceHost, - "path", req.SourcePath, - "dimensions", fmt.Sprintf("%dx%d", req.Size.Width, req.Size.Height), - "format", req.Format, + "host", req.Source.Host, + "path", req.Source.Path, + "dimensions", fmt.Sprintf("%dx%d", + req.Transform.Size.Width, req.Transform.Size.Height), + "format", req.Transform.Format, ) // Fetch and process the image (no signature validation @@ -88,10 +90,10 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { duration := time.Since(start) s.log.Info("image served", "request_id", middleware.GetReqID(ctx), - "cache_key", imgcache.CacheKey(req), - "host", req.SourceHost, - "path", req.SourcePath, - "format", req.Format, + "cache_key", imagerequest.CacheKey(req), + "host", req.Source.Host, + "path", req.Source.Path, + "format", req.Transform.Format, "cache_status", resp.CacheStatus, "served_bytes", written, "duration_ms", duration.Milliseconds(), @@ -100,12 +102,12 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { } // parseImageEncRequest decrypts the token of an encrypted image URL into an -// ImageRequest and checks it. On a token that is missing, does not decrypt, +// image request and checks it. On a token that is missing, does not decrypt, // has expired or asks for something not valid, it writes an error response // and returns false. func (s *Handlers) parseImageEncRequest( w http.ResponseWriter, r *http.Request, -) (*imgcache.ImageRequest, bool) { +) (*imagerequest.Request, bool) { // Extract token from URL token := chi.URLParam(r, "token") if token == "" { @@ -115,7 +117,7 @@ func (s *Handlers) parseImageEncRequest( } // Decrypt and validate the payload - payload, err := s.encGen.Parse(token) + req, err := s.encGen.Parse(token) if err != nil { if errors.Is(err, encurl.ErrExpired) { s.log.Debug("encrypted URL expired", "error", err) @@ -130,14 +132,11 @@ func (s *Handlers) parseImageEncRequest( return nil, false } - // Convert payload to ImageRequest - req := payload.ToImageRequest() - // Apply the same dimension and fit-mode bounds as the plain image // route: a sealed payload is trusted for its origin, not for staying // within limits, so an over-limit size or unknown fit mode is a 400 // here rather than an out-of-memory or a 500 from the processor. - err = imgcache.ValidateImageRequest(req) + err = imagerequest.ValidateImageRequest(req) if err != nil { s.log.Debug("encrypted URL failed validation", "error", err) s.respondError(w, "invalid encrypted URL: "+err.Error(), diff --git a/internal/handlers/imageenc_internal_test.go b/internal/handlers/imageenc_internal_test.go index d4f8c6b..ec034c5 100644 --- a/internal/handlers/imageenc_internal_test.go +++ b/internal/handlers/imageenc_internal_test.go @@ -10,7 +10,7 @@ import ( "github.com/go-chi/chi/v5" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // newEncTestServer builds a router serving the encrypted-URL route with a @@ -54,11 +54,14 @@ func TestHandleImageEnc_OverLimitDimension_Returns400(t *testing.T) { encGen, srv := newEncTestServer(t) - token, err := encGen.Generate(&encurl.Payload{ - SourceHost: "cdn.example.com", - SourcePath: "/photo.jpg", - Width: 100000, - Height: 100000, + token, err := encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: "cdn.example.com", + Path: "/photo.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 100000, Height: 100000}, + }, }) if err != nil { t.Fatalf("Generate() error = %v", err) @@ -79,12 +82,15 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) { encGen, srv := newEncTestServer(t) - token, err := encGen.Generate(&encurl.Payload{ - SourceHost: "cdn.example.com", - SourcePath: "/photo.jpg", - Width: 800, - Height: 600, - FitMode: imgcache.FitMode("bogus"), + token, err := encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: "cdn.example.com", + Path: "/photo.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + FitMode: imagerequest.Fit("bogus"), + }, }) if err != nil { t.Fatalf("Generate() error = %v", err) diff --git a/internal/handlers/imageenc_token_internal_test.go b/internal/handlers/imageenc_token_internal_test.go index f2fcda2..f647a11 100644 --- a/internal/handlers/imageenc_token_internal_test.go +++ b/internal/handlers/imageenc_token_internal_test.go @@ -9,6 +9,7 @@ import ( "time" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imagerequest" ) // requireServedPhoto requires that rec answers 200 with the JPEG at photoPath @@ -59,11 +60,14 @@ func TestHandleImageEnc_RejectedToken(t *testing.T) { h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - photo := encurl.Payload{ - SourceHost: signedHost, - SourcePath: photoPath, - Width: 50, - Height: 50, + photo := imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + }, } valid, err := h.encGen.Generate(&photo) @@ -72,7 +76,7 @@ func TestHandleImageEnc_RejectedToken(t *testing.T) { } expiredPhoto := photo - expiredPhoto.ExpiresAt = time.Now().Add(-time.Minute).Unix() + expiredPhoto.Expires = time.Now().Add(-time.Minute) expired, err := h.encGen.Generate(&expiredPhoto) if err != nil { diff --git a/internal/handlers/jpegxl_default_internal_test.go b/internal/handlers/jpegxl_default_internal_test.go index 1d22bca..219c0f0 100644 --- a/internal/handlers/jpegxl_default_internal_test.go +++ b/internal/handlers/jpegxl_default_internal_test.go @@ -14,7 +14,7 @@ import ( "github.com/davidbyttow/govips/v2/vips" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/signature" ) @@ -58,15 +58,18 @@ func TestImageWithoutFormat_SignedAsJXL(t *testing.T) { route := newImageRoute(t, newPhotoFetcher(t, signedHost)) expires := time.Now().Add(time.Hour) - sig := signature.New(testSigningKey).Sign(&signature.Request{ - SourceHost: signedHost, - SourcePath: photoPath, - Width: 50, - Height: 50, - Format: string(imgcache.FormatJXL), - Quality: encurl.DefaultQuality, - FitMode: string(imgcache.FitCover), - Expires: expires, + sig := signature.New(testSigningKey).Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJXL, + Quality: encurl.DefaultQuality, + FitMode: imagerequest.FitCover, + }, + Expires: expires, }) query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix()) @@ -83,11 +86,14 @@ func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) { h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - token, err := h.encGen.Generate(&encurl.Payload{ - SourceHost: signedHost, - SourcePath: photoPath, - Width: 50, - Height: 50, + token, err := h.encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + }, }) if err != nil { t.Fatalf("Generate() error = %v", err) diff --git a/internal/handlers/jpegxl_internal_test.go b/internal/handlers/jpegxl_internal_test.go index 68bbe14..c788eee 100644 --- a/internal/handlers/jpegxl_internal_test.go +++ b/internal/handlers/jpegxl_internal_test.go @@ -7,7 +7,7 @@ import ( "github.com/davidbyttow/govips/v2/vips" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // jxlType is the content type of JPEG XL. @@ -22,14 +22,14 @@ func TestFormatForAccept_JPEGXL(t *testing.T) { tests := []struct { name string accept string - want imgcache.ImageFormat + want imagerequest.Format }{ {"JPEG XL-capable browser", - "image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imgcache.FormatJXL}, - {"JPEG XL only", jxlType, imgcache.FormatJXL}, + "image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imagerequest.FormatJXL}, + {"JPEG XL only", jxlType, imagerequest.FormatJXL}, {"JPEG XL with a lower q than AVIF", "image/avif,image/jxl;q=0.5", - imgcache.FormatJXL}, - {"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imgcache.FormatAVIF}, + imagerequest.FormatJXL}, + {"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imagerequest.FormatAVIF}, } for _, tt := range tests { diff --git a/internal/handlers/request_id_internal_test.go b/internal/handlers/request_id_internal_test.go index d26a2f6..a785923 100644 --- a/internal/handlers/request_id_internal_test.go +++ b/internal/handlers/request_id_internal_test.go @@ -13,8 +13,7 @@ import ( "github.com/go-chi/chi/v5/middleware" - "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the @@ -22,11 +21,15 @@ import ( func signedPhotoURL(t *testing.T, h *Handlers) string { t.Helper() - signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{ - SourceHost: signedHost, - SourcePath: photoPath, - Size: imgcache.Size{Width: 50, Height: 50}, - Format: imgcache.FormatJPEG, + signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, }, time.Minute) if err != nil { t.Fatalf("GenerateSignedURL() error = %v", err) @@ -40,12 +43,15 @@ func signedPhotoURL(t *testing.T, h *Handlers) string { func encPhotoURL(t *testing.T, h *Handlers) string { t.Helper() - token, err := h.encGen.Generate(&encurl.Payload{ - SourceHost: signedHost, - SourcePath: photoPath, - Width: 50, - Height: 50, - Format: imgcache.FormatJPEG, + token, err := h.encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, }) if err != nil { t.Fatalf("Generate() error = %v", err) diff --git a/internal/handlers/server_busy_internal_test.go b/internal/handlers/server_busy_internal_test.go index 58bae9b..a7cf030 100644 --- a/internal/handlers/server_busy_internal_test.go +++ b/internal/handlers/server_busy_internal_test.go @@ -11,6 +11,7 @@ import ( "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -21,7 +22,9 @@ func TestServerBusyAnswers503(t *testing.T) { t.Parallel() h := &Handlers{log: slog.New(slog.DiscardHandler)} - req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.jpg"} + req := &imagerequest.Request{ + Source: imagerequest.Source{Host: "img.example.com", Path: "/a.jpg"}, + } for _, err := range []error{ fmt.Errorf("upstream fetch failed: %w", httpfetcher.ErrTooManyConnections), @@ -64,7 +67,7 @@ func TestProcessingTimedOutAnswers503(t *testing.T) { imgcache.ErrProcessingTimedOut, context.DeadlineExceeded) plain := httptest.NewRecorder() - h.respondImageError(plain, &imgcache.ImageRequest{}, err) + h.respondImageError(plain, &imagerequest.Request{}, err) encrypted := httptest.NewRecorder() h.handleImageError(encrypted, err) diff --git a/internal/handlers/unsupported_source_internal_test.go b/internal/handlers/unsupported_source_internal_test.go index a10e37e..c319700 100644 --- a/internal/handlers/unsupported_source_internal_test.go +++ b/internal/handlers/unsupported_source_internal_test.go @@ -10,7 +10,7 @@ import ( "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestUnsupportedSourceFormatAnswers415 checks that both image routes answer @@ -21,7 +21,9 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) { t.Parallel() h := &Handlers{log: slog.New(slog.DiscardHandler)} - req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.svg"} + req := &imagerequest.Request{ + Source: imagerequest.Source{Host: "img.example.com", Path: "/a.svg"}, + } for _, err := range []error{ fmt.Errorf("upstream fetch failed: %w: image/svg+xml", diff --git a/internal/imageprocessor/avif_too_large_internal_test.go b/internal/imageprocessor/avif_too_large_internal_test.go index 2770dd4..1c1a576 100644 --- a/internal/imageprocessor/avif_too_large_internal_test.go +++ b/internal/imageprocessor/avif_too_large_internal_test.go @@ -6,6 +6,8 @@ import ( "errors" "os" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_RefusesAVIFAboveMaxPixels verifies that an AVIF output @@ -37,22 +39,30 @@ func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) { tests := []struct { name string input []byte - format Format + format imagerequest.Format width, height int - fit FitMode + fit imagerequest.Fit refused bool }{ - {"AVIF at the limit", cutOff, FormatAVIF, 4096, 4096, FitCover, false}, - {"AVIF above the limit", cutOff, FormatAVIF, 4096, 4097, FitCover, true}, - {"WebP above the AVIF limit", cutOff, FormatWebP, 4096, 4097, FitCover, false}, + {"AVIF at the limit", cutOff, imagerequest.FormatAVIF, + 4096, 4096, imagerequest.FitCover, false}, + {"AVIF above the limit", cutOff, imagerequest.FormatAVIF, + 4096, 4097, imagerequest.FitCover, true}, + {"WebP above the AVIF limit", cutOff, imagerequest.FormatWebP, + 4096, 4097, imagerequest.FitCover, false}, // FitInside does not enlarge, so the output stays 64x32 - {"AVIF inside a larger size", cutOff, FormatAVIF, 8192, 8192, FitInside, false}, + {"AVIF inside a larger size", cutOff, imagerequest.FormatAVIF, + 8192, 8192, imagerequest.FitInside, false}, // FitOutside scales 64x32 by 128 to cover 8192x1: 8192x4096 - {"AVIF outside a smaller size", cutOff, FormatAVIF, 8192, 1, FitOutside, true}, + {"AVIF outside a smaller size", cutOff, imagerequest.FormatAVIF, + 8192, 1, imagerequest.FitOutside, true}, // With no size, the output keeps the source's 4097x4096 - {"AVIF of the source's size", largeAVIF, FormatAVIF, 0, 0, FitCover, true}, - {"orig of an AVIF source", largeAVIF, FormatOriginal, 0, 0, FitCover, true}, - {"JPEG of the source's size", largeAVIF, FormatJPEG, 0, 0, FitCover, false}, + {"AVIF of the source's size", largeAVIF, imagerequest.FormatAVIF, + 0, 0, imagerequest.FitCover, true}, + {"orig of an AVIF source", largeAVIF, imagerequest.FormatOriginal, + 0, 0, imagerequest.FitCover, true}, + {"JPEG of the source's size", largeAVIF, imagerequest.FormatJPEG, + 0, 0, imagerequest.FitCover, false}, } for _, tt := range tests { @@ -60,8 +70,8 @@ func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) { t.Parallel() _, err := New(Params{}).Process(t.Context(), bytes.NewReader(tt.input), - &Request{ - Size: Size{Width: tt.width, Height: tt.height}, + &imagerequest.Transform{ + Size: imagerequest.Size{Width: tt.width, Height: tt.height}, Format: tt.format, FitMode: tt.fit, }) diff --git a/internal/imageprocessor/computed_side_internal_test.go b/internal/imageprocessor/computed_side_internal_test.go index 231087d..fbd49e6 100644 --- a/internal/imageprocessor/computed_side_internal_test.go +++ b/internal/imageprocessor/computed_side_internal_test.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_RefusesComputedSideAboveMaximum verifies that a size @@ -17,13 +19,17 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) { tests := []struct { name string inputWidth, inputHeight int - size Size + size imagerequest.Size wantErr error }{ - {"width at the maximum", 4096, 1, Size{Height: 2}, nil}, - {"width above the maximum", 4096, 1, Size{Height: 3}, ErrOutputTooLarge}, - {"height at the maximum", 1, 4096, Size{Width: 2}, nil}, - {"height above the maximum", 1, 4096, Size{Width: 3}, ErrOutputTooLarge}, + {"width at the maximum", 4096, 1, + imagerequest.Size{Height: 2}, nil}, + {"width above the maximum", 4096, 1, + imagerequest.Size{Height: 3}, ErrOutputTooLarge}, + {"height at the maximum", 1, 4096, + imagerequest.Size{Width: 2}, nil}, + {"height above the maximum", 1, 4096, + imagerequest.Size{Width: 3}, ErrOutputTooLarge}, } for _, tt := range tests { @@ -34,7 +40,11 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover}) + &imagerequest.Transform{ + Size: tt.size, + Format: imagerequest.FormatJPEG, + FitMode: imagerequest.FitCover, + }) if result != nil { t.Logf("Process() made %dx%d", result.Width, result.Height) @@ -64,18 +74,21 @@ func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) { tests := []struct { name string inputWidth, inputHeight int - size Size + size imagerequest.Size wantWidth, wantHeight int }{ - {"8192x1 at 100x0", 8192, 1, Size{Width: 100}, 100, 1}, - {"8192x1 at 1300x0", 8192, 1, Size{Width: 1300}, 1300, 1}, - {"8192x2 at 100x0", 8192, 2, Size{Width: 100}, 100, 1}, - {"1x8192 at 0x100", 1, 8192, Size{Height: 100}, 1, 100}, - {"1x8192 at 0x1300", 1, 8192, Size{Height: 1300}, 1, 1300}, - {"2x8192 at 0x100", 2, 8192, Size{Height: 100}, 1, 100}, + {"8192x1 at 100x0", 8192, 1, imagerequest.Size{Width: 100}, 100, 1}, + {"8192x1 at 1300x0", 8192, 1, imagerequest.Size{Width: 1300}, 1300, 1}, + {"8192x2 at 100x0", 8192, 2, imagerequest.Size{Width: 100}, 100, 1}, + {"1x8192 at 0x100", 1, 8192, imagerequest.Size{Height: 100}, 1, 100}, + {"1x8192 at 0x1300", 1, 8192, imagerequest.Size{Height: 1300}, 1, 1300}, + {"2x8192 at 0x100", 2, 8192, imagerequest.Size{Height: 100}, 1, 100}, } - fits := []FitMode{FitCover, FitContain, FitFill, FitInside, FitOutside} + fits := []imagerequest.Fit{ + imagerequest.FitCover, imagerequest.FitContain, imagerequest.FitFill, + imagerequest.FitInside, imagerequest.FitOutside, + } for _, tt := range tests { for _, fit := range fits { @@ -93,7 +106,11 @@ func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{Size: tt.size, Format: FormatJPEG, FitMode: fit}) + &imagerequest.Transform{ + Size: tt.size, + Format: imagerequest.FormatJPEG, + FitMode: fit, + }) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -103,14 +120,14 @@ func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) { t.Logf("Process() made %dx%d", result.Width, result.Height) ok := result.Width == tt.wantWidth && result.Height == tt.wantHeight - if fit == FitOutside { + if fit == imagerequest.FitOutside { ok = result.Width >= tt.wantWidth && result.Height >= tt.wantHeight } if !ok { t.Errorf("Process() made %dx%d, want %dx%d (at least, for %s)", result.Width, result.Height, tt.wantWidth, tt.wantHeight, - FitOutside) + imagerequest.FitOutside) } }) } diff --git a/internal/imageprocessor/deadline_internal_test.go b/internal/imageprocessor/deadline_internal_test.go index 1bde965..fb9b2ea 100644 --- a/internal/imageprocessor/deadline_internal_test.go +++ b/internal/imageprocessor/deadline_internal_test.go @@ -86,7 +86,7 @@ func checkProcessStopsAtDeadline( proc := New(Params{MaxConcurrentProcessing: 1}) - result, err := proc.Process(ctx, input, smallJPEGRequest()) + result, err := proc.Process(ctx, input, smallJPEGTransform()) if result != nil { t.Logf("Process() made a %dx%d image after its deadline passed", result.Width, result.Height) diff --git a/internal/imageprocessor/empty_format_internal_test.go b/internal/imageprocessor/empty_format_internal_test.go index 4ddda3f..4feda08 100644 --- a/internal/imageprocessor/empty_format_internal_test.go +++ b/internal/imageprocessor/empty_format_internal_test.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_EmptyFormatRefused verifies that a request with no format @@ -13,7 +15,8 @@ func TestImageProcessor_EmptyFormatRefused(t *testing.T) { t.Parallel() _, err := New(Params{}).Process( - t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), &Request{}, + t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), + &imagerequest.Transform{}, ) if !errors.Is(err, ErrUnsupportedOutputFormat) { t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat) diff --git a/internal/imageprocessor/encoder_settings_internal_test.go b/internal/imageprocessor/encoder_settings_internal_test.go index af1b31d..8cf3388 100644 --- a/internal/imageprocessor/encoder_settings_internal_test.go +++ b/internal/imageprocessor/encoder_settings_internal_test.go @@ -8,14 +8,17 @@ import ( "testing" "github.com/davidbyttow/govips/v2/vips" + "sneak.berlin/go/pixa/internal/imagerequest" ) // processedSize runs input through Process and returns the output's size in // bytes. -func processedSize(t *testing.T, input []byte, req *Request) int64 { +func processedSize( + t *testing.T, input []byte, transform *imagerequest.Transform, +) int64 { t.Helper() - result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), req) + result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -68,7 +71,8 @@ func TestImageProcessor_PNGIsCompressed(t *testing.T) { } } - size := processedSize(t, encodePNG(t, flat), &Request{Format: FormatPNG}) + size := processedSize(t, encodePNG(t, flat), + &imagerequest.Transform{Format: imagerequest.FormatPNG}) const rawBytes = width * height * 3 if size > rawBytes/10 { @@ -85,7 +89,8 @@ func TestImageProcessor_WebPAtEffort2(t *testing.T) { input := createTestJPEG(t, 200, 150) - size := processedSize(t, input, &Request{Format: FormatWebP, Quality: 85}) + size := processedSize(t, input, + &imagerequest.Transform{Format: imagerequest.FormatWebP, Quality: 85}) want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{ StripMetadata: true, @@ -126,7 +131,8 @@ func TestImageProcessor_AVIFAtEffort1(t *testing.T) { input := encodePNG(t, source) - size := processedSize(t, input, &Request{Format: FormatAVIF, Quality: 85}) + size := processedSize(t, input, + &imagerequest.Transform{Format: imagerequest.FormatAVIF, Quality: 85}) want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{ StripMetadata: true, @@ -152,7 +158,8 @@ func TestImageProcessor_JPEGXLAtEffort4(t *testing.T) { input := createTestJPEG(t, 200, 150) - size := processedSize(t, input, &Request{Format: FormatJXL, Quality: 85}) + size := processedSize(t, input, + &imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: 85}) params := vips.NewJxlExportParams() params.Quality = 85 diff --git a/internal/imageprocessor/fit_cover_internal_test.go b/internal/imageprocessor/fit_cover_internal_test.go index 0b01211..030074e 100644 --- a/internal/imageprocessor/fit_cover_internal_test.go +++ b/internal/imageprocessor/fit_cover_internal_test.go @@ -7,6 +7,8 @@ import ( "image/png" "math" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_FitCoverServesThinSource verifies that FitCover serves @@ -20,12 +22,12 @@ func TestImageProcessor_FitCoverServesThinSource(t *testing.T) { tests := []struct { name string inputWidth, inputHeight int - size Size + size imagerequest.Size }{ - {"8192x1 at 8192x8192", 8192, 1, Size{Width: 8192, Height: 8192}}, - {"8192x1 at 1x8192", 8192, 1, Size{Width: 1, Height: 8192}}, - {"1x8192 at 8192x8192", 1, 8192, Size{Width: 8192, Height: 8192}}, - {"1x8192 at 8192x1", 1, 8192, Size{Width: 8192, Height: 1}}, + {"8192x1 at 8192x8192", 8192, 1, imagerequest.Size{Width: 8192, Height: 8192}}, + {"8192x1 at 1x8192", 8192, 1, imagerequest.Size{Width: 1, Height: 8192}}, + {"1x8192 at 8192x8192", 1, 8192, imagerequest.Size{Width: 8192, Height: 8192}}, + {"1x8192 at 8192x1", 1, 8192, imagerequest.Size{Width: 8192, Height: 1}}, } for _, tt := range tests { @@ -36,7 +38,11 @@ func TestImageProcessor_FitCoverServesThinSource(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover}) + &imagerequest.Transform{ + Size: tt.size, + Format: imagerequest.FormatJPEG, + FitMode: imagerequest.FitCover, + }) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -102,10 +108,10 @@ func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{ - Size: Size{Width: 50, Height: 50}, - Format: FormatPNG, - FitMode: FitCover, + &imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatPNG, + FitMode: imagerequest.FitCover, }) if err != nil { t.Fatalf("Process() error = %v", err) @@ -208,14 +214,14 @@ func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) { tests := []struct { name string side, squareSide int - size Size + size imagerequest.Size }{ - {"16x16 at 100x60", 16, 8, Size{Width: 100, Height: 60}}, - {"16x16 at 60x100", 16, 8, Size{Width: 60, Height: 100}}, - {"10x10 at 100x35", 10, 2, Size{Width: 100, Height: 35}}, - {"10x10 at 35x100", 10, 2, Size{Width: 35, Height: 100}}, - {"64x64 at 300x100", 64, 16, Size{Width: 300, Height: 100}}, - {"64x64 at 100x300", 64, 16, Size{Width: 100, Height: 300}}, + {"16x16 at 100x60", 16, 8, imagerequest.Size{Width: 100, Height: 60}}, + {"16x16 at 60x100", 16, 8, imagerequest.Size{Width: 60, Height: 100}}, + {"10x10 at 100x35", 10, 2, imagerequest.Size{Width: 100, Height: 35}}, + {"10x10 at 35x100", 10, 2, imagerequest.Size{Width: 35, Height: 100}}, + {"64x64 at 300x100", 64, 16, imagerequest.Size{Width: 300, Height: 100}}, + {"64x64 at 100x300", 64, 16, imagerequest.Size{Width: 100, Height: 300}}, } for _, tt := range tests { @@ -226,7 +232,11 @@ func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{Size: tt.size, Format: FormatPNG, FitMode: FitCover}) + &imagerequest.Transform{ + Size: tt.size, + Format: imagerequest.FormatPNG, + FitMode: imagerequest.FitCover, + }) if err != nil { t.Fatalf("Process() error = %v", err) } diff --git a/internal/imageprocessor/fit_outside_internal_test.go b/internal/imageprocessor/fit_outside_internal_test.go index 62d9674..3d331a5 100644 --- a/internal/imageprocessor/fit_outside_internal_test.go +++ b/internal/imageprocessor/fit_outside_internal_test.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_RefusesFitOutsideAboveMaximum verifies that FitOutside, @@ -17,15 +19,20 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) { tests := []struct { name string inputWidth, inputHeight int - size Size + size imagerequest.Size // wantWidth and wantHeight are the output's, 0 for a refusal wantWidth, wantHeight int }{ - {"width above the maximum", 3000, 600, Size{Width: 1, Height: 8192}, 0, 0}, - {"height above the maximum", 600, 3000, Size{Width: 8192, Height: 1}, 0, 0}, - {"width at the maximum", 1024, 256, Size{Width: 1, Height: 2048}, 8192, 2048}, - {"height at the maximum", 256, 1024, Size{Width: 2048, Height: 1}, 2048, 8192}, - {"within the maximum", 3000, 600, Size{Width: 1, Height: 1200}, 6000, 1200}, + {"width above the maximum", 3000, 600, + imagerequest.Size{Width: 1, Height: 8192}, 0, 0}, + {"height above the maximum", 600, 3000, + imagerequest.Size{Width: 8192, Height: 1}, 0, 0}, + {"width at the maximum", 1024, 256, + imagerequest.Size{Width: 1, Height: 2048}, 8192, 2048}, + {"height at the maximum", 256, 1024, + imagerequest.Size{Width: 2048, Height: 1}, 2048, 8192}, + {"within the maximum", 3000, 600, + imagerequest.Size{Width: 1, Height: 1200}, 6000, 1200}, } for _, tt := range tests { @@ -36,7 +43,11 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside}) + &imagerequest.Transform{ + Size: tt.size, + Format: imagerequest.FormatJPEG, + FitMode: imagerequest.FitOutside, + }) if result != nil { t.Logf("Process() made %dx%d", result.Width, result.Height) } @@ -71,11 +82,11 @@ func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) { tests := []struct { name string inputWidth, inputHeight int - size Size + size imagerequest.Size wantWidth, wantHeight int }{ - {"49x8192 at 0x100", 49, 8192, Size{Height: 100}, 1, 167}, - {"49x49 at 1x1", 49, 49, Size{Width: 1, Height: 1}, 1, 1}, + {"49x8192 at 0x100", 49, 8192, imagerequest.Size{Height: 100}, 1, 167}, + {"49x49 at 1x1", 49, 49, imagerequest.Size{Width: 1, Height: 1}, 1, 1}, } for _, tt := range tests { @@ -86,7 +97,11 @@ func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) { result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), - &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside}) + &imagerequest.Transform{ + Size: tt.size, + Format: imagerequest.FormatJPEG, + FitMode: imagerequest.FitOutside, + }) if err != nil { t.Fatalf("Process() error = %v", err) } diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index c50452b..c344eea 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -12,6 +12,7 @@ import ( "github.com/davidbyttow/govips/v2/vips" "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/imagerequest" ) // vipsOnce ensures vips is initialized exactly once. @@ -54,49 +55,6 @@ func CheckJPEGXLSupport() error { return nil } -// Format represents supported output image formats. -type Format string - -// Supported image output formats. -const ( - FormatOriginal Format = "orig" - FormatJPEG Format = "jpeg" - FormatPNG Format = "png" - FormatWebP Format = "webp" - FormatAVIF Format = "avif" - FormatJXL Format = "jxl" - FormatGIF Format = "gif" -) - -// FitMode represents how to fit an image into requested dimensions. -type FitMode string - -// Supported image fit modes. -const ( - FitCover FitMode = "cover" - FitContain FitMode = "contain" - FitFill FitMode = "fill" - FitInside FitMode = "inside" - FitOutside FitMode = "outside" -) - -// ErrInvalidFitMode is returned when an invalid fit mode is provided. -var ErrInvalidFitMode = errors.New("invalid fit mode") - -// Size represents requested image dimensions. -type Size struct { - Width int - Height int -} - -// Request holds the parameters for image processing. -type Request struct { - Size Size - Format Format - Quality int - FitMode FitMode -} - // Result contains the output of image processing. type Result struct { // Content is the processed image data. @@ -133,8 +91,8 @@ var ErrInputTooLarge = errors.New("input image dimensions exceed maximum") var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size") // MaxOutputDimension is the maximum width or height of an output image, and -// so of a requested size (imgcache.MaxDimension). -const MaxOutputDimension = 8192 +// so of a requested size (imagerequest.MaxDimension). +const MaxOutputDimension = imagerequest.MaxDimension // ErrOutputTooLarge is returned, before resizing, when the output would have a // side above MaxOutputDimension: the side of a size with one side 0, computed @@ -217,7 +175,7 @@ func New(params Params) *ImageProcessor { } } -// Process transforms an image according to the request. When +// Process transforms an image according to transform. When // MaxConcurrentProcessing images are already being processed, it waits up // to ProcessingWaitTimeout for one to finish, then fails with // ErrTooManyImages. When ctx has ended once Process has a slot, or before it @@ -228,7 +186,7 @@ func New(params Params) *ImageProcessor { func (p *ImageProcessor) Process( ctx context.Context, input io.Reader, - req *Request, + transform *imagerequest.Transform, ) (*Result, error) { release, err := p.acquireSlot(ctx) if err != nil { @@ -277,35 +235,36 @@ func (p *ImageProcessor) Process( } // Determine target dimensions; too large ones are refused before resizing - targetWidth, targetHeight, err := targetDimensions(req.Size, origWidth, origHeight) + targetWidth, targetHeight, err := targetDimensions(transform.Size, + origWidth, origHeight) if err != nil { return nil, err } // orig is the source's own format; encode refuses an empty format - outputFormat := req.Format - if outputFormat == FormatOriginal { + outputFormat := transform.Format + if outputFormat == imagerequest.FormatOriginal { outputFormat = sourceFormat } // Determine output dimensions; too large ones are refused before // resizing, which with FitCover already decodes the input outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight, - targetWidth, targetHeight, req.FitMode, outputFormat) + targetWidth, targetHeight, transform.FitMode, outputFormat) if err != nil { return nil, err } // Resize if needed if targetWidth != origWidth || targetHeight != origHeight { - err := p.resize(img, outputWidth, outputHeight, req.FitMode) + err := p.resize(img, outputWidth, outputHeight, transform.FitMode) if err != nil { return nil, fmt.Errorf("failed to resize: %w", err) } } // Encode to target format - output, err := p.encode(ctx, img, outputFormat, req.Quality) + output, err := p.encode(ctx, img, outputFormat, transform.Quality) if err != nil { return nil, fmt.Errorf("failed to encode: %w", err) } @@ -328,7 +287,9 @@ func (p *ImageProcessor) Process( // rounded down but kept at 1 or more, as for a thin image it would round // down to 0, a side of no pixels. It can come out above MaxOutputDimension, // which is ErrOutputTooLarge. -func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) { +func targetDimensions( + size imagerequest.Size, origWidth, origHeight int, +) (int, int, error) { width, height := size.Width, size.Height switch { @@ -374,33 +335,33 @@ func (p *ImageProcessor) SupportedInputFormats() []string { } // SupportedOutputFormats returns formats this processor can write. -func (p *ImageProcessor) SupportedOutputFormats() []Format { - return []Format{ - FormatJPEG, - FormatPNG, - FormatGIF, - FormatWebP, - FormatAVIF, - FormatJXL, +func (p *ImageProcessor) SupportedOutputFormats() []imagerequest.Format { + return []imagerequest.Format{ + imagerequest.FormatJPEG, + imagerequest.FormatPNG, + imagerequest.FormatGIF, + imagerequest.FormatWebP, + imagerequest.FormatAVIF, + imagerequest.FormatJXL, } } // FormatToMIME converts a Format to its MIME type string. -func FormatToMIME(format Format) string { +func FormatToMIME(format imagerequest.Format) string { switch format { - case FormatJPEG: + case imagerequest.FormatJPEG: return mimeJPEG - case FormatPNG: + case imagerequest.FormatPNG: return mimePNG - case FormatWebP: + case imagerequest.FormatWebP: return mimeWebP - case FormatGIF: + case imagerequest.FormatGIF: return mimeGIF - case FormatAVIF: + case imagerequest.FormatAVIF: return mimeAVIF - case FormatJXL: + case imagerequest.FormatJXL: return mimeJXL - case FormatOriginal: + case imagerequest.FormatOriginal, imagerequest.FormatAuto: return "application/octet-stream" default: return "application/octet-stream" @@ -487,9 +448,9 @@ func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { case vips.ImageTypeWEBP: return "webp" case vips.ImageTypeAVIF, vips.ImageTypeHEIF: - return string(FormatAVIF) + return string(imagerequest.FormatAVIF) case vips.ImageTypeJXL: - return string(FormatJXL) + return string(imagerequest.FormatJXL) case vips.ImageTypeUnknown, vips.ImageTypeMagick, vips.ImageTypePDF, vips.ImageTypeSVG, vips.ImageTypeTIFF, vips.ImageTypeBMP, vips.ImageTypeJP2K: @@ -506,17 +467,18 @@ func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { // outputDimensions returns ErrOutputTooLarge. An AVIF output of more than // MaxAVIFOutputPixels is ErrAVIFOutputTooLarge. func outputDimensions( - imgWidth, imgHeight, width, height int, fit FitMode, format Format, + imgWidth, imgHeight, width, height int, + fit imagerequest.Fit, format imagerequest.Format, ) (int, int, error) { scaleW := float64(width) / float64(imgWidth) scaleH := float64(height) / float64(imgHeight) outputWidth, outputHeight := width, height switch fit { - case FitCover, FitFill, "": + case imagerequest.FitCover, imagerequest.FitFill, "": // Cover crops, and fill stretches, to exactly the target dimensions - case FitContain: + case imagerequest.FitContain: // Fit within the target dimensions, maintaining aspect ratio. Scaled // by the smaller scale, the short side of a thin image can round // down to 0, which libvips refuses as a width, so each side is at @@ -525,13 +487,13 @@ func outputDimensions( outputWidth = max(1, int(float64(imgWidth)*scale)) outputHeight = max(1, int(float64(imgHeight)*scale)) - case FitInside: + case imagerequest.FitInside: // Same as contain, but only shrink scale := min(scaleW, scaleH, 1) outputWidth = max(1, int(float64(imgWidth)*scale)) outputHeight = max(1, int(float64(imgHeight)*scale)) - case FitOutside: + case imagerequest.FitOutside: // The smallest dimension fits, the other may exceed the target. A // side scaled to 1 can come out just under 1 in floating point, as // 49 times 1/49 does, and round down to 0, which libvips refuses as @@ -541,7 +503,7 @@ func outputDimensions( outputHeight = max(1, int(float64(imgHeight)*scale)) default: - return 0, 0, fmt.Errorf("%w: %s", ErrInvalidFitMode, fit) + return 0, 0, fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, fit) } // The side FitOutside scales past the target is held to the maximum too @@ -550,7 +512,8 @@ func outputDimensions( outputWidth, outputHeight, MaxOutputDimension) } - if format == FormatAVIF && outputWidth*outputHeight > MaxAVIFOutputPixels { + if format == imagerequest.FormatAVIF && + outputWidth*outputHeight > MaxAVIFOutputPixels { return 0, 0, fmt.Errorf("%w: %dx%d is above %d pixels", ErrAVIFOutputTooLarge, outputWidth, outputHeight, MaxAVIFOutputPixels) } @@ -561,18 +524,18 @@ func outputDimensions( // resize resizes the image to width x height, the dimensions // outputDimensions gives for the fit mode. func (p *ImageProcessor) resize( - img *vips.ImageRef, width, height int, fit FitMode, + img *vips.ImageRef, width, height int, fit imagerequest.Fit, ) error { switch fit { - case FitCover, "": + case imagerequest.FitCover, "": // Crop the centre and resize to fill exact dimensions (default) return cropCentreAndResize(img, width, height) - case FitFill: + case imagerequest.FitFill: // Resize to exact dimensions (may distort) return img.ThumbnailWithSize(width, height, vips.InterestingNone, vips.SizeForce) - case FitInside: + case imagerequest.FitInside: // outputDimensions keeps the dimensions of an image that already fits if img.Width() == width && img.Height() == height { return nil @@ -580,11 +543,11 @@ func (p *ImageProcessor) resize( return img.Thumbnail(width, height, vips.InterestingNone) - case FitContain, FitOutside: + case imagerequest.FitContain, imagerequest.FitOutside: return img.Thumbnail(width, height, vips.InterestingNone) default: - return fmt.Errorf("%w: %s", ErrInvalidFitMode, fit) + return fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, fit) } } @@ -626,7 +589,8 @@ const defaultQuality = 85 // to stop libvips once an encode has begun, so when ctx has ended encode // returns ctx's error without beginning one. func (p *ImageProcessor) encode( - ctx context.Context, img *vips.ImageRef, format Format, quality int, + ctx context.Context, img *vips.ImageRef, format imagerequest.Format, + quality int, ) ([]byte, error) { err := ctx.Err() if err != nil { @@ -649,25 +613,25 @@ func (p *ImageProcessor) encode( } switch format { - case FormatJPEG: + case imagerequest.FormatJPEG: return exportJPEG(img, quality) - case FormatPNG: + case imagerequest.FormatPNG: return exportPNG(img) - case FormatGIF: + case imagerequest.FormatGIF: return exportGIF(img) - case FormatWebP: + case imagerequest.FormatWebP: return exportWebP(img, quality) - case FormatAVIF: + case imagerequest.FormatAVIF: return exportAVIF(img, quality) - case FormatJXL: + case imagerequest.FormatJXL: return exportJXL(img, quality) - case FormatOriginal: + case imagerequest.FormatOriginal, imagerequest.FormatAuto: return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format) default: @@ -849,20 +813,22 @@ func jxlDistance(quality int) float64 { // formatFromString converts a format string from detectFormat to Format. A // format detectFormat could not name is ErrUnsupportedInputFormat. -func (p *ImageProcessor) formatFromString(format string) (Format, error) { +func (p *ImageProcessor) formatFromString( + format string, +) (imagerequest.Format, error) { switch format { case "jpeg": - return FormatJPEG, nil + return imagerequest.FormatJPEG, nil case "png": - return FormatPNG, nil + return imagerequest.FormatPNG, nil case "gif": - return FormatGIF, nil + return imagerequest.FormatGIF, nil case "webp": - return FormatWebP, nil - case string(FormatAVIF): - return FormatAVIF, nil - case string(FormatJXL): - return FormatJXL, nil + return imagerequest.FormatWebP, nil + case string(imagerequest.FormatAVIF): + return imagerequest.FormatAVIF, nil + case string(imagerequest.FormatJXL): + return imagerequest.FormatJXL, nil default: return "", fmt.Errorf("%w: %s", ErrUnsupportedInputFormat, format) } diff --git a/internal/imageprocessor/imageprocessor_internal_test.go b/internal/imageprocessor/imageprocessor_internal_test.go index 73decd0..f5a865e 100644 --- a/internal/imageprocessor/imageprocessor_internal_test.go +++ b/internal/imageprocessor/imageprocessor_internal_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/davidbyttow/govips/v2/vips" + "sneak.berlin/go/pixa/internal/imagerequest" ) func TestMain(m *testing.M) { @@ -87,7 +88,7 @@ func isAVIF(data []byte) bool { brand := string(data[8:12]) - return brand == string(FormatAVIF) || brand == "avis" + return brand == string(imagerequest.FormatAVIF) || brand == "avis" } // detectMIME is a minimal magic-byte detector for test assertions. @@ -123,14 +124,14 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) { input := createTestJPEG(t, 800, 600) - req := &Request{ - Size: Size{Width: 400, Height: 300}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 400, Height: 300}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -169,13 +170,13 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) { input := createTestJPEG(t, 200, 150) - req := &Request{ - Size: Size{Width: 200, Height: 150}, - Format: FormatPNG, - FitMode: FitCover, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 200, Height: 150}, + Format: imagerequest.FormatPNG, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -196,7 +197,7 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) { // processAndCheckSize processes a test JPEG of the given input dimensions // with the requested size and asserts the resulting dimensions. func processAndCheckSize( - t *testing.T, inputW, inputH int, size Size, wantW, wantH int, + t *testing.T, inputW, inputH int, size imagerequest.Size, wantW, wantH int, ) { t.Helper() @@ -205,14 +206,14 @@ func processAndCheckSize( input := createTestJPEG(t, inputW, inputH) - req := &Request{ + transform := &imagerequest.Transform{ Size: size, - Format: FormatJPEG, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -232,7 +233,7 @@ func TestImageProcessor_OriginalSize(t *testing.T) { t.Parallel() // Width and height 0: keep original size - processAndCheckSize(t, 640, 480, Size{Width: 0, Height: 0}, 640, 480) + processAndCheckSize(t, 640, 480, imagerequest.Size{Width: 0, Height: 0}, 640, 480) } func TestImageProcessor_FitContain(t *testing.T) { @@ -245,14 +246,14 @@ func TestImageProcessor_FitContain(t *testing.T) { // Should result in 400x200 (maintaining aspect ratio) input := createTestJPEG(t, 800, 400) - req := &Request{ - Size: Size{Width: 400, Height: 400}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 400, Height: 400}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitContain, + FitMode: imagerequest.FitContain, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -270,7 +271,7 @@ func TestImageProcessor_ProportionalScale_WidthOnly(t *testing.T) { // 800x600 image, request width=400 height=0 // Should scale proportionally to 400x300 - processAndCheckSize(t, 800, 600, Size{Width: 400, Height: 0}, 400, 300) + processAndCheckSize(t, 800, 600, imagerequest.Size{Width: 400, Height: 0}, 400, 300) } func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) { @@ -278,7 +279,7 @@ func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) { // 800x600 image, request width=0 height=300 // Should scale proportionally to 400x300 - processAndCheckSize(t, 800, 600, Size{Width: 0, Height: 300}, 400, 300) + processAndCheckSize(t, 800, 600, imagerequest.Size{Width: 0, Height: 300}, 400, 300) } func TestImageProcessor_ProcessPNG(t *testing.T) { @@ -289,13 +290,13 @@ func TestImageProcessor_ProcessPNG(t *testing.T) { input := createTestPNG(t, 400, 300) - req := &Request{ - Size: Size{Width: 200, Height: 150}, - Format: FormatPNG, - FitMode: FitCover, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 200, Height: 150}, + Format: imagerequest.FormatPNG, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -349,14 +350,14 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { ctx := context.Background() input := createTestJPEG(t, tt.width, tt.height) - req := &Request{ - Size: Size{Width: 100, Height: 100}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 100}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - _, err := proc.Process(ctx, bytes.NewReader(input), req) + _, err := proc.Process(ctx, bytes.NewReader(input), transform) if err == nil { t.Error("Process() should reject oversized input images") } @@ -377,14 +378,14 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { // Create an image at exactly MaxInputDimension - should be accepted input := createTestJPEG(t, MaxInputDimension, 100) - req := &Request{ - Size: Size{Width: 100, Height: 100}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 100}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf( "Process() should accept images at MaxInputDimension, got error: %v", @@ -397,7 +398,9 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { // encodeAndCheck processes a 200x150 test JPEG into a 100x75 output of the // given format and asserts the output MIME type and dimensions. -func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) { +func encodeAndCheck( + t *testing.T, format imagerequest.Format, quality int, wantMIME string, +) { t.Helper() proc := New(Params{}) @@ -405,14 +408,14 @@ func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) { input := createTestJPEG(t, 200, 150) - req := &Request{ - Size: Size{Width: 100, Height: 75}, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 75}, Format: format, Quality: quality, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v, want nil", err) } @@ -443,7 +446,7 @@ func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) { func TestImageProcessor_EncodeWebP(t *testing.T) { t.Parallel() - encodeAndCheck(t, FormatWebP, 80, mimeWebP) + encodeAndCheck(t, imagerequest.FormatWebP, 80, mimeWebP) } func TestImageProcessor_DecodeAVIF(t *testing.T) { @@ -459,14 +462,14 @@ func TestImageProcessor_DecodeAVIF(t *testing.T) { } // Request resize and convert to JPEG - req := &Request{ - Size: Size{Width: 2, Height: 2}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 2, Height: 2}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v, want nil (AVIF decoding should work)", err) } @@ -500,14 +503,14 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) { t.Fatalf("test JPEG must exceed %d bytes, got %d", limit, len(input)) } - req := &Request{ - Size: Size{Width: 100, Height: 75}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 75}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - _, err := proc.Process(ctx, bytes.NewReader(input), req) + _, err := proc.Process(ctx, bytes.NewReader(input), transform) if err == nil { t.Fatal("Process() should reject input exceeding maxInputBytes") } @@ -527,14 +530,14 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) { proc := New(Params{MaxInputBytes: limit}) ctx := context.Background() - req := &Request{ - Size: Size{Width: 10, Height: 10}, - Format: FormatJPEG, + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 10, Height: 10}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), req) + result, err := proc.Process(ctx, bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v, want nil", err) } @@ -561,16 +564,18 @@ func TestImageProcessor_DefaultMaxInputBytes(t *testing.T) { func TestImageProcessor_EncodeAVIF(t *testing.T) { t.Parallel() - encodeAndCheck(t, FormatAVIF, 85, mimeAVIF) + encodeAndCheck(t, imagerequest.FormatAVIF, 85, mimeAVIF) } // processAndDecode runs input through Process and decodes the output with // vips, so a test can inspect the image a client would receive. -func processAndDecode(t *testing.T, input []byte, req *Request) *vips.ImageRef { +func processAndDecode( + t *testing.T, input []byte, transform *imagerequest.Transform, +) *vips.ImageRef { t.Helper() result, err := New(Params{}).Process( - context.Background(), bytes.NewReader(input), req, + context.Background(), bytes.NewReader(input), transform, ) if err != nil { t.Fatalf("Process() error = %v", err) @@ -614,15 +619,17 @@ func TestImageProcessor_StripsEXIF(t *testing.T) { t.Fatal("testdata/gps-exif.jpg has no GPS latitude") } - formats := []Format{ - FormatJPEG, FormatPNG, FormatWebP, FormatAVIF, FormatGIF, FormatOriginal, + formats := []imagerequest.Format{ + imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatWebP, + imagerequest.FormatAVIF, imagerequest.FormatGIF, imagerequest.FormatOriginal, } for _, format := range formats { t.Run(string(format), func(t *testing.T) { t.Parallel() - output := processAndDecode(t, input, &Request{Format: format}) + output := processAndDecode(t, input, + &imagerequest.Transform{Format: format}) if output.HasExif() { t.Errorf("output has EXIF: %v", output.GetExif()) @@ -644,21 +651,21 @@ func TestImageProcessor_AppliesEXIFOrientation(t *testing.T) { tests := []struct { name string - size Size + size imagerequest.Size wantW int wantH int }{ - {name: "original size", size: Size{}, wantW: 8, wantH: 16}, - {name: "width only", size: Size{Width: 4}, wantW: 4, wantH: 8}, + {name: "original size", size: imagerequest.Size{}, wantW: 8, wantH: 16}, + {name: "width only", size: imagerequest.Size{Width: 4}, wantW: 4, wantH: 8}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - output := processAndDecode(t, input, &Request{ + output := processAndDecode(t, input, &imagerequest.Transform{ Size: tt.size, - Format: FormatPNG, + Format: imagerequest.FormatPNG, }) if output.Width() != tt.wantW || output.Height() != tt.wantH { @@ -694,7 +701,8 @@ func TestImageProcessor_ConvertsWideGamutToSRGB(t *testing.T) { t.Fatalf("failed to read test JPEG: %v", err) } - output := processAndDecode(t, input, &Request{Format: FormatPNG}) + output := processAndDecode(t, input, + &imagerequest.Transform{Format: imagerequest.FormatPNG}) if output.HasICCProfile() { t.Error("output has an ICC profile") diff --git a/internal/imageprocessor/jpegxl_output_internal_test.go b/internal/imageprocessor/jpegxl_output_internal_test.go index 71f8f14..44b3f6a 100644 --- a/internal/imageprocessor/jpegxl_output_internal_test.go +++ b/internal/imageprocessor/jpegxl_output_internal_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/davidbyttow/govips/v2/vips" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_EncodeJPEGXL converts a JPEG to a smaller JPEG XL and @@ -15,10 +16,13 @@ import ( func TestImageProcessor_EncodeJPEGXL(t *testing.T) { t.Parallel() - req := &Request{Size: Size{Width: 100, Height: 75}, Format: FormatJXL} + transform := &imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 75}, + Format: imagerequest.FormatJXL, + } result, err := New(Params{}).Process( - t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), req, + t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), transform, ) if err != nil { t.Fatalf("Process() error = %v", err) @@ -61,7 +65,8 @@ func TestImageProcessor_JPEGXLQuality(t *testing.T) { for _, quality := range []int{30, 90} { result, err := New(Params{}).Process(t.Context(), - bytes.NewReader(input), &Request{Format: FormatJXL, Quality: quality}) + bytes.NewReader(input), + &imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: quality}) if err != nil { t.Fatalf("Process() at quality %d error = %v", quality, err) } @@ -89,7 +94,8 @@ func TestImageProcessor_JPEGXLDropsSourceEXIF(t *testing.T) { t.Fatalf("failed to read test JPEG: %v", err) } - exif := processAndDecode(t, input, &Request{Format: FormatJXL}).GetExif() + exif := processAndDecode(t, input, + &imagerequest.Transform{Format: imagerequest.FormatJXL}).GetExif() for _, field := range []string{ "exif-ifd0-Make", "exif-ifd0-Model", "exif-ifd2-BodySerialNumber", @@ -115,7 +121,8 @@ func TestImageProcessor_JPEGXLDropsSourceResolution(t *testing.T) { t.Fatalf("failed to read test JPEG: %v", err) } - output := processAndDecode(t, input, &Request{Format: FormatJXL}) + output := processAndDecode(t, input, + &imagerequest.Transform{Format: imagerequest.FormatJXL}) // libvips gives the resolution in pixels per millimetre. xDPI := math.Round(output.ResX() * 25.4) @@ -140,7 +147,8 @@ func TestImageProcessor_JPEGXLAppliesEXIFOrientation(t *testing.T) { t.Fatalf("failed to read test JPEG: %v", err) } - output := processAndDecode(t, input, &Request{Format: FormatJXL}) + output := processAndDecode(t, input, + &imagerequest.Transform{Format: imagerequest.FormatJXL}) if output.Width() != 8 || output.Height() != 16 { t.Fatalf("output is %dx%d, want 8x16", output.Width(), output.Height()) @@ -176,7 +184,8 @@ func TestImageProcessor_JPEGXLConvertsWideGamutToSRGB(t *testing.T) { t.Fatalf("failed to read test JPEG: %v", err) } - output := processAndDecode(t, input, &Request{Format: FormatJXL}) + output := processAndDecode(t, input, + &imagerequest.Transform{Format: imagerequest.FormatJXL}) pixel, err := output.GetPoint(4, 4) if err != nil { @@ -203,7 +212,8 @@ func TestImageProcessor_JPEGXLFromCMYK(t *testing.T) { t.Fatalf("failed to read test JPEG: %v", err) } - output := processAndDecode(t, input, &Request{Format: FormatJXL}) + output := processAndDecode(t, input, + &imagerequest.Transform{Format: imagerequest.FormatJXL}) if output.Bands() != 3 { t.Fatalf("output has %d bands, want 3", output.Bands()) diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go index 2dbbbbf..c3d428e 100644 --- a/internal/imageprocessor/max_concurrent_processing_internal_test.go +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -11,6 +11,8 @@ import ( "testing" "testing/iotest" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // errTestReadFailed is the error the unreadable test input returns. @@ -76,13 +78,13 @@ func (r *gatedReader) Read(p []byte) (int, error) { return n, err } -// smallJPEGRequest asks for a 5x5 JPEG. -func smallJPEGRequest() *Request { - return &Request{ - Size: Size{Width: 5, Height: 5}, - Format: FormatJPEG, +// smallJPEGTransform asks for a 5x5 JPEG. +func smallJPEGTransform() *imagerequest.Transform { + return &imagerequest.Transform{ + Size: imagerequest.Size{Width: 5, Height: 5}, + Format: imagerequest.FormatJPEG, Quality: 85, - FitMode: FitCover, + FitMode: imagerequest.FitCover, } } @@ -92,7 +94,7 @@ func processInBackground( proc *ImageProcessor, reader *gatedReader, results chan<- error, ) { go func() { - result, err := proc.Process(context.Background(), reader, smallJPEGRequest()) + result, err := proc.Process(context.Background(), reader, smallJPEGTransform()) if err == nil { _ = result.Content.Close() } @@ -237,7 +239,7 @@ func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { start := time.Now() _, err := proc.Process(context.Background(), bytes.NewReader(input), - smallJPEGRequest()) + smallJPEGTransform()) if !errors.Is(err, ErrTooManyImages) { t.Fatalf("Process() error = %v, want ErrTooManyImages", err) } @@ -263,38 +265,38 @@ func TestProcessReleasesSlotOnError(t *testing.T) { valid := createTestJPEG(t, 10, 10) - unsupported := smallJPEGRequest() + unsupported := smallJPEGTransform() unsupported.Format = "bmp" cases := []struct { - name string - input io.Reader - req *Request + name string + input io.Reader + transform *imagerequest.Transform // want is the error Process must return; nil means any error. want error }{ { - name: "input cannot be read", - input: iotest.ErrReader(errTestReadFailed), - req: smallJPEGRequest(), - want: errTestReadFailed, + name: "input cannot be read", + input: iotest.ErrReader(errTestReadFailed), + transform: smallJPEGTransform(), + want: errTestReadFailed, }, { - name: "input over the byte limit", - input: bytes.NewReader(createTestJPEG(t, 800, 600)), - req: smallJPEGRequest(), - want: ErrInputDataTooLarge, + name: "input over the byte limit", + input: bytes.NewReader(createTestJPEG(t, 800, 600)), + transform: smallJPEGTransform(), + want: ErrInputDataTooLarge, }, { - name: "input not an image", - input: strings.NewReader("not an image"), - req: smallJPEGRequest(), + name: "input not an image", + input: strings.NewReader("not an image"), + transform: smallJPEGTransform(), }, { - name: "output format not supported", - input: bytes.NewReader(valid), - req: unsupported, - want: ErrUnsupportedOutputFormat, + name: "output format not supported", + input: bytes.NewReader(valid), + transform: unsupported, + want: ErrUnsupportedOutputFormat, }, } @@ -305,7 +307,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) { proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1}) proc.processingWaitTimeout = 100 * time.Millisecond - _, err := proc.Process(context.Background(), tc.input, tc.req) + _, err := proc.Process(context.Background(), tc.input, tc.transform) if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { t.Fatalf("Process() error = %v, want %v", err, tc.want) } @@ -315,7 +317,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) { } result, err := proc.Process(context.Background(), bytes.NewReader(valid), - smallJPEGRequest()) + smallJPEGTransform()) if err != nil { t.Fatalf("Process() after the error = %v, want nil", err) } diff --git a/internal/imageprocessor/orig_format_internal_test.go b/internal/imageprocessor/orig_format_internal_test.go index 453d176..a0ad8dc 100644 --- a/internal/imageprocessor/orig_format_internal_test.go +++ b/internal/imageprocessor/orig_format_internal_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/davidbyttow/govips/v2/vips" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF @@ -27,7 +28,8 @@ func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) { } result, err := New(Params{}).Process( - t.Context(), bytes.NewReader(tiff), &Request{Format: FormatOriginal}, + t.Context(), bytes.NewReader(tiff), + &imagerequest.Transform{Format: imagerequest.FormatOriginal}, ) if result != nil { t.Logf("Process() returned %s", result.ContentType) diff --git a/internal/imageprocessor/source_formats_internal_test.go b/internal/imageprocessor/source_formats_internal_test.go index 5f1dbbd..d724976 100644 --- a/internal/imageprocessor/source_formats_internal_test.go +++ b/internal/imageprocessor/source_formats_internal_test.go @@ -4,6 +4,8 @@ import ( "bytes" "io" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProcessor_LoadsEachSourceFormat verifies that an image in each of @@ -14,11 +16,13 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { proc := New(Params{}) - for _, format := range []Format{ - FormatJPEG, FormatPNG, FormatGIF, FormatWebP, FormatAVIF, FormatJXL, + for _, format := range []imagerequest.Format{ + imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, + imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, } { saved, err := proc.Process(t.Context(), - bytes.NewReader(createTestPNG(t, 20, 20)), &Request{Format: format}) + bytes.NewReader(createTestPNG(t, 20, 20)), + &imagerequest.Transform{Format: format}) if err != nil { t.Fatalf("saving the test image as %s: %v", format, err) } @@ -29,7 +33,7 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { } result, err := proc.Process(t.Context(), bytes.NewReader(source), - &Request{Format: FormatOriginal}) + &imagerequest.Transform{Format: imagerequest.FormatOriginal}) if err != nil { t.Errorf("orig of the %s image: %v", format, err) diff --git a/internal/imagerequest/imagerequest.go b/internal/imagerequest/imagerequest.go new file mode 100644 index 0000000..7b30a61 --- /dev/null +++ b/internal/imagerequest/imagerequest.go @@ -0,0 +1,147 @@ +// Package imagerequest holds the image request type, its parts, its checks, +// the parsing of the /v1/image/ path and the variant key. It imports no other +// package of this repo, so every package that handles an image request can +// import it. +package imagerequest + +import ( + "errors" + "net/url" + "time" +) + +// ErrInvalidFitMode is returned when an invalid fit mode is provided. +var ErrInvalidFitMode = errors.New("invalid fit mode") + +// Format represents supported output image formats. +type Format string + +// Supported image output formats. +const ( + FormatOriginal Format = "orig" + FormatJPEG Format = "jpeg" + FormatPNG Format = "png" + FormatWebP Format = "webp" + FormatAVIF Format = "avif" + FormatJXL Format = "jxl" + FormatGIF Format = "gif" + + // FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each + // request from its Accept header once the URL's signature or token has + // been checked; it is never processed or cached as itself. + FormatAuto Format = "auto" +) + +// Size represents requested image dimensions +type Size struct { + Width int + Height int +} + +// OriginalSize returns true if this represents "keep original size" +func (s Size) OriginalSize() bool { + return s.Width == 0 && s.Height == 0 +} + +// Fit represents how to fit image into requested dimensions. +type Fit string + +// Supported image fit modes. +const ( + FitCover Fit = "cover" + FitContain Fit = "contain" + FitFill Fit = "fill" + FitInside Fit = "inside" + FitOutside Fit = "outside" +) + +// ValidateFitMode checks if the given fit mode is valid. +// Returns ErrInvalidFitMode for unrecognized fit modes. +func ValidateFitMode(fit Fit) error { + switch fit { + case FitCover, FitContain, FitFill, FitInside, FitOutside, "": + return nil + default: + return ErrInvalidFitMode + } +} + +// ValidateImageRequest checks a request's width and height with +// ValidateDimension and its fit mode with ValidateFitMode. Both the plain +// /v1/image/ route and the encrypted /v1/e/ route validate through this +// function so a request from either source enforces identical bounds. +func ValidateImageRequest(req *Request) error { + err := ValidateDimension("width", req.Transform.Size.Width) + if err != nil { + return err + } + + err = ValidateDimension("height", req.Transform.Size.Height) + if err != nil { + return err + } + + return ValidateFitMode(req.Transform.FitMode) +} + +// Request represents a request for a processed image: the source to fetch +// and the transform to apply to it. +type Request struct { + // Source is what to fetch + Source Source + // Transform is what to do to it + Transform Transform + // Signature is the HMAC signature for non-allowlisted hosts + Signature string + // Expires is when the URL expires: the exp of a signed URL, or the expiry + // of an encrypted URL; the zero time if it has none + Expires time.Time + // AllowHTTP indicates whether HTTP (non-TLS) is allowed for this request + AllowHTTP bool +} + +// Source is the image a request fetches from the origin. +type Source struct { + // Host is the origin host (e.g., "cdn.example.com") + Host string + // Path is the path on the origin with its percent-escapes decoded, as + // url.URL's Path holds it (e.g., "/photos/my cat.jpg") + Path string + // Query is the optional query string for the origin URL as the URL + // writes it, percent-escapes included, as url.URL's RawQuery holds it + Query string +} + +// Transform is what a request does to its source; the image processor +// takes it alone. +type Transform struct { + // Size is the requested output dimensions + Size Size + // Format is the requested output format + Format Format + // Quality is the output quality (1-100) for lossy formats + Quality int + // FitMode is how to fit the image into requested dimensions + FitMode Fit +} + +// SourceURL returns the full upstream URL to fetch: the source path +// percent-encoded again, as net/url writes a path, so each source path is one +// upstream path, and the source query as it is, with the characters +// escapeQuery encodes percent-encoded. Uses http:// scheme when AllowHTTP is +// true, otherwise https://. +func (r *Request) SourceURL() string { + scheme := "https" + if r.AllowHTTP { + scheme = "http" + } + + sourceURL := url.URL{ + Scheme: scheme, + Host: r.Source.Host, + Path: r.Source.Path, + RawQuery: escapeQuery(r.Source.Query), + } + + return sourceURL.String() +} diff --git a/internal/imgcache/sourceurl_internal_test.go b/internal/imagerequest/sourceurl_internal_test.go similarity index 50% rename from internal/imgcache/sourceurl_internal_test.go rename to internal/imagerequest/sourceurl_internal_test.go index 36bd3fe..6e6b671 100644 --- a/internal/imgcache/sourceurl_internal_test.go +++ b/internal/imagerequest/sourceurl_internal_test.go @@ -1,14 +1,16 @@ -package imgcache +package imagerequest import "testing" -func TestImageRequest_SourceURL_DefaultHTTPS(t *testing.T) { +func TestRequest_SourceURL_DefaultHTTPS(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - SourceQuery: "v=2", + req := &Request{ + Source: Source{ + Host: testHostCDN, + Path: testPathCat, + Query: "v=2", + }, } got := req.SourceURL() @@ -19,13 +21,15 @@ func TestImageRequest_SourceURL_DefaultHTTPS(t *testing.T) { } } -func TestImageRequest_SourceURL_AllowHTTP(t *testing.T) { +func TestRequest_SourceURL_AllowHTTP(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: "localhost:8080", - SourcePath: testPathCat, - AllowHTTP: true, + req := &Request{ + Source: Source{ + Host: "localhost:8080", + Path: testPathCat, + }, + AllowHTTP: true, } got := req.SourceURL() @@ -36,13 +40,15 @@ func TestImageRequest_SourceURL_AllowHTTP(t *testing.T) { } } -func TestImageRequest_SourceURL_AllowHTTPFalse(t *testing.T) { +func TestRequest_SourceURL_AllowHTTPFalse(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: "/img.jpg", - AllowHTTP: false, + req := &Request{ + Source: Source{ + Host: testHostCDN, + Path: "/img.jpg", + }, + AllowHTTP: false, } got := req.SourceURL() diff --git a/internal/imgcache/urlparser.go b/internal/imagerequest/urlparser.go similarity index 88% rename from internal/imgcache/urlparser.go rename to internal/imagerequest/urlparser.go index 46bba16..ce96231 100644 --- a/internal/imgcache/urlparser.go +++ b/internal/imagerequest/urlparser.go @@ -1,4 +1,4 @@ -package imgcache +package imagerequest import ( "errors" @@ -8,8 +8,6 @@ import ( "regexp" "strconv" "strings" - - "sneak.berlin/go/pixa/internal/imageprocessor" ) // URL parsing errors. @@ -25,8 +23,8 @@ var ( ) // MaxDimension is the maximum allowed width or height, the same as the image -// processor's maximum for an output image. -const MaxDimension = imageprocessor.MaxOutputDimension +// processor's maximum for an output image (imageprocessor.MaxOutputDimension). +const MaxDimension = 8192 // ValidateDimension checks one requested width or height; name ("width" or // "height") appears in the error. 0 is valid: that side is then worked out from @@ -68,20 +66,6 @@ var queryEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}|[^!-~]|["#'<>%]`) var hostRegex = regexp.MustCompile( `^(?:[A-Za-z0-9_.-]+|\[([0-9A-Fa-f:.]+)\])(?::[0-9]*)?$`) -// ParsedURL contains the parsed components of an image proxy URL. -type ParsedURL struct { - // Host is the source origin host (e.g., "cdn.example.com") - Host string - // Path is the path on the origin (e.g., "/photos/cat.jpg") - Path string - // Query is the optional query string for the origin - Query string - // Size is the requested output dimensions - Size Size - // Format is the requested output format - Format ImageFormat -} - // ParseImagePath parses an image URL's path after /v1/image/, decoded once: // //., or // for JPEG XL. // Examples: @@ -89,7 +73,7 @@ type ParsedURL struct { // - cdn.example.com/photos/cat.jpg/0x0.jpeg // - cdn.example.com/photos/cat.jpg/orig.png // - cdn.example.com/photos/cat.jpg/800x600 -func ParseImagePath(path string) (*ParsedURL, error) { +func ParseImagePath(path string) (*Request, error) { // Strip leading slash if present (chi may include it) path = strings.TrimPrefix(path, "/") if path == "" { @@ -103,7 +87,7 @@ func ParseImagePath(path string) (*ParsedURL, error) { // it: /v1/image///., or /v1/image/// // for JPEG XL. A source query follows the source path after a "?", which the // URL writes as %3F. -func ParseImageURL(urlPath string) (*ParsedURL, error) { +func ParseImageURL(urlPath string) (*Request, error) { // Remove the /v1/image/ prefix const prefix = "/v1/image/" if !strings.HasPrefix(urlPath, prefix) { @@ -120,7 +104,7 @@ func ParseImageURL(urlPath string) (*ParsedURL, error) { // parseImageComponents parses //., or // // for JPEG XL. -func parseImageComponents(remainder string) (*ParsedURL, error) { +func parseImageComponents(remainder string) (*Request, error) { // Find the last path segment, which holds "size" or "size.format" lastSlash := strings.LastIndex(remainder, "/") if lastSlash == -1 { @@ -209,12 +193,9 @@ func parseImageComponents(remainder string) (*ParsedURL, error) { path = "/" + path } - return &ParsedURL{ - Host: host, - Path: path, - Query: query, - Size: size, - Format: format, + return &Request{ + Source: Source{Host: host, Path: path, Query: query}, + Transform: Transform{Size: size, Format: format}, }, nil } @@ -297,7 +278,7 @@ func escapeQuery(query string) string { } // parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600" -func parseSizeFormat(s string) (Size, ImageFormat, error) { +func parseSizeFormat(s string) (Size, Format, error) { matches := sizeFormatRegex.FindStringSubmatch(s) if matches == nil { return Size{}, "", ErrInvalidSize @@ -351,8 +332,8 @@ func parseSizeFormat(s string) (Size, ImageFormat, error) { return size, format, nil } -// parseFormat converts a format string to ImageFormat. -func parseFormat(s string) (ImageFormat, error) { +// parseFormat converts a format string to Format. +func parseFormat(s string) (Format, error) { switch strings.ToLower(s) { case "orig", "original": return FormatOriginal, nil @@ -374,14 +355,3 @@ func parseFormat(s string) (ImageFormat, error) { return "", fmt.Errorf("%w: %s", ErrInvalidFormat, s) } } - -// ToImageRequest converts a ParsedURL to an ImageRequest. -func (p *ParsedURL) ToImageRequest() *ImageRequest { - return &ImageRequest{ - SourceHost: p.Host, - SourcePath: p.Path, - SourceQuery: p.Query, - Size: p.Size, - Format: p.Format, - } -} diff --git a/internal/imgcache/urlparser_internal_test.go b/internal/imagerequest/urlparser_internal_test.go similarity index 72% rename from internal/imgcache/urlparser_internal_test.go rename to internal/imagerequest/urlparser_internal_test.go index 603d516..83742ca 100644 --- a/internal/imgcache/urlparser_internal_test.go +++ b/internal/imagerequest/urlparser_internal_test.go @@ -1,33 +1,41 @@ -package imgcache +package imagerequest import ( "errors" "testing" ) -// assertParsedURL compares all fields of a parsed URL against the -// expected value. -func assertParsedURL(t *testing.T, got, want *ParsedURL) { +// Shared test fixture strings. +const ( + testHostCDN = "cdn.example.com" + testHostExample = "example.com" + testPathCat = "/photos/cat.jpg" +) + +// assertRequest compares the source host, path and query and the size and +// format of got against want. +func assertRequest(t *testing.T, got, want *Request) { t.Helper() - if got.Host != want.Host { - t.Errorf("Host = %q, want %q", got.Host, want.Host) + if got.Source.Host != want.Source.Host { + t.Errorf("Host = %q, want %q", got.Source.Host, want.Source.Host) } - if got.Path != want.Path { - t.Errorf("Path = %q, want %q", got.Path, want.Path) + if got.Source.Path != want.Source.Path { + t.Errorf("Path = %q, want %q", got.Source.Path, want.Source.Path) } - if got.Query != want.Query { - t.Errorf("Query = %q, want %q", got.Query, want.Query) + if got.Source.Query != want.Source.Query { + t.Errorf("Query = %q, want %q", got.Source.Query, want.Source.Query) } - if got.Size != want.Size { - t.Errorf("Size = %v, want %v", got.Size, want.Size) + if got.Transform.Size != want.Transform.Size { + t.Errorf("Size = %v, want %v", got.Transform.Size, want.Transform.Size) } - if got.Format != want.Format { - t.Errorf("Format = %q, want %q", got.Format, want.Format) + if got.Transform.Format != want.Transform.Format { + t.Errorf("Format = %q, want %q", + got.Transform.Format, want.Transform.Format) } } @@ -37,62 +45,64 @@ func TestParseImageURL(t *testing.T) { tests := []struct { name string input string - want *ParsedURL + want *Request }{ { name: "basic path with size", input: "/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp", - want: &ParsedURL{ - Host: testHostCDN, Path: testPathCat, - Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + want: &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat}, + Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP}, }, }, { name: "original size with 0x0", input: "/v1/image/cdn.example.com/photos/cat.jpg/0x0.jpeg", - want: &ParsedURL{ - Host: testHostCDN, Path: testPathCat, - Size: Size{Width: 0, Height: 0}, Format: FormatJPEG, + want: &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat}, + Transform: Transform{Size: Size{Width: 0, Height: 0}, Format: FormatJPEG}, }, }, { name: "original size with orig keyword", input: "/v1/image/cdn.example.com/photos/cat.jpg/orig.png", - want: &ParsedURL{ - Host: testHostCDN, Path: testPathCat, - Size: Size{Width: 0, Height: 0}, Format: FormatPNG, + want: &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat}, + Transform: Transform{Size: Size{Width: 0, Height: 0}, Format: FormatPNG}, }, }, { name: "path with query string", input: "/v1/image/cdn.example.com/photos/cat.jpg?arg1=val1&arg2=val2/800x600.webp", - want: &ParsedURL{ - Host: testHostCDN, Path: testPathCat, Query: "arg1=val1&arg2=val2", - Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + want: &Request{ + Source: Source{ + Host: testHostCDN, Path: testPathCat, Query: "arg1=val1&arg2=val2", + }, + Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP}, }, }, { name: "deep nested path", input: "/v1/image/cdn.example.com/a/b/c/d/image.jpg/1920x1080.avif", - want: &ParsedURL{ - Host: testHostCDN, Path: "/a/b/c/d/image.jpg", - Size: Size{Width: 1920, Height: 1080}, Format: FormatAVIF, + want: &Request{ + Source: Source{Host: testHostCDN, Path: "/a/b/c/d/image.jpg"}, + Transform: Transform{Size: Size{Width: 1920, Height: 1080}, Format: FormatAVIF}, }, }, { name: "jpg alias for jpeg", input: "/v1/image/example.com/img.png/100x100.jpg", - want: &ParsedURL{ - Host: testHostExample, Path: "/img.png", - Size: Size{Width: 100, Height: 100}, Format: FormatJPEG, + want: &Request{ + Source: Source{Host: testHostExample, Path: "/img.png"}, + Transform: Transform{Size: Size{Width: 100, Height: 100}, Format: FormatJPEG}, }, }, { name: "gif format", input: "/v1/image/example.com/animated.gif/200x200.gif", - want: &ParsedURL{ - Host: testHostExample, Path: "/animated.gif", - Size: Size{Width: 200, Height: 200}, Format: FormatGIF, + want: &Request{ + Source: Source{Host: testHostExample, Path: "/animated.gif"}, + Transform: Transform{Size: Size{Width: 200, Height: 200}, Format: FormatGIF}, }, }, } @@ -106,7 +116,7 @@ func TestParseImageURL(t *testing.T) { t.Fatalf("ParseImageURL() unexpected error = %v", err) } - assertParsedURL(t, got, tt.want) + assertRequest(t, got, tt.want) }) } } @@ -121,8 +131,8 @@ func TestParseImageURL_AutoFormat(t *testing.T) { t.Fatalf("ParseImageURL() error = %v", err) } - if got.Format != FormatAuto { - t.Errorf("Format = %q, want %q", got.Format, FormatAuto) + if got.Transform.Format != FormatAuto { + t.Errorf("Format = %q, want %q", got.Transform.Format, FormatAuto) } } @@ -189,27 +199,35 @@ func TestParseImagePath(t *testing.T) { tests := []struct { name string input string - want *ParsedURL + want *Request wantErr bool }{ { name: "chi wildcard capture", input: "cdn.example.com/photos/cat.jpg/800x600.webp", - want: &ParsedURL{ - Host: testHostCDN, - Path: testPathCat, - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, + want: &Request{ + Source: Source{ + Host: testHostCDN, + Path: testPathCat, + }, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, + Format: FormatWebP, + }, }, }, { name: "with leading slash from chi", input: "/cdn.example.com/photos/cat.jpg/800x600.webp", - want: &ParsedURL{ - Host: testHostCDN, - Path: testPathCat, - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, + want: &Request{ + Source: Source{ + Host: testHostCDN, + Path: testPathCat, + }, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, + Format: FormatWebP, + }, }, }, } @@ -229,7 +247,7 @@ func TestParseImagePath(t *testing.T) { return } - assertParsedURL(t, got, tt.want) + assertRequest(t, got, tt.want) }) } } @@ -246,14 +264,14 @@ func TestParseImagePath_LowerCasesHost(t *testing.T) { t.Fatalf("ParseImagePath() error = %v", err) } - t.Logf("parsed host %q, path %q", got.Host, got.Path) + t.Logf("parsed host %q, path %q", got.Source.Host, got.Source.Path) - if got.Host != testHostCDN { - t.Errorf("Host = %q, want %q", got.Host, testHostCDN) + if got.Source.Host != testHostCDN { + t.Errorf("Host = %q, want %q", got.Source.Host, testHostCDN) } - if got.Path != "/Photos/Cat.jpg" { - t.Errorf("Path = %q, want %q", got.Path, "/Photos/Cat.jpg") + if got.Source.Path != "/Photos/Cat.jpg" { + t.Errorf("Path = %q, want %q", got.Source.Path, "/Photos/Cat.jpg") } } @@ -277,40 +295,6 @@ func TestParseImagePath_RefusesUserinfo(t *testing.T) { } } -func TestParsedURL_ToImageRequest(t *testing.T) { - t.Parallel() - - parsed := &ParsedURL{ - Host: testHostCDN, - Path: testPathCat, - Query: "version=2", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - } - - req := parsed.ToImageRequest() - - if req.SourceHost != parsed.Host { - t.Errorf("SourceHost = %q, want %q", req.SourceHost, parsed.Host) - } - - if req.SourcePath != parsed.Path { - t.Errorf("SourcePath = %q, want %q", req.SourcePath, parsed.Path) - } - - if req.SourceQuery != parsed.Query { - t.Errorf("SourceQuery = %q, want %q", req.SourceQuery, parsed.Query) - } - - if req.Size != parsed.Size { - t.Errorf("Size = %v, want %v", req.Size, parsed.Size) - } - - if req.Format != parsed.Format { - t.Errorf("Format = %q, want %q", req.Format, parsed.Format) - } -} - func TestParseImageURL_PathTraversal(t *testing.T) { t.Parallel() @@ -417,22 +401,24 @@ func TestParseImageURL_SourceURLHoldingPercent25(t *testing.T) { tests := []struct { name string input string - want *ParsedURL + want *Request }{ { name: "path", input: "/v1/image/cdn.example.com/sale/50%25off.jpg/800x600.webp", - want: &ParsedURL{ - Host: testHostCDN, Path: "/sale/50%25off.jpg", - Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + want: &Request{ + Source: Source{Host: testHostCDN, Path: "/sale/50%25off.jpg"}, + Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP}, }, }, { name: "query", input: "/v1/image/cdn.example.com/photos/cat.jpg?discount=50%25/800x600.webp", - want: &ParsedURL{ - Host: testHostCDN, Path: testPathCat, Query: "discount=50%25", - Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + want: &Request{ + Source: Source{ + Host: testHostCDN, Path: testPathCat, Query: "discount=50%25", + }, + Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP}, }, }, } @@ -446,7 +432,7 @@ func TestParseImageURL_SourceURLHoldingPercent25(t *testing.T) { t.Fatalf("ParseImageURL() unexpected error = %v", err) } - assertParsedURL(t, got, tt.want) + assertRequest(t, got, tt.want) }) } } diff --git a/internal/imgcache/validate_internal_test.go b/internal/imagerequest/validate_internal_test.go similarity index 54% rename from internal/imgcache/validate_internal_test.go rename to internal/imagerequest/validate_internal_test.go index 99ecf36..8cb3d23 100644 --- a/internal/imgcache/validate_internal_test.go +++ b/internal/imagerequest/validate_internal_test.go @@ -1,4 +1,4 @@ -package imgcache +package imagerequest import ( "errors" @@ -10,35 +10,43 @@ func TestValidateImageRequest(t *testing.T) { tests := []struct { name string - req ImageRequest + req Request wantErr error }{ { name: "within bounds", - req: ImageRequest{Size: Size{Width: 800, Height: 600}, FitMode: FitCover}, + req: Request{Transform: Transform{ + Size: Size{Width: 800, Height: 600}, FitMode: FitCover, + }}, }, { name: "original size and empty fit", - req: ImageRequest{Size: Size{Width: 0, Height: 0}}, + req: Request{Transform: Transform{Size: Size{Width: 0, Height: 0}}}, }, { - name: "width over limit", - req: ImageRequest{Size: Size{Width: MaxDimension + 1, Height: 600}}, + name: "width over limit", + req: Request{Transform: Transform{ + Size: Size{Width: MaxDimension + 1, Height: 600}, + }}, wantErr: ErrDimensionTooLarge, }, { - name: "height over limit", - req: ImageRequest{Size: Size{Width: 800, Height: MaxDimension + 1}}, + name: "height over limit", + req: Request{Transform: Transform{ + Size: Size{Width: 800, Height: MaxDimension + 1}, + }}, wantErr: ErrDimensionTooLarge, }, { name: "negative width", - req: ImageRequest{Size: Size{Width: -1, Height: 600}}, + req: Request{Transform: Transform{Size: Size{Width: -1, Height: 600}}}, wantErr: ErrInvalidSize, }, { - name: "invalid fit mode", - req: ImageRequest{Size: Size{Width: 800, Height: 600}, FitMode: "bogus"}, + name: "invalid fit mode", + req: Request{Transform: Transform{ + Size: Size{Width: 800, Height: 600}, FitMode: "bogus", + }}, wantErr: ErrInvalidFitMode, }, } diff --git a/internal/imagerequest/variantkey.go b/internal/imagerequest/variantkey.go new file mode 100644 index 0000000..bfa0f64 --- /dev/null +++ b/internal/imagerequest/variantkey.go @@ -0,0 +1,39 @@ +package imagerequest + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// VariantKey is a SHA256 hash identifying a specific image variant (hex-encoded). +type VariantKey string + +// CacheKey generates a unique key for a request variant. +// Format: sha256(host:path:query:width:height:format:quality:fit_mode) +// The path and query go through escapePercentAndColon, so that a ":" in +// either cannot be taken for the one between them. +func CacheKey(req *Request) VariantKey { + data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s", + req.Source.Host, + escapePercentAndColon(req.Source.Path), + escapePercentAndColon(req.Source.Query), + req.Transform.Size.Width, + req.Transform.Size.Height, + req.Transform.Format, + req.Transform.Quality, + req.Transform.FitMode, + ) + h := sha256.Sum256([]byte(data)) + + return VariantKey(hex.EncodeToString(h[:])) +} + +// escapePercentAndColon returns s with each "%" written as "%25" and each +// ":" as "%3A", so that the result holds no ":". +func escapePercentAndColon(s string) string { + s = strings.ReplaceAll(s, "%", "%25") + + return strings.ReplaceAll(s, ":", "%3A") +} diff --git a/internal/imagerequest/variantkey_internal_test.go b/internal/imagerequest/variantkey_internal_test.go new file mode 100644 index 0000000..9d6c775 --- /dev/null +++ b/internal/imagerequest/variantkey_internal_test.go @@ -0,0 +1,142 @@ +package imagerequest + +import "testing" + +func TestCacheKey(t *testing.T) { + t.Parallel() + + req1 := &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""}, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + Quality: 85, FitMode: FitCover, + }, + } + + req2 := &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""}, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + Quality: 85, FitMode: FitCover, + }, + } + + // Same request should produce same key + key1 := CacheKey(req1) + key2 := CacheKey(req2) + + if key1 != key2 { + t.Errorf("CacheKey() not deterministic: %s vs %s", key1, key2) + } + + // Key should be 64 hex chars + if len(string(key1)) != 64 { + t.Errorf("CacheKey() length = %d, want 64", len(string(key1))) + } + + // Different size should produce different key + req3 := &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""}, + Transform: Transform{ + Size: Size{Width: 400, Height: 300}, Format: FormatWebP, // Different size + Quality: 85, FitMode: FitCover, + }, + } + + key3 := CacheKey(req3) + if key1 == key3 { + t.Error("CacheKey() produced same key for different sizes") + } + + // Different format should produce different key + req4 := &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""}, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, Format: FormatPNG, // Different format + Quality: 85, FitMode: FitCover, + }, + } + + key4 := CacheKey(req4) + if key1 == key4 { + t.Error("CacheKey() produced same key for different formats") + } + + // Different quality should produce different key + req5 := &Request{ + Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""}, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, Format: FormatWebP, + Quality: 50, FitMode: FitCover, // Different quality + }, + } + + key5 := CacheKey(req5) + if key1 == key5 { + t.Error("CacheKey() produced same key for different quality") + } +} + +// TestCacheKey_ColonOrPercentInPathOrQuery checks that two different source +// URLs whose path or query holds ":" or "%" get different keys. +func TestCacheKey_ColonOrPercentInPathOrQuery(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + path1, query1 string + path2, query2 string + }{ + { + name: "colon moved from path to query", + path1: "/a:b", query1: "", + path2: "/a", query2: "b:", + }, + { + name: "colon in path and its percent-encoding", + path1: "/a:b", query1: "", + path2: "/a%3Ab", query2: "", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + key1 := CacheKey(&Request{ + Source: Source{ + Host: testHostCDN, + Path: tt.path1, + Query: tt.query1, + }, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, + Format: FormatWebP, + Quality: 85, + FitMode: FitCover, + }, + }) + key2 := CacheKey(&Request{ + Source: Source{ + Host: testHostCDN, + Path: tt.path2, + Query: tt.query2, + }, + Transform: Transform{ + Size: Size{Width: 800, Height: 600}, + Format: FormatWebP, + Quality: 85, + FitMode: FitCover, + }, + }) + + t.Logf("CacheKey() of path %q query %q = %s", tt.path1, tt.query1, key1) + t.Logf("CacheKey() of path %q query %q = %s", tt.path2, tt.query2, key2) + + if key1 == key2 { + t.Errorf("CacheKey() gave path %q query %q and path %q query %q "+ + "the same key", tt.path1, tt.query1, tt.path2, tt.query2) + } + }) + } +} diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 4a78953..35bac85 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -17,6 +17,7 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" ) // Cache errors. @@ -111,7 +112,7 @@ type Cache struct { // metaCache holds the content types of the variants most recently // stored or served, so a hit does not read the variant's .meta file. // It never stands in for the variant file, which is always opened. - metaCache *lru.Cache[VariantKey, string] + metaCache *lru.Cache[imagerequest.VariantKey, string] // contentLocks serializes StoreSource and evictSourceBlob per // content hash, closing the race window between an eviction's row @@ -153,7 +154,7 @@ func newCache( log = slog.Default() } - metaCache, err := lru.New[VariantKey, string](metaCacheSize) + metaCache, err := lru.New[imagerequest.VariantKey, string](metaCacheSize) if err != nil { return nil, fmt.Errorf("failed to create variant content type cache: %w", err) } @@ -223,7 +224,7 @@ func newCache( // LookupResult contains the result of a cache lookup. type LookupResult struct { Hit bool - CacheKey VariantKey + CacheKey imagerequest.VariantKey ContentType string SizeBytes int64 CacheStatus CacheStatus @@ -231,8 +232,10 @@ type LookupResult struct { // Lookup checks if a processed variant exists on disk. Hits touch the // variant's LRU timestamp; a disabled cache always misses. -func (c *Cache) Lookup(ctx context.Context, req *ImageRequest) (*LookupResult, error) { - cacheKey := CacheKey(req) +func (c *Cache) Lookup( + ctx context.Context, req *imagerequest.Request, +) (*LookupResult, error) { + cacheKey := imagerequest.CacheKey(req) // Check variant storage directly - no DB needed for cache hits if !c.disabled && c.variants.Exists(cacheKey) { @@ -257,7 +260,9 @@ func (c *Cache) Lookup(ctx context.Context, req *ImageRequest) (*LookupResult, e // variant's .meta file and is then kept in metaCache. A variant with // no .meta file is served as application/octet-stream, which is not // kept. -func (c *Cache) GetVariant(cacheKey VariantKey) (io.ReadCloser, int64, string, error) { +func (c *Cache) GetVariant( + cacheKey imagerequest.VariantKey, +) (io.ReadCloser, int64, string, error) { if c.disabled { return nil, 0, "", ErrNotFound } @@ -282,7 +287,7 @@ func (c *Cache) GetVariant(cacheKey VariantKey) (io.ReadCloser, int64, string, e // disabled cache it is a no-op returning an empty hash. func (c *Cache) StoreSource( ctx context.Context, - req *ImageRequest, + req *imagerequest.Request, content io.Reader, result *httpfetcher.FetchResult, ) (ContentHash, error) { @@ -322,7 +327,7 @@ func (c *Cache) StoreSource( } // Store in database - pathHash := HashPath(req.SourcePath + "?" + req.SourceQuery) + pathHash := HashPath(req.Source.Path + "?" + req.Source.Query) headersJSON, err := json.Marshal(result.Headers) if err != nil { @@ -349,7 +354,7 @@ func (c *Cache) StoreSource( content_type = excluded.content_type, response_headers = excluded.response_headers, fetched_at = CURRENT_TIMESTAMP - `, req.SourceHost, req.SourcePath, req.SourceQuery, pathHash, + `, req.Source.Host, req.Source.Path, req.Source.Query, pathHash, contentHash, httpStatusOK, result.ContentType, string(headersJSON)) if err != nil { return "", fmt.Errorf("failed to insert source metadata: %w", err) @@ -367,7 +372,8 @@ func (c *Cache) StoreSource( // accounting insert is best-effort (the startup reconciliation pass // adopts any variant file that misses its accounting row). func (c *Cache) StoreVariant( - ctx context.Context, cacheKey VariantKey, content io.Reader, contentType string, + ctx context.Context, cacheKey imagerequest.VariantKey, + content io.Reader, contentType string, ) error { if c.disabled { return nil @@ -403,7 +409,7 @@ func (c *Cache) StoreVariant( // if not. Hits touch the blob's LRU timestamp; a disabled cache always // reports no cached source. func (c *Cache) LookupSource( - ctx context.Context, req *ImageRequest, + ctx context.Context, req *imagerequest.Request, ) (ContentHash, string, error) { if c.disabled { return "", "", nil @@ -414,7 +420,7 @@ func (c *Cache) LookupSource( err := c.db.QueryRowContext(ctx, ` SELECT content_hash, content_type FROM source_metadata WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.SourceHost, req.SourcePath, req.SourceQuery).Scan(&hashStr, &contentType) + `, req.Source.Host, req.Source.Path, req.Source.Query).Scan(&hashStr, &contentType) if errors.Is(err, sql.ErrNoRows) { return "", "", nil @@ -438,7 +444,7 @@ func (c *Cache) LookupSource( // StoreNegative stores a negative cache entry for a failed fetch. func (c *Cache) StoreNegative( - ctx context.Context, req *ImageRequest, statusCode int, errMsg string, + ctx context.Context, req *imagerequest.Request, statusCode int, errMsg string, ) error { expiresAt := time.Now().UTC().Add(c.config.NegativeTTL) @@ -452,7 +458,7 @@ func (c *Cache) StoreNegative( error_message = excluded.error_message, fetched_at = CURRENT_TIMESTAMP, expires_at = excluded.expires_at - `, req.SourceHost, req.SourcePath, req.SourceQuery, statusCode, errMsg, expiresAt) + `, req.Source.Host, req.Source.Path, req.Source.Query, statusCode, errMsg, expiresAt) if err != nil { return fmt.Errorf("failed to insert negative cache: %w", err) } @@ -462,14 +468,14 @@ func (c *Cache) StoreNegative( // GetSourceMetadataID returns the source metadata ID for a request. func (c *Cache) GetSourceMetadataID( - ctx context.Context, req *ImageRequest, + ctx context.Context, req *imagerequest.Request, ) (int64, error) { var id int64 err := c.db.QueryRowContext(ctx, ` SELECT id FROM source_metadata WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.SourceHost, req.SourcePath, req.SourceQuery).Scan(&id) + `, req.Source.Host, req.Source.Path, req.Source.Query).Scan(&id) if err != nil { return 0, fmt.Errorf("failed to get source metadata ID: %w", err) } @@ -665,7 +671,7 @@ func (c *Cache) readHitAndMissCounts(ctx context.Context, stats *CacheStats) err // application/octet-stream and keeps nothing, so metaCache only ever // holds a type read from a .meta file or passed to StoreVariant. func (c *Cache) loadVariantWithMeta( - cacheKey VariantKey, + cacheKey imagerequest.VariantKey, ) (io.ReadCloser, int64, string, error) { reader, size, contentType, err := c.variants.LoadWithMeta(cacheKey) if err != nil { @@ -685,15 +691,15 @@ func (c *Cache) loadVariantWithMeta( // A failure is logged and is otherwise non-fatal; the metadata is in the // database. func (c *Cache) writeMetadataSidecar( - req *ImageRequest, + req *imagerequest.Request, pathHash PathHash, contentHash ContentHash, result *httpfetcher.FetchResult, ) { meta := &SourceMetadata{ - Host: req.SourceHost, - Path: req.SourcePath, - Query: req.SourceQuery, + Host: req.Source.Host, + Path: req.Source.Path, + Query: req.Source.Query, ContentHash: string(contentHash), StatusCode: result.StatusCode, ContentType: result.ContentType, @@ -704,16 +710,16 @@ func (c *Cache) writeMetadataSidecar( RemoteAddr: result.RemoteAddr, } - err := c.srcMetadata.Store(req.SourceHost, pathHash, meta) + err := c.srcMetadata.Store(req.Source.Host, pathHash, meta) if err != nil { c.log.Warn("failed to write metadata sidecar", - "host", req.SourceHost, "path_hash", pathHash, "error", err) + "host", req.Source.Host, "path_hash", pathHash, "error", err) } } // touchVariant updates the LRU timestamp of a variant, best-effort: // a failed touch only makes the entry look colder to eviction. -func (c *Cache) touchVariant(ctx context.Context, cacheKey VariantKey) { +func (c *Cache) touchVariant(ctx context.Context, cacheKey imagerequest.VariantKey) { _, err := c.db.ExecContext(ctx, ` UPDATE variant_content SET last_accessed_at = CURRENT_TIMESTAMP WHERE cache_key = ? @@ -739,14 +745,14 @@ func (c *Cache) touchSourceContent(ctx context.Context, contentHash ContentHash) // checkNegativeCache checks if a request is in the negative cache. func (c *Cache) checkNegativeCache( - ctx context.Context, req *ImageRequest, + ctx context.Context, req *imagerequest.Request, ) (bool, error) { var expiresAt time.Time err := c.db.QueryRowContext(ctx, ` SELECT expires_at FROM negative_cache WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.SourceHost, req.SourcePath, req.SourceQuery).Scan(&expiresAt) + `, req.Source.Host, req.Source.Path, req.Source.Query).Scan(&expiresAt) if errors.Is(err, sql.ErrNoRows) { return false, nil @@ -762,10 +768,10 @@ func (c *Cache) checkNegativeCache( _, err = c.db.ExecContext(ctx, ` DELETE FROM negative_cache WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.SourceHost, req.SourcePath, req.SourceQuery) + `, req.Source.Host, req.Source.Path, req.Source.Query) if err != nil { c.log.Warn("failed to delete expired negative cache entry", - "host", req.SourceHost, "path", req.SourcePath, "error", err) + "host", req.Source.Host, "path", req.Source.Path, "error", err) } return false, nil diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index 6659b93..95966e2 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -10,6 +10,7 @@ import ( _ "modernc.org/sqlite" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" ) func setupTestDB(t *testing.T) *sql.DB { @@ -117,13 +118,17 @@ func TestCache_LookupMiss(t *testing.T) { cache := setupTestCache(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: testPathCat, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } result, err := cache.Lookup(ctx, req) @@ -146,13 +151,17 @@ func TestCache_StoreAndLookup(t *testing.T) { cache := setupTestCache(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: testPathCat, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Store source content @@ -173,7 +182,7 @@ func TestCache_StoreAndLookup(t *testing.T) { } // Store variant - cacheKey := CacheKey(req) + cacheKey := imagerequest.CacheKey(req) outputContent := []byte("fake webp data") err = cache.StoreVariant( @@ -207,11 +216,15 @@ func TestCache_NegativeCache(t *testing.T) { cache := setupTestCache(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: "/photos/notfound.jpg", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: "/photos/notfound.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + }, } // Store negative cache entry @@ -248,11 +261,15 @@ func TestCache_NegativeCacheExpiry(t *testing.T) { ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: "/photos/expired.jpg", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: "/photos/expired.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + }, } // Store negative cache entry @@ -281,17 +298,21 @@ func TestCache_VariantLookup(t *testing.T) { cache := setupTestCache(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: "/photos/variant.jpg", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: "/photos/variant.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Store variant - cacheKey := CacheKey(req) + cacheKey := imagerequest.CacheKey(req) outputContent := []byte("output data") err := cache.StoreVariant( @@ -331,17 +352,21 @@ func TestCache_GetVariant_ReturnsContentType(t *testing.T) { cache := setupTestCache(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: "/photos/variantct.jpg", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: "/photos/variantct.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Store variant - cacheKey := CacheKey(req) + cacheKey := imagerequest.CacheKey(req) outputContent := []byte("output webp data") err := cache.StoreVariant( @@ -383,17 +408,21 @@ func TestCache_GetVariant(t *testing.T) { cache := setupTestCache(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: "/photos/output.jpg", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: "/photos/output.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: imagerequest.FormatWebP, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Store variant - cacheKey := CacheKey(req) + cacheKey := imagerequest.CacheKey(req) outputContent := []byte("the actual output content") err := cache.StoreVariant( diff --git a/internal/imgcache/cache_usage_internal_test.go b/internal/imgcache/cache_usage_internal_test.go index 64666aa..a4d9315 100644 --- a/internal/imgcache/cache_usage_internal_test.go +++ b/internal/imgcache/cache_usage_internal_test.go @@ -5,6 +5,8 @@ import ( "strconv" "strings" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestEvictionReadsTheUsageTotal adds, stores again, resizes and evicts @@ -171,7 +173,8 @@ func TestReconciliationReadsAPageAtATime(t *testing.T) { storeEvictionTestSource(t, cache, "pages.example.com", "/"+strconv.Itoa(i)+".jpg", bytes.Repeat(content, 100)) - storeEvictionTestVariant(t, cache, VariantKey("aabbccdd000"+strconv.Itoa(i)), + storeEvictionTestVariant(t, cache, + imagerequest.VariantKey("aabbccdd000"+strconv.Itoa(i)), bytes.Repeat(content, 10)) } diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index d763b99..cb1570f 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -16,6 +16,7 @@ import ( "github.com/getsentry/sentry-go" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/magic" ) @@ -80,14 +81,20 @@ func setupHeldFetchService(t *testing.T) (*Service, *TestFixtures, *heldFetcher) // photoVariant asks for the test photo, 100x100, at 50x25 as a JPEG of the // given quality and fit mode. Each call returns a new request, as Get writes // to the request it is given. -func photoVariant(fixtures *TestFixtures, quality int, fit FitMode) *ImageRequest { - return &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 25}, - Format: FormatJPEG, - Quality: quality, - FitMode: fit, +func photoVariant( + fixtures *TestFixtures, quality int, fit imagerequest.Fit, +) *imagerequest.Request { + return &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 25}, + Format: imagerequest.FormatJPEG, + Quality: quality, + FitMode: fit, + }, } } @@ -100,7 +107,7 @@ type getResult struct { // startGet calls Get in a goroutine of its own and delivers what it returned // on the channel. func startGet( - ctx context.Context, svc *Service, req *ImageRequest, + ctx context.Context, svc *Service, req *imagerequest.Request, ) <-chan getResult { results := make(chan getResult, 1) @@ -146,7 +153,7 @@ func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { pending := make([]<-chan getResult, 0, requests) for range requests { pending = append(pending, - startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover))) + startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover))) } <-fetcher.started @@ -199,9 +206,11 @@ func TestService_Get_ConcurrentVariantsStayApart(t *testing.T) { svc, fixtures, fetcher := setupHeldFetchService(t) - cover := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) - lowQuality := startGet(t.Context(), svc, photoVariant(fixtures, 40, FitCover)) - contain := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitContain)) + cover := startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + lowQuality := startGet(t.Context(), svc, + photoVariant(fixtures, 40, imagerequest.FitCover)) + contain := startGet(t.Context(), svc, + photoVariant(fixtures, 85, imagerequest.FitContain)) for range 3 { select { @@ -253,12 +262,12 @@ func TestService_Get_WaiterStopsWhenItsContextEnds(t *testing.T) { svc, fixtures, fetcher := setupHeldFetchService(t) - first := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + first := startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover)) <-fetcher.started waiterCtx, cancelWaiter := context.WithCancel(t.Context()) - waiter := startGet(waiterCtx, svc, photoVariant(fixtures, 85, FitCover)) + waiter := startGet(waiterCtx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) time.Sleep(arrivalWait) cancelWaiter() @@ -300,11 +309,11 @@ func TestService_Get_FirstRequestLeavingKeepsTheWork(t *testing.T) { svc, fixtures, fetcher := setupHeldFetchService(t) firstCtx, cancelFirst := context.WithCancel(t.Context()) - first := startGet(firstCtx, svc, photoVariant(fixtures, 85, FitCover)) + first := startGet(firstCtx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) <-fetcher.started - second := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + second := startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover)) time.Sleep(arrivalWait) cancelFirst() @@ -355,9 +364,9 @@ func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { svc, fixtures, fetcher := setupHeldFetchService(t) - request := func() *ImageRequest { - req := photoVariant(fixtures, 85, FitCover) - req.SourcePath = tc.path + request := func() *imagerequest.Request { + req := photoVariant(fixtures, 85, imagerequest.FitCover) + req.Source.Path = tc.path return req } @@ -399,7 +408,7 @@ func TestService_Get_EndedRequestFetchesNothing(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - _, err := svc.Get(ctx, photoVariant(fixtures, 85, FitCover)) + _, err := svc.Get(ctx, photoVariant(fixtures, 85, imagerequest.FitCover)) if !errors.Is(err, context.Canceled) { t.Errorf("Get() error = %v, want %v", err, context.Canceled) } @@ -423,7 +432,7 @@ func TestService_Get_ReturnsByItsDeadline(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), timeout) defer cancel() - results := startGet(ctx, svc, photoVariant(fixtures, 85, FitCover)) + results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) select { case got := <-results: @@ -456,11 +465,11 @@ func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) { firstCtx, cancelFirst := context.WithTimeout(t.Context(), firstTimeout) defer cancelFirst() - first := startGet(firstCtx, svc, photoVariant(fixtures, 85, FitCover)) + first := startGet(firstCtx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) <-fetcher.started - waiter := startGet(t.Context(), svc, photoVariant(fixtures, 85, FitCover)) + waiter := startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover)) time.Sleep(arrivalWait) fetcher.releaseFetches() @@ -503,7 +512,7 @@ func TestService_Get_PanicBecomesAnError(t *testing.T) { } }() - _, err = svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover)) + _, err = svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) }() if err == nil || !strings.Contains(err.Error(), "upstream fetcher panicked") { @@ -540,7 +549,7 @@ func TestService_Get_PanicIsReportedToSentry(t *testing.T) { } }() - _, _ = svc.Get(ctx, photoVariant(fixtures, 85, FitCover)) + _, _ = svc.Get(ctx, photoVariant(fixtures, 85, imagerequest.FitCover)) }() events := transport.Events() diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 83828c4..4c7a9f7 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -11,6 +11,8 @@ import ( "path/filepath" "strings" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // DefaultEvictionInterval is how often the background evictor checks @@ -85,7 +87,7 @@ func (c *Cache) UsageBytes(ctx context.Context) (int64, error) { // source content blob (identified by contentHash). type evictionCandidate struct { isVariant bool - cacheKey VariantKey + cacheKey imagerequest.VariantKey contentHash ContentHash sizeBytes int64 lastAccessedAt string @@ -240,7 +242,7 @@ func (c *Cache) variantCandidates(ctx context.Context) ([]evictionCandidate, err return nil, fmt.Errorf("failed to scan variant candidate: %w", err) } - candidate.cacheKey = VariantKey(key) + candidate.cacheKey = imagerequest.VariantKey(key) candidates = append(candidates, candidate) } @@ -300,7 +302,9 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro // file. The metaCache entry goes before the files; a GetVariant that // read them just before may put it back, and the next GetVariant then // fails to open the file and removes it again. -func (c *Cache) evictVariant(ctx context.Context, cacheKey VariantKey) error { +func (c *Cache) evictVariant( + ctx context.Context, cacheKey imagerequest.VariantKey, +) error { _, err := c.db.ExecContext(ctx, `DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey)) if err != nil { @@ -648,7 +652,7 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error { return nil } - return c.reconcileVariantFile(ctx, path, entry, VariantKey(name)) + return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name)) }, ) } @@ -659,7 +663,7 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error { // crash or a restored snapshot can leave it, is removed with its row and // a warning naming it, so the variant is converted again. func (c *Cache) reconcileVariantFile( - ctx context.Context, path string, entry fs.DirEntry, cacheKey VariantKey, + ctx context.Context, path string, entry fs.DirEntry, cacheKey imagerequest.VariantKey, ) error { info, err := entry.Info() if err != nil { @@ -691,7 +695,7 @@ func (c *Cache) reconcileVariantFile( // adoptVariantFile inserts an accounting row for a variant file that // has none, using the file's size and modification time. func (c *Cache) adoptVariantFile( - ctx context.Context, path string, info fs.FileInfo, cacheKey VariantKey, + ctx context.Context, path string, info fs.FileInfo, cacheKey imagerequest.VariantKey, ) error { modTime := info.ModTime().UTC().Format(sqliteTimestampLayout) contentType := c.variantContentTypeFromSidecar(path) @@ -731,7 +735,7 @@ func (c *Cache) variantContentTypeFromSidecar(variantPath string) string { // reconcileVariantRows drops accounting rows whose variant files are // missing, so the database never references deleted content. func (c *Cache) reconcileVariantRows(ctx context.Context) error { - var after VariantKey + var after imagerequest.VariantKey for { keys, err := c.variantKeysAfter(ctx, after) @@ -772,9 +776,9 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error { // variantKeysAfter returns, in order, up to c.reconciliationPageSize // tracked variant cache keys that sort after the given one. func (c *Cache) variantKeysAfter( - ctx context.Context, after VariantKey, -) ([]VariantKey, error) { - return queryStringColumn[VariantKey](ctx, c.db, ` + ctx context.Context, after imagerequest.VariantKey, +) ([]imagerequest.VariantKey, error) { + return queryStringColumn[imagerequest.VariantKey](ctx, c.db, ` SELECT cache_key FROM variant_content WHERE cache_key > ? ORDER BY cache_key LIMIT ? `, "variant keys", "variant key", string(after), c.reconciliationPageSize) diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index d1ddf31..ce5087d 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -16,6 +16,7 @@ import ( _ "modernc.org/sqlite" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" ) // sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP @@ -27,10 +28,10 @@ const sqliteTimestampFormat = "2006-01-02 15:04:05" // keys reused across the eviction tests, in the order the tests store // them. const ( - testVariantKeyOne VariantKey = "aabbccdd0001" - testVariantKeyTwo VariantKey = "aabbccdd0002" - testVariantKeyThree VariantKey = "aabbccdd0003" - testVariantKeyFour VariantKey = "aabbccdd0004" + testVariantKeyOne imagerequest.VariantKey = "aabbccdd0001" + testVariantKeyTwo imagerequest.VariantKey = "aabbccdd0002" + testVariantKeyThree imagerequest.VariantKey = "aabbccdd0003" + testVariantKeyFour imagerequest.VariantKey = "aabbccdd0004" ) // evictionTestDB creates an in-memory SQLite database with the real @@ -88,12 +89,16 @@ func storeEvictionTestSource( ) ContentHash { t.Helper() - req := &ImageRequest{ - SourceHost: host, - SourcePath: path, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: host, + Path: path, + }, + Transform: imagerequest.Transform{ + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } result := &httpfetcher.FetchResult{ @@ -118,7 +123,7 @@ func storeEvictionTestSource( // storeEvictionTestVariant stores content as a processed variant under // the given cache key. func storeEvictionTestVariant( - t *testing.T, cache *Cache, key VariantKey, content []byte, + t *testing.T, cache *Cache, key imagerequest.VariantKey, content []byte, ) { t.Helper() @@ -131,7 +136,7 @@ func storeEvictionTestVariant( // setVariantLastAccessed backdates the last access time of a tracked // variant, to make LRU ordering deterministic in tests. func setVariantLastAccessed( - t *testing.T, cache *Cache, key VariantKey, when time.Time, + t *testing.T, cache *Cache, key imagerequest.VariantKey, when time.Time, ) { t.Helper() @@ -244,7 +249,7 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) { t.Fatalf("failed to scan cache_key: %v", err) } - if !cache.variants.Exists(VariantKey(key)) { + if !cache.variants.Exists(imagerequest.VariantKey(key)) { t.Errorf("variant_content references key %s but the file is missing", key) } } @@ -343,7 +348,7 @@ func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) { cache, _ := newEvictionTestCache(t, limit) now := time.Now() - keys := []VariantKey{ + keys := []imagerequest.VariantKey{ testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree, testVariantKeyFour, } @@ -516,12 +521,16 @@ func TestZeroMaxBytesDisablesDiskCache(t *testing.T) { cache, tmpDir := newEvictionTestCache(t, 0) - req := &ImageRequest{ - SourceHost: "src.example.com", - SourcePath: "/a.jpg", - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: "src.example.com", + Path: "/a.jpg", + }, + Transform: imagerequest.Transform{ + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } assertDisabledCacheWritesAreNoOps(t, cache, req) @@ -533,14 +542,14 @@ func TestZeroMaxBytesDisablesDiskCache(t *testing.T) { // assertDisabledCacheWritesAreNoOps verifies that stores against a // disabled cache report success without recording anything. func assertDisabledCacheWritesAreNoOps( - t *testing.T, cache *Cache, req *ImageRequest, + t *testing.T, cache *Cache, req *imagerequest.Request, ) { t.Helper() ctx := t.Context() err := cache.StoreVariant( - ctx, CacheKey(req), bytes.NewReader([]byte("data")), "image/webp", + ctx, imagerequest.CacheKey(req), bytes.NewReader([]byte("data")), "image/webp", ) if err != nil { t.Fatalf("StoreVariant on disabled cache must be a no-op, got error: %v", err) @@ -566,7 +575,7 @@ func assertDisabledCacheWritesAreNoOps( // assertDisabledCacheReadsAlwaysMiss verifies that lookups against a // disabled cache never report a hit. func assertDisabledCacheReadsAlwaysMiss( - t *testing.T, cache *Cache, req *ImageRequest, + t *testing.T, cache *Cache, req *imagerequest.Request, ) { t.Helper() @@ -661,7 +670,7 @@ func TestEvictionRunsUnderWritePressure(t *testing.T) { cache.StartEviction(time.Hour) defer func() { _ = cache.StopEviction(t.Context()) }() - keys := []VariantKey{ + keys := []imagerequest.VariantKey{ testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree, } fills := []byte{0x11, 0x12, 0x13} @@ -715,7 +724,7 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) { time.Sleep(10 * time.Millisecond) } - keys := []VariantKey{ + keys := []imagerequest.VariantKey{ testVariantKeyOne, testVariantKeyTwo, testVariantKeyThree, } fills := []byte{0x21, 0x22, 0x23} @@ -1308,9 +1317,11 @@ func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) assertNoDanglingReferences(t, cache) - dupHash, _, err := cache.LookupSource(ctx, &ImageRequest{ - SourceHost: "race.example.com", - SourcePath: "/dup.jpg", + dupHash, _, err := cache.LookupSource(ctx, &imagerequest.Request{ + Source: imagerequest.Source{ + Host: "race.example.com", + Path: "/dup.jpg", + }, }) if err != nil { t.Fatalf("LookupSource failed: %v", err) @@ -1336,12 +1347,16 @@ func storeIdenticalContentConcurrently( storeDone := make(chan error, 1) go func() { - req := &ImageRequest{ - SourceHost: "race.example.com", - SourcePath: "/dup.jpg", - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: "race.example.com", + Path: "/dup.jpg", + }, + Transform: imagerequest.Transform{ + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } result := &httpfetcher.FetchResult{ StatusCode: 200, diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index 51b33b3..8354741 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -3,134 +3,12 @@ package imgcache import ( "context" - "errors" "io" - "net/url" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) -// ErrInvalidFitMode is returned when an invalid fit mode is provided. -var ErrInvalidFitMode = errors.New("invalid fit mode") - -// ImageFormat represents supported output image formats. -type ImageFormat string - -// Supported image output formats. -const ( - FormatOriginal ImageFormat = "orig" - FormatJPEG ImageFormat = "jpeg" - FormatPNG ImageFormat = "png" - FormatWebP ImageFormat = "webp" - FormatAVIF ImageFormat = "avif" - FormatJXL ImageFormat = "jxl" - FormatGIF ImageFormat = "gif" - - // FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each - // request from its Accept header once the URL's signature or token has - // been checked; it is never processed or cached as itself. - FormatAuto ImageFormat = "auto" -) - -// Size represents requested image dimensions -type Size struct { - Width int - Height int -} - -// OriginalSize returns true if this represents "keep original size" -func (s Size) OriginalSize() bool { - return s.Width == 0 && s.Height == 0 -} - -// FitMode represents how to fit image into requested dimensions. -type FitMode string - -// Supported image fit modes. -const ( - FitCover FitMode = "cover" - FitContain FitMode = "contain" - FitFill FitMode = "fill" - FitInside FitMode = "inside" - FitOutside FitMode = "outside" -) - -// ValidateFitMode checks if the given fit mode is valid. -// Returns ErrInvalidFitMode for unrecognized fit modes. -func ValidateFitMode(fit FitMode) error { - switch fit { - case FitCover, FitContain, FitFill, FitInside, FitOutside, "": - return nil - default: - return ErrInvalidFitMode - } -} - -// ValidateImageRequest checks a request's width and height with -// ValidateDimension and its fit mode with ValidateFitMode. Both the plain -// /v1/image/ route and the encrypted /v1/e/ route validate through this -// function so a request from either source enforces identical bounds. -func ValidateImageRequest(req *ImageRequest) error { - err := ValidateDimension("width", req.Size.Width) - if err != nil { - return err - } - - err = ValidateDimension("height", req.Size.Height) - if err != nil { - return err - } - - return ValidateFitMode(req.FitMode) -} - -// ImageRequest represents a request for a processed image -type ImageRequest struct { - // SourceHost is the origin host (e.g., "cdn.example.com") - SourceHost string - // SourcePath is the path on the origin with its percent-escapes decoded, - // as url.URL's Path holds it (e.g., "/photos/my cat.jpg") - SourcePath string - // SourceQuery is the optional query string for the origin URL as the URL - // writes it, percent-escapes included, as url.URL's RawQuery holds it - SourceQuery string - // Size is the requested output dimensions - Size Size - // Format is the requested output format - Format ImageFormat - // Quality is the output quality (1-100) for lossy formats - Quality int - // FitMode is how to fit the image into requested dimensions - FitMode FitMode - // Signature is the HMAC signature for non-allowlisted hosts - Signature string - // Expires is when the URL expires: the exp of a signed URL, or the expiry - // of an encrypted URL; the zero time if it has none - Expires time.Time - // AllowHTTP indicates whether HTTP (non-TLS) is allowed for this request - AllowHTTP bool -} - -// SourceURL returns the full upstream URL to fetch: the source path -// percent-encoded again, as net/url writes a path, so each source path is one -// upstream path, and the source query as it is, with the characters -// escapeQuery encodes percent-encoded. Uses http:// scheme when AllowHTTP is -// true, otherwise https://. -func (r *ImageRequest) SourceURL() string { - scheme := "https" - if r.AllowHTTP { - scheme = "http" - } - - sourceURL := url.URL{ - Scheme: scheme, - Host: r.SourceHost, - Path: r.SourcePath, - RawQuery: escapeQuery(r.SourceQuery), - } - - return sourceURL.String() -} - // ImageResponse represents a processed image ready to serve type ImageResponse struct { // Content is the image data reader @@ -162,10 +40,10 @@ const ( // ImageCache is the main interface for the image caching proxy type ImageCache interface { // Get retrieves a processed image, fetching and processing if necessary - Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) + Get(ctx context.Context, req *imagerequest.Request) (*ImageResponse, error) // Warm pre-fetches and caches an image without returning it - Warm(ctx context.Context, req *ImageRequest) error + Warm(ctx context.Context, req *imagerequest.Request) error // Stats returns cache statistics Stats(ctx context.Context) (*CacheStats, error) diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imgcache/max_concurrent_processing_internal_test.go index e4c310e..69aeb31 100644 --- a/internal/imgcache/max_concurrent_processing_internal_test.go +++ b/internal/imgcache/max_concurrent_processing_internal_test.go @@ -9,18 +9,23 @@ import ( "time" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" ) // widthOnlyRequest asks for the test photo at width, its height scaled to // keep the photo's aspect ratio. -func widthOnlyRequest(fixtures *TestFixtures, width int) *ImageRequest { - return &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: width}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, +func widthOnlyRequest(fixtures *TestFixtures, width int) *imagerequest.Request { + return &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: width}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } } @@ -36,7 +41,7 @@ func holdProcessingSlot( input, feed := io.Pipe() go func() { - _, _ = proc.Process(t.Context(), input, &imageprocessor.Request{}) + _, _ = proc.Process(t.Context(), input, &imagerequest.Transform{}) }() _, err := feed.Write([]byte{0}) diff --git a/internal/imgcache/metacache_internal_test.go b/internal/imgcache/metacache_internal_test.go index bade726..1eecbad 100644 --- a/internal/imgcache/metacache_internal_test.go +++ b/internal/imgcache/metacache_internal_test.go @@ -10,23 +10,31 @@ import ( "sync" "testing" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // webpRequest returns a request for a 100x100 WebP variant of path. -func webpRequest(path string) *ImageRequest { - return &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: path, - Size: Size{Width: 100, Height: 100}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, +func webpRequest(path string) *imagerequest.Request { + return &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: path, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 100}, + Format: imagerequest.FormatWebP, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } } // assertVariantServed checks that GetVariant serves key with the given // content and the image/webp content type storeEvictionTestVariant stores. -func assertVariantServed(t *testing.T, cache *Cache, key VariantKey, content []byte) { +func assertVariantServed( + t *testing.T, cache *Cache, key imagerequest.VariantKey, content []byte, +) { t.Helper() reader, size, contentType, err := cache.GetVariant(key) @@ -57,7 +65,9 @@ func assertVariantServed(t *testing.T, cache *Cache, key VariantKey, content []b // assertVariantNotFound checks that GetVariant refuses key with // ErrNotFound. -func assertVariantNotFound(t *testing.T, cache *Cache, key VariantKey) { +func assertVariantNotFound( + t *testing.T, cache *Cache, key imagerequest.VariantKey, +) { t.Helper() reader, _, _, err := cache.GetVariant(key) @@ -71,16 +81,18 @@ func assertVariantNotFound(t *testing.T, cache *Cache, key VariantKey) { } // assertLookupMisses checks that Lookup reports request as a miss. -func assertLookupMisses(t *testing.T, cache *Cache, request *ImageRequest) { +func assertLookupMisses( + t *testing.T, cache *Cache, request *imagerequest.Request, +) { t.Helper() lookup, err := cache.Lookup(t.Context(), request) if err != nil { - t.Fatalf("Lookup(%s) error = %v", request.SourcePath, err) + t.Fatalf("Lookup(%s) error = %v", request.Source.Path, err) } if lookup.Hit { - t.Errorf("Lookup(%s) is a hit, want a miss", request.SourcePath) + t.Errorf("Lookup(%s) is a hit, want a miss", request.Source.Path) } } @@ -251,8 +263,8 @@ func TestEvictedVariantIsNotServed(t *testing.T) { oldRequest := webpRequest("/old.jpg") newRequest := webpRequest("/new.jpg") - oldKey := CacheKey(oldRequest) - newKey := CacheKey(newRequest) + oldKey := imagerequest.CacheKey(oldRequest) + newKey := imagerequest.CacheKey(newRequest) oldContent := bytes.Repeat([]byte{0x01}, 1000) newContent := bytes.Repeat([]byte{0x02}, 1000) @@ -280,7 +292,7 @@ func TestVariantDeletedFromDiskIsNotServed(t *testing.T) { cache, _ := newEvictionTestCache(t, 1<<20) request := webpRequest("/deleted.jpg") - key := CacheKey(request) + key := imagerequest.CacheKey(request) content := []byte("webp variant bytes") storeEvictionTestVariant(t, cache, key, content) @@ -307,7 +319,7 @@ func TestConcurrentVariantStoreReadAndEvict(t *testing.T) { for goroutine := range 8 { wg.Go(func() { - key := VariantKey(fmt.Sprintf("aabbccdd01%02d", goroutine)) + key := imagerequest.VariantKey(fmt.Sprintf("aabbccdd01%02d", goroutine)) content := []byte(key) for range 20 { diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index dae2433..1c3b2bf 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -6,6 +6,8 @@ import ( "strconv" "testing" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) func TestNegativeCache_StoreAndCheck(t *testing.T) { @@ -23,9 +25,11 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { } ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostExample, - SourcePath: "/missing.jpg", + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostExample, + Path: "/missing.jpg", + }, } // Initially should not be in negative cache @@ -70,9 +74,11 @@ func TestNegativeCache_Expired(t *testing.T) { } ctx := context.Background() - req := &ImageRequest{ - SourceHost: testHostExample, - SourcePath: "/expired.jpg", + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostExample, + Path: "/expired.jpg", + }, } // Store a negative entry with very short TTL @@ -106,20 +112,21 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { svc, fixtures := SetupTestService(t) - resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover)) + resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) if err != nil { t.Fatalf("Get() first request error = %v", err) } _ = resp.Content.Close() - err = svc.cache.StoreNegative(t.Context(), photoVariant(fixtures, 85, FitCover), + err = svc.cache.StoreNegative(t.Context(), + photoVariant(fixtures, 85, imagerequest.FitCover), httpStatusBadGateway, "upstream answered 503") if err != nil { t.Fatalf("StoreNegative() error = %v", err) } - resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, FitCover)) + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) if err != nil { t.Fatalf("Get() of the cached variant error = %v", err) } @@ -131,7 +138,7 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { resp.CacheStatus, CacheHit) } - resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, FitCover)) + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover)) t.Logf("Get() of a variant made from the cached source: error = %v", err) if err != nil { @@ -141,7 +148,7 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { _ = resp.Content.Close() contentHash, _, err := svc.cache.LookupSource(t.Context(), - photoVariant(fixtures, 85, FitCover)) + photoVariant(fixtures, 85, imagerequest.FitCover)) if err != nil || contentHash == "" { t.Fatalf("LookupSource() = %q, %v, want the cached source", contentHash, err) } @@ -151,7 +158,7 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { t.Fatalf("evictSourceBlob() error = %v", err) } - _, err = svc.Get(t.Context(), photoVariant(fixtures, 60, FitCover)) + _, err = svc.Get(t.Context(), photoVariant(fixtures, 60, imagerequest.FitCover)) t.Logf("Get() of a variant whose source was evicted: error = %v", err) if !errors.Is(err, ErrNegativeCached) { @@ -190,8 +197,8 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { t.Fatalf("failed to create cache: %v", err) } - expired := &ImageRequest{ - SourceHost: testHostExample, SourcePath: "/gone.jpg", + expired := &imagerequest.Request{ + Source: imagerequest.Source{Host: testHostExample, Path: "/gone.jpg"}, } err = cache.StoreNegative(t.Context(), expired, 404, "not found") @@ -215,7 +222,7 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { cache.StartEviction(100 * time.Millisecond) defer func() { _ = cache.StopEviction(t.Context()) }() - for negativeCacheRowCount(t, cache, expired.SourcePath) > 0 { + for negativeCacheRowCount(t, cache, expired.Source.Path) > 0 { if time.Since(started) > 10*time.Second { t.Fatal("the expired row is still in negative_cache") } diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index 283ac13..364d5cc 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -10,23 +10,27 @@ import ( "path/filepath" "strings" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // getImage calls Get for req and returns the image it answered with and // its cache status. -func getImage(t *testing.T, svc *Service, req *ImageRequest) ([]byte, CacheStatus) { +func getImage( + t *testing.T, svc *Service, req *imagerequest.Request, +) ([]byte, CacheStatus) { t.Helper() resp, err := svc.Get(t.Context(), req) if err != nil { - t.Fatalf("Get() of quality %d error = %v", req.Quality, err) + t.Fatalf("Get() of quality %d error = %v", req.Transform.Quality, err) } defer func() { _ = resp.Content.Close() }() body, err := io.ReadAll(resp.Content) if err != nil { - t.Fatalf("reading the image of quality %d: %v", req.Quality, err) + t.Fatalf("reading the image of quality %d: %v", req.Transform.Quality, err) } return body, resp.CacheStatus @@ -58,7 +62,9 @@ func truncateFile(t *testing.T, path string, size int64) { } // cachedSourcePath returns the path of the cached source file req uses. -func cachedSourcePath(t *testing.T, svc *Service, req *ImageRequest) string { +func cachedSourcePath( + t *testing.T, svc *Service, req *imagerequest.Request, +) string { t.Helper() contentHash, _, err := svc.cache.LookupSource(t.Context(), req) @@ -113,16 +119,16 @@ func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) { svc, fixtures := SetupTestService(t) - getImage(t, svc, photoVariant(fixtures, 85, FitCover)) + getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover)) + path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) wholeSize := fileSize(t, path) truncateFile(t, path, 0) assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) - getImage(t, svc, photoVariant(fixtures, 40, FitCover)) - getImage(t, svc, photoVariant(fixtures, 30, FitCover)) + getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) + getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount t.Logf("upstream fetches = %d", fetches) @@ -149,16 +155,16 @@ func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) { svc, fixtures := SetupTestService(t) - getImage(t, svc, photoVariant(fixtures, 85, FitCover)) + getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover)) + path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) wholeSize := fileSize(t, path) truncateFile(t, path, wholeSize/2) assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) - getImage(t, svc, photoVariant(fixtures, 40, FitCover)) - getImage(t, svc, photoVariant(fixtures, 30, FitCover)) + getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) + getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount t.Logf("upstream fetches = %d", fetches) @@ -182,11 +188,11 @@ func assertConvertedAgain(t *testing.T, size int64) { t.Helper() svc, fixtures := SetupTestService(t) - req := photoVariant(fixtures, 85, FitCover) + req := photoVariant(fixtures, 85, imagerequest.FitCover) want, _ := getImage(t, svc, req) - path := svc.cache.variants.keyToPath(CacheKey(req)) + path := svc.cache.variants.keyToPath(imagerequest.CacheKey(req)) truncateFile(t, path, size) assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) diff --git a/internal/imgcache/pendingcounts_internal_test.go b/internal/imgcache/pendingcounts_internal_test.go index 77dd73e..e8a559a 100644 --- a/internal/imgcache/pendingcounts_internal_test.go +++ b/internal/imgcache/pendingcounts_internal_test.go @@ -6,6 +6,8 @@ import ( "errors" "testing" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // pendingCountsWait is how long a test waits for the database's connection, @@ -92,7 +94,7 @@ func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { deadline, _ := ctx.Deadline() - results := startGet(ctx, svc, photoVariant(fixtures, 85, FitCover)) + results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) // The request has made its database reads by the time it fetches. select { diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index ced31b4..25d19fa 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -19,6 +19,7 @@ import ( "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/magic" "sneak.berlin/go/pixa/internal/signature" ) @@ -139,7 +140,9 @@ var ErrProcessingTimedOut = errors.New( // While its source URL is in the negative cache, a cached variant is still // served and a variant not in the cache is still made from a cached source; // only a request that needs the source fetched is refused. -func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, error) { +func (s *Service) Get( + ctx context.Context, req *imagerequest.Request, +) (*ImageResponse, error) { // Propagate AllowHTTP setting to the request req.AllowHTTP = s.allowHTTP @@ -186,7 +189,7 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e } // Warm pre-fetches and caches an image without returning it. -func (s *Service) Warm(ctx context.Context, req *ImageRequest) error { +func (s *Service) Warm(ctx context.Context, req *imagerequest.Request) error { _, err := s.Get(ctx, req) return err @@ -204,7 +207,7 @@ func (s *Service) WaitForProcessing(ctx context.Context) int { } // ValidateRequest validates the request signature if required. -func (s *Service) ValidateRequest(req *ImageRequest) error { +func (s *Service) ValidateRequest(req *imagerequest.Request) error { // Check if host is allowed (no signature required) sourceURL := req.SourceURL() @@ -227,7 +230,7 @@ func (s *Service) ValidateRequest(req *ImageRequest) error { } // Signature required for non-allowed hosts - return s.signer.Verify(signatureRequest(req)) + return s.signer.Verify(req) } // GenerateSignedURL generates a signed URL for the given request. The URL @@ -236,26 +239,22 @@ func (s *Service) ValidateRequest(req *ImageRequest) error { // set to 85 or cover, the values the route uses when a URL has no q or fit. func (s *Service) GenerateSignedURL( baseURL string, - req *ImageRequest, + req *imagerequest.Request, ttl time.Duration, ) (string, error) { - if req.Quality == 0 { - req.Quality = 85 + if req.Transform.Quality == 0 { + req.Transform.Quality = 85 } - if req.FitMode == "" { - req.FitMode = FitCover + if req.Transform.FitMode == "" { + req.Transform.FitMode = imagerequest.FitCover } - sigReq := signatureRequest(req) - path, sig, exp := s.signer.GenerateSignedURL(sigReq, ttl) - - // Propagate the generated signature and expiration back onto the request. - req.Expires = sigReq.Expires - req.Signature = sigReq.Signature + // Sets the request's Expires and Signature + path, sig, exp := s.signer.GenerateSignedURL(req, ttl) return fmt.Sprintf("%s%s?sig=%s&exp=%d&q=%d&fit=%s", - baseURL, path, sig, exp, req.Quality, req.FitMode), nil + baseURL, path, sig, exp, req.Transform.Quality, req.Transform.FitMode), nil } // errPanicked is returned when processing a variant panicked. @@ -268,14 +267,14 @@ var errPanicked = errors.New("panic while processing image") // the first request's cancellation, so the others are still served if that // client goes away, but keeps its deadline. func (s *Service) processOrWait( - ctx context.Context, req *ImageRequest, + ctx context.Context, req *imagerequest.Request, ) (*ImageResponse, error) { // A request that has already ended starts no processing if ctx.Err() != nil { return nil, ctx.Err() } - cacheKey := CacheKey(req) + cacheKey := imagerequest.CacheKey(req) // Closed when this request's own function runs, which singleflight does // only when no other request is processing the variant @@ -293,7 +292,7 @@ func (s *Service) processOrWait( recovered := recover() if recovered != nil { s.log.Error("panic while processing image", - "host", req.SourceHost, "path", req.SourcePath, + "host", req.Source.Host, "path", req.Source.Path, "panic", recovered, "stack", string(debug.Stack())) if hub := sentry.GetHubFromContext(ctx); hub != nil { @@ -393,8 +392,8 @@ func (s *Service) loadCachedSource( // if available. func (s *Service) processFromSourceOrFetch( ctx context.Context, - req *ImageRequest, - cacheKey VariantKey, + req *imagerequest.Request, + cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { // Check if we have cached source content contentHash, _, err := s.cache.LookupSource(ctx, req) @@ -431,8 +430,8 @@ func (s *Service) processFromSourceOrFetch( // source URL is in the negative cache. func (s *Service) fetchAndProcess( ctx context.Context, - req *ImageRequest, - cacheKey VariantKey, + req *imagerequest.Request, + cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { // Check negative cache - skip fetching for recently-failed URLs negHit, err := s.cache.checkNegativeCache(ctx, req) @@ -442,8 +441,8 @@ func (s *Service) fetchAndProcess( if negHit { s.log.Debug("negative cache hit", - "host", req.SourceHost, - "path", req.SourcePath, + "host", req.Source.Host, + "path", req.Source.Path, ) return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached) @@ -493,8 +492,8 @@ func (s *Service) fetchAndProcess( // Log upstream fetch details s.log.Info("upstream fetched", "request_id", middleware.GetReqID(ctx), - "host", req.SourceHost, - "path", req.SourcePath, + "host", req.Source.Host, + "path", req.Source.Path, "bytes", fetchBytes, "fetch_ms", fetchResult.FetchDurationMs, "rate", downloadRate, @@ -510,7 +509,7 @@ func (s *Service) fetchAndProcess( err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) if err != nil { s.log.Warn("source refused", "request_id", middleware.GetReqID(ctx), - "host", req.SourceHost, "path", req.SourcePath, + "host", req.Source.Host, "path", req.Source.Path, "content_type", fetchResult.ContentType, "error", err) s.storeNegative(ctx, req, httpStatusUnsupportedMediaType, err) @@ -533,12 +532,12 @@ func (s *Service) fetchAndProcess( // failed with err, so the source is not fetched again until the entry // expires. A failure to store the entry is only logged. func (s *Service) storeNegative( - ctx context.Context, req *ImageRequest, statusCode int, err error, + ctx context.Context, req *imagerequest.Request, statusCode int, err error, ) { storeErr := s.cache.StoreNegative(ctx, req, statusCode, err.Error()) if storeErr != nil { s.log.Warn("failed to store negative cache entry", - "host", req.SourceHost, "path", req.SourcePath, "error", storeErr) + "host", req.Source.Host, "path", req.Source.Path, "error", storeErr) } } @@ -554,24 +553,17 @@ type processedVariant struct { // result. func (s *Service) processAndStore( ctx context.Context, - req *ImageRequest, - cacheKey VariantKey, + req *imagerequest.Request, + cacheKey imagerequest.VariantKey, source io.Reader, fetchBytes int64, ) (*processedVariant, error) { // Process the image processStart := time.Now() - processReq := &imageprocessor.Request{ - Size: imageprocessor.Size{Width: req.Size.Width, Height: req.Size.Height}, - Format: imageprocessor.Format(req.Format), - Quality: req.Quality, - FitMode: imageprocessor.FitMode(req.FitMode), - } - // ctx has the deadline of the request doing the processing, which can // end while a request waiting for its image has time left - processResult, err := s.processor.Process(ctx, source, processReq) + processResult, err := s.processor.Process(ctx, source, &req.Transform) if errors.Is(err, context.DeadlineExceeded) { return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err) } @@ -604,10 +596,10 @@ func (s *Service) processAndStore( s.log.Info("image converted", "request_id", middleware.GetReqID(ctx), - "host", req.SourceHost, - "path", req.SourcePath, + "host", req.Source.Host, + "path", req.Source.Path, "src_format", processResult.InputFormat, - "dst_format", req.Format, + "dst_format", req.Transform.Format, "src_bytes", fetchBytes, "dst_bytes", outputSize, "src_dimensions", fmt.Sprintf("%dx%d", @@ -616,8 +608,8 @@ func (s *Service) processAndStore( processResult.Width, processResult.Height), "size_ratio", fmt.Sprintf("%.1f%%", sizePercent), "convert_ms", processDuration.Milliseconds(), - "quality", req.Quality, - "fit", req.FitMode, + "quality", req.Transform.Quality, + "fit", req.Transform.FitMode, ) // Store variant to cache @@ -636,24 +628,6 @@ func (s *Service) processAndStore( }, nil } -// signatureRequest projects an ImageRequest onto the standalone -// signature.Request type used by the signature package. This keeps the -// import edge one-way: imgcache depends on signature, never the reverse. -func signatureRequest(req *ImageRequest) *signature.Request { - return &signature.Request{ - SourceHost: req.SourceHost, - SourcePath: req.SourcePath, - SourceQuery: req.SourceQuery, - Width: req.Size.Width, - Height: req.Size.Height, - Format: string(req.Format), - Quality: req.Quality, - FitMode: string(req.FitMode), - Signature: req.Signature, - Expires: req.Expires, - } -} - // HTTP status codes for error responses. const ( httpStatusBadGateway = 502 @@ -680,7 +654,7 @@ func extractStatusCode(err error) int { const etagHashLength = 16 // formatETag formats a VariantKey as a quoted ETag value. -func formatETag(key VariantKey) string { +func formatETag(key imagerequest.VariantKey) string { hash := string(key) // Use first 16 characters of hash for a shorter but still unique ETag if len(hash) > etagHashLength { diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index e4bbb30..712955c 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/magic" "sneak.berlin/go/pixa/internal/signature" ) @@ -25,13 +26,17 @@ func TestService_Get_AllowlistedHost(t *testing.T) { svc, fixtures := SetupTestService(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } resp, err := svc.Get(ctx, req) @@ -61,13 +66,17 @@ func TestService_Get_NonAllowlistedHost_NoSignature(t *testing.T) { svc, fixtures := SetupTestService(t, WithSigningKey("test-key")) - req := &ImageRequest{ - SourceHost: fixtures.OtherHost, - SourcePath: testPathUpload, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.OtherHost, + Path: testPathUpload, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Should fail validation - not allowlisted and no signature @@ -84,19 +93,23 @@ func TestService_Get_NonAllowlistedHost_ValidSignature(t *testing.T) { svc, fixtures := SetupTestService(t, WithSigningKey(signingKey)) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.OtherHost, - SourcePath: testPathUpload, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.OtherHost, + Path: testPathUpload, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Generate a valid signature signer := signature.New(signingKey) req.Expires = time.Now().Add(time.Hour) - req.Signature = signer.Sign(signatureRequest(req)) + req.Signature = signer.Sign(req) // Should pass validation err := svc.ValidateRequest(req) @@ -128,19 +141,23 @@ func TestService_Get_NonAllowlistedHost_ExpiredSignature(t *testing.T) { signingKey := testSigningKey svc, fixtures := SetupTestService(t, WithSigningKey(signingKey)) - req := &ImageRequest{ - SourceHost: fixtures.OtherHost, - SourcePath: testPathUpload, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.OtherHost, + Path: testPathUpload, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Generate an expired signature signer := signature.New(signingKey) req.Expires = time.Now().Add(-time.Hour) // Already expired - req.Signature = signer.Sign(signatureRequest(req)) + req.Signature = signer.Sign(req) // Should fail validation err := svc.ValidateRequest(req) @@ -155,13 +172,17 @@ func TestService_Get_NonAllowlistedHost_InvalidSignature(t *testing.T) { signingKey := testSigningKey svc, fixtures := SetupTestService(t, WithSigningKey(signingKey)) - req := &ImageRequest{ - SourceHost: fixtures.OtherHost, - SourcePath: testPathUpload, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.OtherHost, + Path: testPathUpload, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // Set an invalid signature @@ -191,16 +212,20 @@ func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) { signer := signature.New(signingKey) // Sign a request for "cdn.example.com" - signedReq := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, - Expires: time.Now().Add(time.Hour), + signedReq := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHostCDN, + Path: testPathCat, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: time.Now().Add(time.Hour), } - signedReq.Signature = signer.Sign(signatureRequest(signedReq)) + signedReq.Signature = signer.Sign(signedReq) // The original request should pass validation t.Run("exact host passes", func(t *testing.T) { @@ -228,23 +253,22 @@ func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) { t.Run(tt.name+" rejected", func(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: tt.host, - SourcePath: signedReq.SourcePath, - SourceQuery: signedReq.SourceQuery, - Size: signedReq.Size, - Format: signedReq.Format, - Quality: signedReq.Quality, - FitMode: signedReq.FitMode, - Expires: signedReq.Expires, - Signature: signedReq.Signature, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: tt.host, + Path: signedReq.Source.Path, + Query: signedReq.Source.Query, + }, + Transform: signedReq.Transform, + Expires: signedReq.Expires, + Signature: signedReq.Signature, } err := svc.ValidateRequest(req) if err == nil { t.Errorf( "ValidateRequest() should reject signature for host %q (signed for %q)", - tt.host, signedReq.SourceHost) + tt.host, signedReq.Source.Host) } }) } @@ -280,13 +304,17 @@ func TestService_ValidateRequest_HostWithPortNeedsSignature(t *testing.T) { t.Run(tt.host, func(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: tt.host, - SourcePath: testPathCat, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: tt.host, + Path: testPathCat, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } err := svc.ValidateRequest(req) @@ -320,13 +348,17 @@ func TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature( t.Run(host, func(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: host, - SourcePath: testPathCat, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: host, + Path: testPathCat, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } err := svc.ValidateRequest(req) @@ -347,14 +379,9 @@ func TestService_Get_InvalidFile(t *testing.T) { svc, fixtures := SetupTestService(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: "/images/fake.jpg", - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, - } + req := photoVariant(fixtures, 85, imagerequest.FitCover) + req.Source.Path = "/images/fake.jpg" + req.Transform.Size = imagerequest.Size{Width: 50, Height: 50} // Should fail because magic bytes don't match _, err := svc.Get(ctx, req) @@ -369,14 +396,9 @@ func TestService_Get_NotFound(t *testing.T) { svc, fixtures := SetupTestService(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: "/images/nonexistent.jpg", - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, - } + req := photoVariant(fixtures, 85, imagerequest.FitCover) + req.Source.Path = "/images/nonexistent.jpg" + req.Transform.Size = imagerequest.Size{Width: 50, Height: 50} _, err := svc.Get(ctx, req) if err == nil { @@ -393,25 +415,25 @@ func TestService_Get_FormatConversion(t *testing.T) { tests := []struct { name string sourcePath string - outFormat ImageFormat + outFormat imagerequest.Format wantMIME string }{ { name: "JPEG to PNG", sourcePath: testPathPhoto, - outFormat: FormatPNG, + outFormat: imagerequest.FormatPNG, wantMIME: "image/png", }, { name: "PNG to JPEG", sourcePath: "/images/logo.png", - outFormat: FormatJPEG, + outFormat: imagerequest.FormatJPEG, wantMIME: testContentTypeJPEG, }, { name: "GIF to PNG", sourcePath: "/images/animation.gif", - outFormat: FormatPNG, + outFormat: imagerequest.FormatPNG, wantMIME: "image/png", }, } @@ -420,13 +442,12 @@ func TestService_Get_FormatConversion(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: tt.sourcePath, - Size: Size{Width: 50, Height: 50}, - Format: tt.outFormat, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{Host: fixtures.GoodHost, Path: tt.sourcePath}, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: tt.outFormat, Quality: 85, FitMode: imagerequest.FitCover, + }, } resp, err := svc.Get(ctx, req) @@ -474,13 +495,17 @@ func TestService_Get_Caching(t *testing.T) { svc, fixtures := SetupTestService(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // First request - should be a cache miss @@ -544,7 +569,7 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { t.Parallel() svc, fixtures := SetupTestService(t) - variant := photoVariant(fixtures, 85, FitCover) + variant := photoVariant(fixtures, 85, imagerequest.FitCover) resp, err := svc.Get(t.Context(), variant) if err != nil { @@ -570,7 +595,7 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { } // Another quality is a new variant, made from the cached source - resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, FitCover)) + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover)) if err != nil { t.Fatalf("Get() of a new variant error = %v", err) } @@ -586,20 +611,20 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { // backdateSource sets every time pixa stored for req's cached source to when, // as listed on TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo. func backdateSource( - t *testing.T, cache *Cache, req *ImageRequest, when time.Time, + t *testing.T, cache *Cache, req *imagerequest.Request, when time.Time, ) { t.Helper() - pathHash := HashPath(req.SourcePath + "?" + req.SourceQuery) + pathHash := HashPath(req.Source.Path + "?" + req.Source.Query) - meta, err := cache.srcMetadata.Load(req.SourceHost, pathHash) + meta, err := cache.srcMetadata.Load(req.Source.Host, pathHash) if err != nil { t.Fatalf("reading the source's metadata file: %v", err) } meta.FetchedAt = when.Unix() - err = cache.srcMetadata.Store(req.SourceHost, pathHash, meta) + err = cache.srcMetadata.Store(req.Source.Host, pathHash, meta) if err != nil { t.Fatalf("writing the source's metadata file: %v", err) } @@ -609,7 +634,7 @@ func backdateSource( updateOneRow(t, cache, ` UPDATE source_metadata SET fetched_at = ? WHERE source_host = ? AND source_path = ? AND source_query = ? - `, dbTime, req.SourceHost, req.SourcePath, req.SourceQuery) + `, dbTime, req.Source.Host, req.Source.Path, req.Source.Query) updateOneRow(t, cache, `UPDATE source_content SET fetched_at = ? WHERE content_hash = ?`, dbTime, meta.ContentHash) @@ -617,7 +642,7 @@ func backdateSource( for _, path := range []string{ cache.srcContent.hashToPath(ContentHash(meta.ContentHash)), - cache.srcMetadata.metaPath(req.SourceHost, pathHash), + cache.srcMetadata.metaPath(req.Source.Host, pathHash), } { err = os.Chtimes(path, when, when) if err != nil { @@ -630,11 +655,11 @@ func backdateSource( // for to when, as listed on // TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo. func backdateVariant( - t *testing.T, cache *Cache, req *ImageRequest, when time.Time, + t *testing.T, cache *Cache, req *imagerequest.Request, when time.Time, ) { t.Helper() - key := CacheKey(req) + key := imagerequest.CacheKey(req) content, size, contentType, err := cache.variants.LoadWithMeta(key) if err != nil || contentType == "" { @@ -696,7 +721,7 @@ func TestService_Get_DifferentSizes(t *testing.T) { ctx := context.Background() // Request same image at different sizes - sizes := []Size{ + sizes := []imagerequest.Size{ {Width: 25, Height: 25}, {Width: 50, Height: 50}, {Width: 75, Height: 75}, @@ -705,13 +730,17 @@ func TestService_Get_DifferentSizes(t *testing.T) { responses := make([][]byte, 0, len(sizes)) for _, size := range sizes { - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: size, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: size, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } resp, err := svc.Get(ctx, req) @@ -745,13 +774,17 @@ func TestService_ValidateRequest_NoSigningKey(t *testing.T) { // Service with no signing key - all non-allowlisted requests should fail svc, fixtures := SetupTestService(t, WithNoAllowlist()) - req := &ImageRequest{ - SourceHost: fixtures.OtherHost, - SourcePath: testPathUpload, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.OtherHost, + Path: testPathUpload, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } err := svc.ValidateRequest(req) @@ -770,13 +803,17 @@ func TestService_Get_ContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } _, err := svc.Get(ctx, req) @@ -791,13 +828,17 @@ func TestService_Get_ReturnsETag(t *testing.T) { svc, fixtures := SetupTestService(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } resp, err := svc.Get(ctx, req) @@ -824,13 +865,17 @@ func TestService_Get_ETagConsistency(t *testing.T) { svc, fixtures := SetupTestService(t) ctx := context.Background() - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // First request @@ -866,22 +911,30 @@ func TestService_Get_DifferentETagsForDifferentContent(t *testing.T) { ctx := context.Background() // Request same image at different sizes - should get different ETags - req1 := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 25, Height: 25}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req1 := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 25, Height: 25}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } - req2 := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req2 := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } resp1, err := svc.Get(ctx, req1) diff --git a/internal/imgcache/stats_internal_test.go b/internal/imgcache/stats_internal_test.go index 2fe9dbb..0ff7dae 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imgcache/stats_internal_test.go @@ -15,6 +15,7 @@ import ( "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" ) func setupStatsTestDB(t *testing.T) *sql.DB { @@ -319,13 +320,17 @@ func TestService_Get_CountsStats(t *testing.T) { } for _, step := range steps { - resp, err := svc.Get(t.Context(), &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: step.path, - Size: Size{Width: step.size, Height: step.size}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + resp, err := svc.Get(t.Context(), &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: step.path, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: step.size, Height: step.size}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, }) if (err != nil) != step.wantErr { t.Fatalf("%s: Get() error = %v, want error %t", step.name, err, step.wantErr) @@ -419,13 +424,17 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { svc.fetcher = upstream - resp, err := svc.Get(ctx, &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + resp, err := svc.Get(ctx, &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, }) t.Logf("Get() error = %v", err) @@ -453,13 +462,17 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { svc, fixtures := SetupTestService(t) - req := &ImageRequest{ - SourceHost: fixtures.GoodHost, - SourcePath: testPathPhoto, - Size: Size{Width: 50, Height: 50}, - Format: FormatJPEG, - Quality: 85, - FitMode: FitCover, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: testPathPhoto, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, } // A first request caches the variant. diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index de55df4..a243fab 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -10,8 +10,9 @@ import ( "log/slog" "os" "path/filepath" - "strings" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // Storage constants. @@ -32,9 +33,6 @@ var ( // ContentHash is a SHA256 hash of file content (hex-encoded). type ContentHash string -// VariantKey is a SHA256 hash identifying a specific image variant (hex-encoded). -type VariantKey string - // PathHash is a SHA256 hash of a URL path (hex-encoded). type PathHash string @@ -403,34 +401,6 @@ func HashPath(path string) PathHash { return PathHash(hex.EncodeToString(h[:])) } -// CacheKey generates a unique key for a request variant. -// Format: sha256(host:path:query:width:height:format:quality:fit_mode) -// The path and query go through escapePercentAndColon, so that a ":" in -// either cannot be taken for the one between them. -func CacheKey(req *ImageRequest) VariantKey { - data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s", - req.SourceHost, - escapePercentAndColon(req.SourcePath), - escapePercentAndColon(req.SourceQuery), - req.Size.Width, - req.Size.Height, - req.Format, - req.Quality, - req.FitMode, - ) - h := sha256.Sum256([]byte(data)) - - return VariantKey(hex.EncodeToString(h[:])) -} - -// escapePercentAndColon returns s with each "%" written as "%25" and each -// ":" as "%3A", so that the result holds no ":". -func escapePercentAndColon(s string) string { - s = strings.ReplaceAll(s, "%", "%25") - - return strings.ReplaceAll(s, ":", "%3A") -} - // VariantStorage handles key-based file storage for processed image variants. // Files are stored at: /// // Metadata is stored at: ///.meta @@ -464,7 +434,7 @@ func NewVariantStorage(baseDir string, log *slog.Logger) (*VariantStorage, error // with writeFileSynced. The .meta file is written first, so that a crash // cannot leave the content in place without it. func (s *VariantStorage) Store( - key VariantKey, r io.Reader, contentType string, + key imagerequest.VariantKey, r io.Reader, contentType string, ) (int64, error) { data, err := io.ReadAll(r) if err != nil { @@ -508,7 +478,7 @@ func (s *VariantStorage) Store( } // Load returns a reader for the content at the given key. -func (s *VariantStorage) Load(key VariantKey) (io.ReadCloser, error) { +func (s *VariantStorage) Load(key imagerequest.VariantKey) (io.ReadCloser, error) { path := s.keyToPath(key) f, err := os.Open(path) //nolint:gosec // path derived from cache key @@ -525,7 +495,9 @@ func (s *VariantStorage) Load(key VariantKey) (io.ReadCloser, error) { // LoadWithSize returns a reader and file size for the content at the // given key. -func (s *VariantStorage) LoadWithSize(key VariantKey) (io.ReadCloser, int64, error) { +func (s *VariantStorage) LoadWithSize( + key imagerequest.VariantKey, +) (io.ReadCloser, int64, error) { path := s.keyToPath(key) f, err := os.Open(path) //nolint:gosec // path derived from cache key @@ -551,7 +523,7 @@ func (s *VariantStorage) LoadWithSize(key VariantKey) (io.ReadCloser, int64, err // the given key. The content type is read from the .meta file, and is // empty when that file is missing or unreadable. func (s *VariantStorage) LoadWithMeta( - key VariantKey, + key imagerequest.VariantKey, ) (io.ReadCloser, int64, string, error) { f, size, err := s.LoadWithSize(key) if err != nil { @@ -574,7 +546,7 @@ func (s *VariantStorage) LoadWithMeta( } // Exists checks if content exists at the given key. -func (s *VariantStorage) Exists(key VariantKey) bool { +func (s *VariantStorage) Exists(key imagerequest.VariantKey) bool { path := s.keyToPath(key) _, err := os.Stat(path) @@ -583,7 +555,7 @@ func (s *VariantStorage) Exists(key VariantKey) bool { // Delete removes the content at the given key together with its .meta // sidecar file. A missing file is not an error. -func (s *VariantStorage) Delete(key VariantKey) error { +func (s *VariantStorage) Delete(key imagerequest.VariantKey) error { path := s.keyToPath(key) err := os.Remove(path) @@ -602,7 +574,7 @@ func (s *VariantStorage) Delete(key VariantKey) error { } // keyToPath converts a key to a file path: /// -func (s *VariantStorage) keyToPath(key VariantKey) string { +func (s *VariantStorage) keyToPath(key imagerequest.VariantKey) string { k := string(key) if len(k) < MinHashLength { return filepath.Join(s.baseDir, k) diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go index 1d3996d..0819b9f 100644 --- a/internal/imgcache/storage_internal_test.go +++ b/internal/imgcache/storage_internal_test.go @@ -9,6 +9,8 @@ import ( "path/filepath" "strings" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) func TestContentStorage_StoreAndLoad(t *testing.T) { @@ -322,147 +324,6 @@ func TestHashPath(t *testing.T) { } } -func TestCacheKey(t *testing.T) { - t.Parallel() - - req1 := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - SourceQuery: "", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, - } - - req2 := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - SourceQuery: "", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, - } - - // Same request should produce same key - key1 := CacheKey(req1) - key2 := CacheKey(req2) - - if key1 != key2 { - t.Errorf("CacheKey() not deterministic: %s vs %s", key1, key2) - } - - // Key should be 64 hex chars - if len(string(key1)) != 64 { - t.Errorf("CacheKey() length = %d, want 64", len(string(key1))) - } - - // Different size should produce different key - req3 := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - SourceQuery: "", - Size: Size{Width: 400, Height: 300}, // Different size - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, - } - - key3 := CacheKey(req3) - if key1 == key3 { - t.Error("CacheKey() produced same key for different sizes") - } - - // Different format should produce different key - req4 := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - SourceQuery: "", - Size: Size{Width: 800, Height: 600}, - Format: FormatPNG, // Different format - Quality: 85, - FitMode: FitCover, - } - - key4 := CacheKey(req4) - if key1 == key4 { - t.Error("CacheKey() produced same key for different formats") - } - - // Different quality should produce different key - req5 := &ImageRequest{ - SourceHost: testHostCDN, - SourcePath: testPathCat, - SourceQuery: "", - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 50, // Different quality - FitMode: FitCover, - } - - key5 := CacheKey(req5) - if key1 == key5 { - t.Error("CacheKey() produced same key for different quality") - } -} - -// TestCacheKey_ColonOrPercentInPathOrQuery checks that two different source -// URLs whose path or query holds ":" or "%" get different keys. -func TestCacheKey_ColonOrPercentInPathOrQuery(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - path1, query1 string - path2, query2 string - }{ - { - name: "colon moved from path to query", - path1: "/a:b", query1: "", - path2: "/a", query2: "b:", - }, - { - name: "colon in path and its percent-encoding", - path1: "/a:b", query1: "", - path2: "/a%3Ab", query2: "", - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - key1 := CacheKey(&ImageRequest{ - SourceHost: testHostCDN, - SourcePath: tt.path1, - SourceQuery: tt.query1, - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, - }) - key2 := CacheKey(&ImageRequest{ - SourceHost: testHostCDN, - SourcePath: tt.path2, - SourceQuery: tt.query2, - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - Quality: 85, - FitMode: FitCover, - }) - - t.Logf("CacheKey() of path %q query %q = %s", tt.path1, tt.query1, key1) - t.Logf("CacheKey() of path %q query %q = %s", tt.path2, tt.query2, key2) - - if key1 == key2 { - t.Errorf("CacheKey() gave path %q query %q and path %q query %q "+ - "the same key", tt.path1, tt.query1, tt.path2, tt.query2) - } - }) - } -} - // TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write // that fails is logged at warn and the store still succeeds. func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { @@ -476,7 +337,9 @@ func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { t.Fatalf("NewVariantStorage() error = %v", err) } - key := CacheKey(&ImageRequest{SourceHost: testHostCDN, SourcePath: testPathCat}) + key := imagerequest.CacheKey(&imagerequest.Request{ + Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, + }) // A directory where the .meta file goes makes the .meta write fail. err = os.MkdirAll(storage.keyToPath(key)+".meta", StorageDirPerm) @@ -497,7 +360,7 @@ func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { // storeTestVariant stores one variant, with its .meta file, in a new // VariantStorage and returns the storage and the variant's key. -func storeTestVariant(t *testing.T) (*VariantStorage, VariantKey) { +func storeTestVariant(t *testing.T) (*VariantStorage, imagerequest.VariantKey) { t.Helper() storage, err := NewVariantStorage(t.TempDir(), slog.New(slog.DiscardHandler)) @@ -505,7 +368,9 @@ func storeTestVariant(t *testing.T) (*VariantStorage, VariantKey) { t.Fatalf("NewVariantStorage() error = %v", err) } - key := CacheKey(&ImageRequest{SourceHost: testHostCDN, SourcePath: testPathCat}) + key := imagerequest.CacheKey(&imagerequest.Request{ + Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, + }) _, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp") if err != nil { diff --git a/internal/server/computed_side_internal_test.go b/internal/server/computed_side_internal_test.go index ded083f..ad81740 100644 --- a/internal/server/computed_side_internal_test.go +++ b/internal/server/computed_side_internal_test.go @@ -9,7 +9,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProxyRefusesComputedSideAboveMaximum requests sizes of a 3000x600 @@ -70,9 +70,12 @@ func TestImageProxyRefusesComputedSideAboveMaximum(t *testing.T) { } for _, tt := range tests { - token, err := generator.Generate(&encurl.Payload{ - SourceHost: upstreamHost, SourcePath: tt.path, - Width: tt.width, Height: tt.height, Format: imgcache.FormatJPEG, + token, err := generator.Generate(&imagerequest.Request{ + Source: imagerequest.Source{Host: upstreamHost, Path: tt.path}, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: tt.width, Height: tt.height}, + Format: imagerequest.FormatJPEG, + }, }) if err != nil { t.Fatalf("encurl Generate: %v", err) diff --git a/internal/server/fit_outside_internal_test.go b/internal/server/fit_outside_internal_test.go index f0ada85..cbc6a80 100644 --- a/internal/server/fit_outside_internal_test.go +++ b/internal/server/fit_outside_internal_test.go @@ -7,7 +7,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestImageProxyRefusesFitOutsideAboveMaximum requests sizes with fit=outside @@ -59,10 +59,13 @@ func TestImageProxyRefusesFitOutsideAboveMaximum(t *testing.T) { } for _, tt := range tests { - token, err := generator.Generate(&encurl.Payload{ - SourceHost: upstreamHost, SourcePath: tt.path, - Width: tt.width, Height: tt.height, Format: imgcache.FormatJPEG, - FitMode: imgcache.FitOutside, + token, err := generator.Generate(&imagerequest.Request{ + Source: imagerequest.Source{Host: upstreamHost, Path: tt.path}, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: tt.width, Height: tt.height}, + Format: imagerequest.FormatJPEG, + FitMode: imagerequest.FitOutside, + }, }) if err != nil { t.Fatalf("encurl Generate: %v", err) diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index 27b963b..05e8ef4 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -14,6 +14,7 @@ import ( "time" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/signature" @@ -513,12 +514,16 @@ func signedImageURL(t *testing.T, sourceHost string) string { t.Helper() path, sig, exp := signature.New(testSigningKey).GenerateSignedURL( - &signature.Request{ - SourceHost: sourceHost, - SourcePath: ipv6SourcePath, - Format: string(imgcache.FormatPNG), - Quality: 85, - FitMode: string(imgcache.FitCover), + &imagerequest.Request{ + Source: imagerequest.Source{ + Host: sourceHost, + Path: ipv6SourcePath, + }, + Transform: imagerequest.Transform{ + Format: imagerequest.FormatPNG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, }, time.Hour) return path + "?sig=" + sig + "&exp=" + strconv.FormatInt(exp, 10) @@ -534,9 +539,9 @@ func encryptedImageURL(t *testing.T, sourceHost string) string { t.Fatalf("encurl.NewGenerator: %v", err) } - token, err := generator.Generate(&encurl.Payload{ - SourceHost: sourceHost, SourcePath: ipv6SourcePath, - Format: imgcache.FormatPNG, + token, err := generator.Generate(&imagerequest.Request{ + Source: imagerequest.Source{Host: sourceHost, Path: ipv6SourcePath}, + Transform: imagerequest.Transform{Format: imagerequest.FormatPNG}, }) if err != nil { t.Fatalf("encurl Generate: %v", err) diff --git a/internal/server/svg_as_avif_internal_test.go b/internal/server/svg_as_avif_internal_test.go index cd05947..af75b24 100644 --- a/internal/server/svg_as_avif_internal_test.go +++ b/internal/server/svg_as_avif_internal_test.go @@ -8,7 +8,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imagerequest" ) // svgServedAsAVIF is an SVG whose first bytes pass the first-bytes check for @@ -38,10 +38,13 @@ func TestImageProxyRefusesSVGServedAsAVIF(t *testing.T) { t.Fatalf("encurl.NewGenerator: %v", err) } - encryptedURL := func(format imgcache.ImageFormat) string { - token, err := generator.Generate(&encurl.Payload{ - SourceHost: upstreamHost, SourcePath: "/drawing.avif", - Width: 32, Height: 24, Format: format, + encryptedURL := func(format imagerequest.Format) string { + token, err := generator.Generate(&imagerequest.Request{ + Source: imagerequest.Source{Host: upstreamHost, Path: "/drawing.avif"}, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 32, Height: 24}, + Format: format, + }, }) if err != nil { t.Fatalf("encurl Generate: %v", err) @@ -53,8 +56,8 @@ func TestImageProxyRefusesSVGServedAsAVIF(t *testing.T) { for _, target := range []string{ "/v1/image/" + upstreamHost + "/drawing.avif/32x24.png", "/v1/image/" + upstreamHost + "/drawing.avif/64x48.orig", - encryptedURL(imgcache.FormatPNG), - encryptedURL(imgcache.FormatOriginal), + encryptedURL(imagerequest.FormatPNG), + encryptedURL(imagerequest.FormatOriginal), } { rec := httptest.NewRecorder() s.ServeHTTP(rec, httptest.NewRequestWithContext( diff --git a/internal/signature/escaping_test.go b/internal/signature/escaping_test.go index 550805c..032df24 100644 --- a/internal/signature/escaping_test.go +++ b/internal/signature/escaping_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/signature" ) @@ -16,32 +17,35 @@ func TestSigner_Verify_ColonCannotMovePathQueryBoundary(t *testing.T) { signer := signature.New(goldenSigningKey) - signed := &signature.Request{ - SourceHost: testHost, - SourcePath: "/a:b", - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 85, - FitMode: testFitCover, - Expires: time.Now().Add(1 * time.Hour), + signed := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: "/a:b", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + }, + Expires: time.Now().Add(1 * time.Hour), } signed.Signature = signer.Sign(signed) replayed := *signed - replayed.SourcePath = "/a" - replayed.SourceQuery = "b:" + replayed.Source.Path = "/a" + replayed.Source.Query = "b:" t.Logf("signature for path %q query %q: %s", - signed.SourcePath, signed.SourceQuery, signed.Signature) + signed.Source.Path, signed.Source.Query, signed.Signature) t.Logf("signature for path %q query %q: %s", - replayed.SourcePath, replayed.SourceQuery, signer.Sign(&replayed)) + replayed.Source.Path, replayed.Source.Query, signer.Sign(&replayed)) err := signer.Verify(&replayed) if !errors.Is(err, signature.ErrInvalid) { t.Errorf("Verify() of path %q query %q with the signature for "+ "path %q = %v, want %v", - replayed.SourcePath, replayed.SourceQuery, signed.SourcePath, + replayed.Source.Path, replayed.Source.Query, signed.Source.Path, err, signature.ErrInvalid) } } @@ -90,16 +94,19 @@ func TestSigner_EscapedGoldenVectors(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - req := &signature.Request{ - SourceHost: testHost, - SourcePath: tt.path, - SourceQuery: tt.query, - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 85, - FitMode: testFitCover, - Expires: time.Unix(goldenExpiresUnix, 0), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: tt.path, + Query: tt.query, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + }, + Expires: time.Unix(goldenExpiresUnix, 0), } gotSignature := signer.Sign(req) diff --git a/internal/signature/golden_test.go b/internal/signature/golden_test.go index 126b05c..bb8eda2 100644 --- a/internal/signature/golden_test.go +++ b/internal/signature/golden_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/signature" ) @@ -16,7 +17,7 @@ const goldenSigningKey = "golden-test-key" type goldenVector struct { name string - req signature.Request + req imagerequest.Request // wantSignature is the exact base64url (RFC 4648 URL-safe, // padded) HMAC-SHA256 signature for the request with Expires // set to goldenExpiresUnix. @@ -34,15 +35,18 @@ func goldenVectors() []goldenVector { return []goldenVector{ { name: "resized without query", - req: signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "", - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 85, - FitMode: testFitCover, + req: imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + }, }, // Signed data: // "cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover" @@ -51,15 +55,18 @@ func goldenVectors() []goldenVector { }, { name: "resized with query string", - req: signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "token=abc&v=2", - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 85, - FitMode: testFitCover, + req: imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "token=abc&v=2", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + }, }, // Signed data: // "cdn.example.com:/photos/cat.jpg:token=abc&v=2:800:600:webp:1704067200:85:cover" @@ -69,15 +76,18 @@ func goldenVectors() []goldenVector { }, { name: "original size without query", - req: signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "", - Width: 0, - Height: 0, - Format: testFormatPNG, - Quality: 85, - FitMode: testFitCover, + req: imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 0, Height: 0}, + Format: testFormatPNG, + Quality: 85, + FitMode: testFitCover, + }, }, // Signed data: // "cdn.example.com:/photos/cat.jpg::0:0:png:1704067200:85:cover" @@ -86,15 +96,18 @@ func goldenVectors() []goldenVector { }, { name: "non-default quality and fit", - req: signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "", - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 40, - FitMode: testFitContain, + req: imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 40, + FitMode: testFitContain, + }, }, // Signed data: // "cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:40:contain" diff --git a/internal/signature/quality_fit_test.go b/internal/signature/quality_fit_test.go index bebbec4..8821b7b 100644 --- a/internal/signature/quality_fit_test.go +++ b/internal/signature/quality_fit_test.go @@ -5,22 +5,26 @@ import ( "testing" "time" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/signature" ) // signedQualityFitRequest returns a request signed for quality 85 and fit // mode "cover", the effective defaults the handler applies before // verification. -func signedQualityFitRequest(signer *signature.Signer) *signature.Request { - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 85, - FitMode: testFitCover, - Expires: time.Now().Add(1 * time.Hour), +func signedQualityFitRequest(signer *signature.Signer) *imagerequest.Request { + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + }, + Expires: time.Now().Add(1 * time.Hour), } req.Signature = signer.Sign(req) @@ -40,15 +44,15 @@ func TestSigner_Verify_QualityAndFitAreSigned(t *testing.T) { cases := []struct { name string - tamper func(r *signature.Request) + tamper func(r *imagerequest.Request) }{ { name: "replayed with different quality", - tamper: func(r *signature.Request) { r.Quality = 40 }, + tamper: func(r *imagerequest.Request) { r.Transform.Quality = 40 }, }, { name: "replayed with different fit mode", - tamper: func(r *signature.Request) { r.FitMode = testFitContain }, + tamper: func(r *imagerequest.Request) { r.Transform.FitMode = testFitContain }, }, } diff --git a/internal/signature/signature.go b/internal/signature/signature.go index a507240..beff9b4 100644 --- a/internal/signature/signature.go +++ b/internal/signature/signature.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // Signature errors. @@ -22,37 +24,6 @@ var ( ErrMissingExpiration = errors.New("signature expiration is required") ) -// Request carries the components an image request signature covers. It is a -// standalone type so that this package does not depend on imgcache, keeping -// the import edge one-way (imgcache depends on signature, never the reverse). -type Request struct { - // SourceHost is the origin host (e.g. "cdn.example.com"). - SourceHost string - // SourcePath is the path on the origin (e.g. "/photos/cat.jpg"). - SourcePath string - // SourceQuery is the optional query string for the origin URL. - SourceQuery string - // Width is the requested output width in pixels. - Width int - // Height is the requested output height in pixels. - Height int - // Format is the requested output format (e.g. "webp"). - Format string - // Quality is the requested output quality (1-100) for lossy formats. - // It is the effective value the request resolves to: callers pass the - // default quality when the request omits the parameter, so an omitted - // quality signs identically to that same value stated explicitly. - Quality int - // FitMode is how the image is fit into the requested dimensions - // (e.g. "cover"). Like Quality it is the effective value: callers pass - // the default fit mode when the request omits the parameter. - FitMode string - // Signature is the HMAC signature to verify. - Signature string - // Expires is the signature expiration timestamp. - Expires time.Time -} - // Signer handles HMAC-SHA256 signature generation and verification. type Signer struct { secretKey []byte @@ -67,8 +38,11 @@ func New(secretKey string) *Signer { // Sign generates an HMAC-SHA256 signature for the given request. // The signature covers: host + path + query + width + height + format + -// expiration + quality + fit. -func (s *Signer) Sign(req *Request) string { +// expiration + quality + fit. The quality and fit mode are the effective +// values the request resolves to: callers pass the defaults when the request +// omits those parameters, so an omitted quality or fit mode signs +// identically to that same value stated explicitly. +func (s *Signer) Sign(req *imagerequest.Request) string { data := s.buildSignatureData(req) mac := hmac.New(sha256.New, s.secretKey) mac.Write([]byte(data)) @@ -84,7 +58,7 @@ func (s *Signer) Sign(req *Request) string { // No suffix matching, wildcard matching, or partial matching is supported. // A signature for "cdn.example.com" will NOT verify for "example.com" or // "other.cdn.example.com", and vice versa. -func (s *Signer) Verify(req *Request) error { +func (s *Signer) Verify(req *imagerequest.Request) error { // Check expiration first if req.Expires.IsZero() { return ErrMissingExpiration @@ -108,7 +82,7 @@ func (s *Signer) Verify(req *Request) error { // GenerateSignedURL creates a complete URL with signature and expiration. // Returns the path portion that should be appended to the base URL. func (s *Signer) GenerateSignedURL( - req *Request, ttl time.Duration, + req *imagerequest.Request, ttl time.Duration, ) (string, string, int64) { // Set expiration req.Expires = time.Now().Add(ttl) @@ -120,10 +94,11 @@ func (s *Signer) GenerateSignedURL( // Build the size component var sizeStr string - if req.Width == 0 && req.Height == 0 { + if req.Transform.Size.Width == 0 && req.Transform.Size.Height == 0 { sizeStr = "orig" } else { - sizeStr = fmt.Sprintf("%dx%d", req.Width, req.Height) + sizeStr = fmt.Sprintf("%dx%d", + req.Transform.Size.Width, req.Transform.Size.Height) } // Build the path. A source query follows the source path after a "?". @@ -131,13 +106,13 @@ func (s *Signer) GenerateSignedURL( // written in Go's own percent-encoding of a URL path, which writes "?" // as %3F and "%" as %25, and leaves "/", ";", ",", "=" and "&" as they // are. - sourcePathAndQuery := req.SourcePath - if req.SourceQuery != "" { - sourcePathAndQuery += "?" + req.SourceQuery + sourcePathAndQuery := req.Source.Path + if req.Source.Query != "" { + sourcePathAndQuery += "?" + req.Source.Query } unescapedPath := fmt.Sprintf("/v1/image/%s%s/%s.%s", - req.SourceHost, sourcePathAndQuery, sizeStr, req.Format) + req.Source.Host, sourcePathAndQuery, sizeStr, req.Transform.Format) return (&url.URL{Path: unescapedPath}).EscapedPath(), sig, exp } @@ -150,17 +125,17 @@ func (s *Signer) GenerateSignedURL( // expansion is performed. Quality and fit are the effective transform // values, so replaying a signed URL with a different quality or fit mode // fails verification. -func (s *Signer) buildSignatureData(req *Request) string { +func (s *Signer) buildSignatureData(req *imagerequest.Request) string { return fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%d:%s", - req.SourceHost, - escapePercentAndColon(req.SourcePath), - escapePercentAndColon(req.SourceQuery), - req.Width, - req.Height, - req.Format, + req.Source.Host, + escapePercentAndColon(req.Source.Path), + escapePercentAndColon(req.Source.Query), + req.Transform.Size.Width, + req.Transform.Size.Height, + req.Transform.Format, req.Expires.Unix(), - req.Quality, - req.FitMode, + req.Transform.Quality, + req.Transform.FitMode, ) } diff --git a/internal/signature/signature_test.go b/internal/signature/signature_test.go index 559d19c..119fb14 100644 --- a/internal/signature/signature_test.go +++ b/internal/signature/signature_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/signature" ) @@ -26,14 +27,17 @@ func TestSigner_Sign(t *testing.T) { signer := signature.New("test-secret-key") - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "", - Width: 800, - Height: 600, - Format: testFormatWebP, - Expires: time.Unix(1704067200, 0), // Fixed timestamp for reproducibility + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, + Expires: time.Unix(1704067200, 0), // Fixed timestamp for reproducibility } sig1 := signer.Sign(req) @@ -51,14 +55,17 @@ func TestSigner_Sign(t *testing.T) { } // Different input should produce different signature - req2 := &signature.Request{ - SourceHost: testHost, - SourcePath: "/photos/dog.jpg", // Different path - SourceQuery: "", - Width: 800, - Height: 600, - Format: testFormatWebP, - Expires: time.Unix(1704067200, 0), + req2 := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: "/photos/dog.jpg", // Different path + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, + Expires: time.Unix(1704067200, 0), } sig3 := signer.Sign(req2) @@ -69,20 +76,23 @@ func TestSigner_Sign(t *testing.T) { // validVerifyRequest returns a fully-populated request that verifies // successfully once signed. -func validVerifyRequest() *signature.Request { - return &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - Width: 800, - Height: 600, - Format: testFormatWebP, - Expires: time.Now().Add(1 * time.Hour), +func validVerifyRequest() *imagerequest.Request { + return &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, + Expires: time.Now().Add(1 * time.Hour), } } type verifyCase struct { name string - setup func() *signature.Request + setup func() *imagerequest.Request wantErr error } @@ -90,7 +100,7 @@ func verifyCases(signer *signature.Signer) []verifyCase { return []verifyCase{ { name: "valid signature", - setup: func() *signature.Request { + setup: func() *imagerequest.Request { req := validVerifyRequest() req.Signature = signer.Sign(req) @@ -100,7 +110,7 @@ func verifyCases(signer *signature.Signer) []verifyCase { }, { name: "expired signature", - setup: func() *signature.Request { + setup: func() *imagerequest.Request { req := validVerifyRequest() req.Expires = time.Now().Add(-1 * time.Hour) req.Signature = signer.Sign(req) @@ -111,7 +121,7 @@ func verifyCases(signer *signature.Signer) []verifyCase { }, { name: "invalid signature", - setup: func() *signature.Request { + setup: func() *imagerequest.Request { req := validVerifyRequest() req.Signature = "invalid-signature" @@ -121,7 +131,7 @@ func verifyCases(signer *signature.Signer) []verifyCase { }, { name: "missing expiration", - setup: func() *signature.Request { + setup: func() *imagerequest.Request { req := validVerifyRequest() req.Expires = time.Time{} req.Signature = "some-signature" @@ -132,10 +142,10 @@ func verifyCases(signer *signature.Signer) []verifyCase { }, { name: "tampered request", - setup: func() *signature.Request { + setup: func() *imagerequest.Request { req := validVerifyRequest() req.Signature = signer.Sign(req) - req.SourcePath = "/photos/secret.jpg" + req.Source.Path = "/photos/secret.jpg" return req }, @@ -173,7 +183,7 @@ func TestSigner_Verify(t *testing.T) { type tamperCase struct { name string - tamper func(r *signature.Request) + tamper func(r *imagerequest.Request) } // exactMatchTamperCases mutates one signed component per case; every @@ -182,59 +192,59 @@ func exactMatchTamperCases() []tamperCase { return []tamperCase{ { name: "parent domain does not match subdomain", - tamper: func(r *signature.Request) { r.SourceHost = "example.com" }, + tamper: func(r *imagerequest.Request) { r.Source.Host = "example.com" }, }, { name: "subdomain does not match parent domain", - tamper: func(r *signature.Request) { r.SourceHost = "images.cdn.example.com" }, + tamper: func(r *imagerequest.Request) { r.Source.Host = "images.cdn.example.com" }, }, { name: "sibling subdomain does not match", - tamper: func(r *signature.Request) { r.SourceHost = "images.example.com" }, + tamper: func(r *imagerequest.Request) { r.Source.Host = "images.example.com" }, }, { name: "host with suffix appended does not match", - tamper: func(r *signature.Request) { r.SourceHost = testHost + ".evil.com" }, + tamper: func(r *imagerequest.Request) { r.Source.Host = testHost + ".evil.com" }, }, { name: "host with prefix does not match", - tamper: func(r *signature.Request) { r.SourceHost = "evilcdn.example.com" }, + tamper: func(r *imagerequest.Request) { r.Source.Host = "evilcdn.example.com" }, }, { name: "different path does not match", - tamper: func(r *signature.Request) { r.SourcePath = "/photos/dog.jpg" }, + tamper: func(r *imagerequest.Request) { r.Source.Path = "/photos/dog.jpg" }, }, { name: "path suffix does not match", - tamper: func(r *signature.Request) { r.SourcePath = testPath + "/extra" }, + tamper: func(r *imagerequest.Request) { r.Source.Path = testPath + "/extra" }, }, { name: "path prefix does not match", - tamper: func(r *signature.Request) { r.SourcePath = "/other" + testPath }, + tamper: func(r *imagerequest.Request) { r.Source.Path = "/other" + testPath }, }, { name: "different query does not match", - tamper: func(r *signature.Request) { r.SourceQuery = "token=xyz" }, + tamper: func(r *imagerequest.Request) { r.Source.Query = "token=xyz" }, }, { name: "added query does not match empty query", - tamper: func(r *signature.Request) { r.SourceQuery = "extra=1" }, + tamper: func(r *imagerequest.Request) { r.Source.Query = "extra=1" }, }, { name: "removed query does not match", - tamper: func(r *signature.Request) { r.SourceQuery = "" }, + tamper: func(r *imagerequest.Request) { r.Source.Query = "" }, }, { name: "different width does not match", - tamper: func(r *signature.Request) { r.Width = 801 }, + tamper: func(r *imagerequest.Request) { r.Transform.Size.Width = 801 }, }, { name: "different height does not match", - tamper: func(r *signature.Request) { r.Height = 601 }, + tamper: func(r *imagerequest.Request) { r.Transform.Size.Height = 601 }, }, { name: "different format does not match", - tamper: func(r *signature.Request) { r.Format = testFormatPNG }, + tamper: func(r *imagerequest.Request) { r.Transform.Format = testFormatPNG }, }, } } @@ -248,15 +258,18 @@ func TestSigner_Verify_ExactMatchOnly(t *testing.T) { signer := signature.New("test-secret-key") // Base request that we'll sign, then tamper with individual fields. - baseReq := func() *signature.Request { - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "token=abc", - Width: 800, - Height: 600, - Format: testFormatWebP, - Expires: time.Now().Add(1 * time.Hour), + baseReq := func() *imagerequest.Request { + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "token=abc", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, + Expires: time.Now().Add(1 * time.Hour), } req.Signature = signer.Sign(req) @@ -309,14 +322,17 @@ func TestSigner_Sign_ExactHostInData(t *testing.T) { sigs := make(map[string]string) for _, host := range hosts { - req := &signature.Request{ - SourceHost: host, - SourcePath: testPath, - SourceQuery: "", - Width: 800, - Height: 600, - Format: testFormatWebP, - Expires: time.Unix(1704067200, 0), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: host, + Path: testPath, + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, + Expires: time.Unix(1704067200, 0), } sig := signer.Sign(req) @@ -334,13 +350,16 @@ func TestSigner_DifferentKeys(t *testing.T) { signer1 := signature.New("secret-key-1") signer2 := signature.New("secret-key-2") - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - Width: 800, - Height: 600, - Format: testFormatWebP, - Expires: time.Now().Add(1 * time.Hour), + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, + Expires: time.Now().Add(1 * time.Hour), } // Sign with key 1 @@ -364,13 +383,16 @@ func TestGenerateSignedURL(t *testing.T) { signer := signature.New("test-secret-key") - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "", - Width: 800, - Height: 600, - Format: testFormatWebP, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, } ttl := 1 * time.Hour @@ -405,12 +427,15 @@ func TestGenerateSignedURL_OrigSize(t *testing.T) { signer := signature.New("test-secret-key") - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - Width: 0, // Original size - Height: 0, - Format: testFormatPNG, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 0, Height: 0}, // Original size + Format: testFormatPNG, + }, } path, _, _ := signer.GenerateSignedURL(req, time.Hour) @@ -426,13 +451,16 @@ func TestGenerateSignedURL_WithQueryString(t *testing.T) { signer := signature.New("test-secret-key-for-testing!") - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - SourceQuery: "token=abc&v=2", - Width: 800, - Height: 600, - Format: testFormatWebP, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + Query: "token=abc&v=2", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, } path, _, _ := signer.GenerateSignedURL(req, time.Hour) @@ -460,12 +488,15 @@ func TestGenerateSignedURL_WithoutQueryString(t *testing.T) { signer := signature.New("test-secret-key-for-testing!") - req := &signature.Request{ - SourceHost: testHost, - SourcePath: testPath, - Width: 800, - Height: 600, - Format: testFormatWebP, + req := &imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: testPath, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + }, } path, _, _ := signer.GenerateSignedURL(req, time.Hour) @@ -485,16 +516,19 @@ func TestGenerateSignedURL_ReadmeQueryExample(t *testing.T) { signer := signature.New("example-signing-key-for-documentation") - req := signature.Request{ - SourceHost: testHost, - SourcePath: "/img", - SourceQuery: "url=https://origin.example.com/my%20cat.jpg", - Width: 800, - Height: 600, - Format: testFormatWebP, - Quality: 85, - FitMode: testFitCover, - Expires: time.Unix(1704067200, 0), + req := imagerequest.Request{ + Source: imagerequest.Source{ + Host: testHost, + Path: "/img", + Query: "url=https://origin.example.com/my%20cat.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 800, Height: 600}, + Format: testFormatWebP, + Quality: 85, + FitMode: testFitCover, + }, + Expires: time.Unix(1704067200, 0), } const wantSig = "37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=" From a19cca2f347cf3f255f4f3ee40228e897574b9ac Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 21 Oct 2026 16:31:33 +0200 Subject: [PATCH 098/131] Store a source the way a variant is stored, one row and one file per source URL (closes #308) A source is now one source_content row and one file named by its SourceKey, the SHA-256 of its host, path and query, made as the variant key is. A store writes the file, then upserts the row, and removes the file when the row cannot be written; an eviction deletes the row, then the file, with no lock. source_metadata, the JSON files under cache/metadata/, MetadataStorage, the content lock, the eviction transaction and its test hook are gone, as is the reconciliation pass that dropped a source row whose file is missing. Two source URLs serving the same bytes store them twice. Model: opus-5-5 --- README.md | 22 +- TODO.md | 8 + .../concurrent_writes_internal_test.go | 3 +- internal/database/database_internal_test.go | 2 +- internal/db/migrations/001_schema.sql | 34 +- internal/imagerequest/imagerequest.go | 6 +- internal/imagerequest/sourcekey.go | 26 + .../imagerequest/sourcekey_internal_test.go | 40 ++ internal/imgcache/cache.go | 213 ++------ internal/imgcache/cache_internal_test.go | 27 +- .../imgcache/cache_usage_internal_test.go | 18 +- internal/imgcache/contentlock.go | 64 --- .../imgcache/contentlock_internal_test.go | 142 ------ internal/imgcache/eviction.go | 248 ++------- internal/imgcache/eviction_internal_test.go | 480 ++++++------------ ...max_concurrent_processing_internal_test.go | 8 +- .../imgcache/negative_cache_internal_test.go | 8 +- .../partial_cache_files_internal_test.go | 29 +- internal/imgcache/service.go | 14 +- internal/imgcache/service_internal_test.go | 58 +-- .../imgcache/source_content_internal_test.go | 146 ++++++ internal/imgcache/storage.go | 265 ++-------- internal/imgcache/storage_internal_test.go | 201 +------- internal/server/image_proxy_internal_test.go | 28 +- 24 files changed, 622 insertions(+), 1468 deletions(-) create mode 100644 internal/imagerequest/sourcekey.go create mode 100644 internal/imagerequest/sourcekey_internal_test.go delete mode 100644 internal/imgcache/contentlock.go delete mode 100644 internal/imgcache/contentlock_internal_test.go create mode 100644 internal/imgcache/source_content_internal_test.go diff --git a/README.md b/README.md index 9d1e2c4..cc09929 100644 --- a/README.md +++ b/README.md @@ -76,10 +76,9 @@ database and the disk cache: pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user must be able to write the directory. - `cache_max_bytes` limits the source and transformed images together. The - database, the metadata files, the `.meta` file beside each transformed image - and files still being written come on top, and eviction runs in the - background, so the cache can pass the limit for a while: leave room on the - volume beyond it. + database, the `.meta` file beside each transformed image and files still being + written come on top, and eviction runs in the background, so the cache can + pass the limit for a while: leave room on the volume beyond it. - Set `cache_max_bytes` for a lasting deployment. Its default, worked out each time pixa starts, is 75% of the sum of the space free on the volume and the space the cached images already take, so a restart keeps the limit the cache @@ -177,11 +176,9 @@ allowlisted source hosts for open access. ### Storage -- **Source content**: - `/cache/sources///` -- **Source metadata**: - `/cache/metadata//.json` (host, - path and query, content hash, upstream status and headers, fetch time) +- **Source images**: + `/cache/sources///`, one + file for each source URL - **Database**: `/state.sqlite3` (SQLite) - **Transformed images**: `/cache/variants///`, @@ -190,8 +187,11 @@ allowlisted source hosts for open access. `` and `` are the first and second pairs of characters of the file's name. -Multiple source paths may reference the same content blob; the database tracks -references rather than using filesystem refcounting. +Each source image and each transformed image has one row in the database, +holding its size and when it was last used. Two source URLs that serve the same +bytes are stored twice, once under each URL. A source image is stored by writing +its file and then its row; when the row cannot be written, the file is removed +again. Source and transformed image files, and the `.meta` file beside each transformed image, are written under a temporary name, synced to disk, and only then renamed diff --git a/TODO.md b/TODO.md index 2290071..049f2c8 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,14 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-21 a source is stored the way a variant is (closes #308): one + `source_content` row and one file per source URL, the file named by the + source's `SourceKey`, the SHA-256 of its host, path and query made as the + variant key is, under `cache/sources/`. A store writes the file, then the row, + and removes the file when the row cannot be written; an eviction deletes the + row, then the file, with no lock and no transaction. `source_metadata`, the + JSON files under `cache/metadata/`, `MetadataStorage` and the content lock are + gone. Two source URLs that serve the same bytes now store them twice. - 2026-10-21 one image request type instead of five (closes #307): the new package `internal/imagerequest`, which imports no other package of pixa, holds `imagerequest.Request` with its two parts, `Source` (host, path and query) and diff --git a/internal/database/concurrent_writes_internal_test.go b/internal/database/concurrent_writes_internal_test.go index c874a6f..9644aea 100644 --- a/internal/database/concurrent_writes_internal_test.go +++ b/internal/database/concurrent_writes_internal_test.go @@ -131,7 +131,8 @@ func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error { } _, err = db.ExecContext(ctx, `INSERT INTO source_content - (content_hash, content_type, size_bytes) VALUES (?, 'image/png', 1)`, key) + (source_key, source_host, source_path, content_type, size_bytes) + VALUES (?, 'example.com', '/image.png', 'image/png', 1)`, key) if err != nil { return fmt.Errorf("storing a source: %w", err) } diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go index 4b70909..c19511a 100644 --- a/internal/database/database_internal_test.go +++ b/internal/database/database_internal_test.go @@ -160,7 +160,7 @@ func TestApplyMigrations_CreatesSchemaAndTables(t *testing.T) { // Verify that the application tables created by 001.sql exist. tables := []string{ - "source_content", "source_metadata", "output_content", + "source_content", "output_content", "request_cache", "negative_cache", "cache_stats", } for _, table := range tables { diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index 234a141..71d9ec9 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -1,10 +1,14 @@ -- Migration 001: Initial schema -- Creates all tables for the pixa caching image proxy --- Source content blobs --- Files stored at: cache/sources/// +-- Source images, one row per source URL +-- Files stored at: cache/sources///, where source_key +-- is the SHA-256 of the source's host, path and query CREATE TABLE IF NOT EXISTS source_content ( - content_hash TEXT PRIMARY KEY, + source_key TEXT PRIMARY KEY, + source_host TEXT NOT NULL, + source_path TEXT NOT NULL, + source_query TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL, size_bytes INTEGER NOT NULL, fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, @@ -13,30 +17,6 @@ CREATE TABLE IF NOT EXISTS source_content ( CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed ON source_content(last_accessed_at); --- Source URL metadata - maps URLs to content hashes --- JSON stored at: cache/metadata//.json -CREATE TABLE IF NOT EXISTS source_metadata ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - source_host TEXT NOT NULL, - source_path TEXT NOT NULL, - source_query TEXT NOT NULL DEFAULT '', - path_hash TEXT NOT NULL, - content_hash TEXT, - status_code INTEGER NOT NULL, - content_type TEXT, - response_headers TEXT, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, - expires_at DATETIME, - etag TEXT, - last_modified TEXT, - UNIQUE(source_host, source_path, source_query), - FOREIGN KEY (content_hash) REFERENCES source_content(content_hash) -); -CREATE INDEX IF NOT EXISTS idx_source_meta_host ON source_metadata(source_host); -CREATE INDEX IF NOT EXISTS idx_source_meta_path_hash ON source_metadata(path_hash); -CREATE INDEX IF NOT EXISTS idx_source_meta_expires ON source_metadata(expires_at); -CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(content_hash); - -- Processed variant blobs -- Files stored at: cache/variants/// (plus a .meta -- sidecar with the content type). Tracked here (like source content diff --git a/internal/imagerequest/imagerequest.go b/internal/imagerequest/imagerequest.go index 7b30a61..ecc86b4 100644 --- a/internal/imagerequest/imagerequest.go +++ b/internal/imagerequest/imagerequest.go @@ -1,7 +1,7 @@ // Package imagerequest holds the image request type, its parts, its checks, -// the parsing of the /v1/image/ path and the variant key. It imports no other -// package of this repo, so every package that handles an image request can -// import it. +// the parsing of the /v1/image/ path, the variant key and the source key. It +// imports no other package of this repo, so every package that handles an +// image request can import it. package imagerequest import ( diff --git a/internal/imagerequest/sourcekey.go b/internal/imagerequest/sourcekey.go new file mode 100644 index 0000000..28cb632 --- /dev/null +++ b/internal/imagerequest/sourcekey.go @@ -0,0 +1,26 @@ +package imagerequest + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" +) + +// SourceKey is a SHA256 hash identifying a cached source image (hex-encoded). +type SourceKey string + +// Key returns the key the source is cached under, made as CacheKey makes a +// variant key. +// Format: sha256(host:path:query) +// The path and query go through escapePercentAndColon, so that a ":" in +// either cannot be taken for the one between them. +func (s Source) Key() SourceKey { + data := fmt.Sprintf("%s:%s:%s", + s.Host, + escapePercentAndColon(s.Path), + escapePercentAndColon(s.Query), + ) + h := sha256.Sum256([]byte(data)) + + return SourceKey(hex.EncodeToString(h[:])) +} diff --git a/internal/imagerequest/sourcekey_internal_test.go b/internal/imagerequest/sourcekey_internal_test.go new file mode 100644 index 0000000..9793d5b --- /dev/null +++ b/internal/imagerequest/sourcekey_internal_test.go @@ -0,0 +1,40 @@ +package imagerequest + +import "testing" + +// TestSourceKey_DiffersForEachSource checks that each source gets a key of +// 64 hex characters, the same each time, and that no two of these sources +// share one, also when they differ only by a ":" or "%" in the path or query. +func TestSourceKey_DiffersForEachSource(t *testing.T) { + t.Parallel() + + sources := []Source{ + {Host: testHostCDN, Path: testPathCat}, + {Host: testHostCDN, Path: testPathCat, Query: "v=2"}, + {Host: "other.example.com", Path: testPathCat}, + {Host: testHostCDN, Path: "/c:d"}, + {Host: testHostCDN, Path: "/c", Query: "d:"}, + {Host: testHostCDN, Path: "/c%3Ad"}, + } + + seen := make(map[SourceKey]Source) + + for _, source := range sources { + key := source.Key() + t.Logf("Key() of %+v = %s", source, key) + + if len(key) != 64 { + t.Errorf("Key() of %+v length = %d, want 64", source, len(key)) + } + + if again := source.Key(); again != key { + t.Errorf("Key() of %+v not deterministic: %s vs %s", source, key, again) + } + + if earlier, found := seen[key]; found { + t.Errorf("Key() gave %+v and %+v the same key", earlier, source) + } + + seen[key] = source + } +} diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 35bac85..abefe36 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -2,10 +2,7 @@ package imgcache import ( "context" - "crypto/sha256" "database/sql" - "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -26,9 +23,6 @@ var ( ErrNegativeCache = errors.New("negative cache hit") ) -// HTTP status code for successful fetch. -const httpStatusOK = 200 - // metaCacheSize is how many variants' content types metaCache holds. A // variant not among them is served as before, reading its .meta file. const metaCacheSize = 10000 @@ -69,12 +63,11 @@ type CacheConfig struct { // Cache implements the caching layer for the image proxy. type Cache struct { - db *sql.DB - srcContent *ContentStorage // source images by content hash - variants *VariantStorage // processed variants by cache key - srcMetadata *MetadataStorage // source metadata by host/path - config CacheConfig - log *slog.Logger + db *sql.DB + srcContent *ContentStorage // source images by source key + variants *VariantStorage // processed variants by cache key + config CacheConfig + log *slog.Logger // disabled means the disk cache is turned off entirely: lookups // always miss, stores are no-ops, and no eviction runs. @@ -114,19 +107,6 @@ type Cache struct { // It never stands in for the variant file, which is always opened. metaCache *lru.Cache[imagerequest.VariantKey, string] - // contentLocks serializes StoreSource and evictSourceBlob per - // content hash, closing the race window between an eviction's row - // deletion and its file unlink against a concurrent store of - // identical content. - contentLocks *contentLock - - // evictSourceBlobTestHook, when set, is invoked by evictSourceBlob - // after its row-deletion transaction commits and before the - // content file is unlinked. It exists solely so tests can - // deterministically pause inside that window to exercise - // concurrent stores against it; production code leaves it nil. - evictSourceBlobTestHook func(ContentHash) - // reconciliationPageSize is the most rows one read of a content table // returns in the reconciliation pass and in Stats. newCache sets it to // defaultReconciliationPageSize; tests set it smaller. @@ -167,7 +147,6 @@ func newCache( evictionPressure: make(chan struct{}, 1), evictionDone: make(chan struct{}), metaCache: metaCache, - contentLocks: newContentLock(), pendingCountsAdded: make(chan struct{}, 1), pendingCountsDone: make(chan struct{}), @@ -193,16 +172,8 @@ func newCache( return nil, fmt.Errorf("failed to create variant storage: %w", err) } - srcMetadata, err := NewMetadataStorage( - filepath.Join(config.StateDir, "cache", "metadata"), - ) - if err != nil { - return nil, fmt.Errorf("failed to create source metadata storage: %w", err) - } - c.srcContent = srcContent c.variants = variants - c.srcMetadata = srcMetadata if config.UseDefaultMaxBytes { limit, err := c.computeDefaultMaxBytes(context.Background(), probe) @@ -283,88 +254,59 @@ func (c *Cache) GetVariant( return reader, size, contentType, nil } -// StoreSource stores fetched source content and metadata. On a -// disabled cache it is a no-op returning an empty hash. +// StoreSource stores fetched source content: it writes the file named by +// the source's key, then the source's row. When the row cannot be written +// the file is removed and the store fails, as a file with no row is never +// served or evicted; if that removal fails too, the next reconciliation +// pass removes the file. On a disabled cache it is a no-op returning an +// empty key. func (c *Cache) StoreSource( ctx context.Context, req *imagerequest.Request, content io.Reader, result *httpfetcher.FetchResult, -) (ContentHash, error) { +) (imagerequest.SourceKey, error) { if c.disabled { return "", nil } - // Hash the content ourselves (rather than via srcContent.Store, - // which would hash internally) so the content hash is known before - // any file or database work happens: that lets the entire store be - // serialized, per hash, against a concurrent eviction of the same - // content below. data, err := io.ReadAll(content) if err != nil { return "", fmt.Errorf("failed to read source content: %w", err) } - sum := sha256.Sum256(data) - contentHash := ContentHash(hex.EncodeToString(sum[:])) + key := req.Source.Key() - // Hold the content hash's lock for the whole store operation. A - // concurrent eviction of this exact hash (the real SHA-256 dedup - // case: a different source path whose bytes hash identically) - // deletes the accounting rows and unlinks the file inside the same - // lock, so the two can never interleave: either this store - // completes first (and a subsequent eviction removes it together - // with its rows and file, correctly), or eviction completes first - // (and this store finds the file already gone and recreates it - // fresh) — never a fresh row left pointing at a file eviction is - // mid-unlink on. - unlock := c.contentLocks.Lock(string(contentHash)) - defer unlock() - - size, err := c.srcContent.StoreHashed(contentHash, data) + size, err := c.srcContent.Store(key, data) if err != nil { return "", fmt.Errorf("failed to store source content: %w", err) } - // Store in database - pathHash := HashPath(req.Source.Path + "?" + req.Source.Query) - - headersJSON, err := json.Marshal(result.Headers) - if err != nil { - return "", fmt.Errorf("failed to marshal response headers: %w", err) - } - _, err = c.db.ExecContext(ctx, ` - INSERT INTO source_content (content_hash, content_type, size_bytes) - VALUES (?, ?, ?) - ON CONFLICT(content_hash) DO NOTHING - `, contentHash, result.ContentType, size) + INSERT INTO source_content + (source_key, source_host, source_path, source_query, + content_type, size_bytes) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(source_key) DO UPDATE SET + content_type = excluded.content_type, + size_bytes = excluded.size_bytes, + fetched_at = CURRENT_TIMESTAMP, + last_accessed_at = CURRENT_TIMESTAMP + `, string(key), req.Source.Host, req.Source.Path, req.Source.Query, + result.ContentType, size) if err != nil { + deleteErr := c.srcContent.Delete(key) + if deleteErr != nil { + c.log.Warn("failed to remove source file whose row was not written", + "source_key", key, "error", deleteErr) + } + return "", fmt.Errorf("failed to insert source content: %w", err) } - _, err = c.db.ExecContext(ctx, ` - INSERT INTO source_metadata - (source_host, source_path, source_query, path_hash, - content_hash, status_code, content_type, response_headers) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(source_host, source_path, source_query) DO UPDATE SET - content_hash = excluded.content_hash, - status_code = excluded.status_code, - content_type = excluded.content_type, - response_headers = excluded.response_headers, - fetched_at = CURRENT_TIMESTAMP - `, req.Source.Host, req.Source.Path, req.Source.Query, pathHash, - contentHash, httpStatusOK, result.ContentType, string(headersJSON)) - if err != nil { - return "", fmt.Errorf("failed to insert source metadata: %w", err) - } - - c.writeMetadataSidecar(req, pathHash, contentHash, result) - c.notifyWritePressure() - return contentHash, nil + return key, nil } // StoreVariant stores a processed variant by its cache key and records @@ -405,22 +347,24 @@ func (c *Cache) StoreVariant( } // LookupSource checks if we have cached source content for a request. -// Returns the content hash and content type if found, or empty values -// if not. Hits touch the blob's LRU timestamp; a disabled cache always -// reports no cached source. +// Returns the source key and content type if found, or empty values +// if not; a row whose file is missing is not found. Hits touch the +// source's LRU timestamp; a disabled cache always reports no cached +// source. func (c *Cache) LookupSource( ctx context.Context, req *imagerequest.Request, -) (ContentHash, string, error) { +) (imagerequest.SourceKey, string, error) { if c.disabled { return "", "", nil } - var hashStr, contentType string + key := req.Source.Key() + + var contentType string err := c.db.QueryRowContext(ctx, ` - SELECT content_hash, content_type FROM source_metadata - WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.Source.Host, req.Source.Path, req.Source.Query).Scan(&hashStr, &contentType) + SELECT content_type FROM source_content WHERE source_key = ? + `, string(key)).Scan(&contentType) if errors.Is(err, sql.ErrNoRows) { return "", "", nil @@ -430,16 +374,14 @@ func (c *Cache) LookupSource( return "", "", fmt.Errorf("failed to lookup source: %w", err) } - contentHash := ContentHash(hashStr) - // Verify the content file exists - if !c.srcContent.Exists(contentHash) { + if !c.srcContent.Exists(key) { return "", "", nil } - c.touchSourceContent(ctx, contentHash) + c.touchSourceContent(ctx, key) - return contentHash, contentType, nil + return key, contentType, nil } // StoreNegative stores a negative cache entry for a failed fetch. @@ -466,33 +408,16 @@ func (c *Cache) StoreNegative( return nil } -// GetSourceMetadataID returns the source metadata ID for a request. -func (c *Cache) GetSourceMetadataID( - ctx context.Context, req *imagerequest.Request, -) (int64, error) { - var id int64 - - err := c.db.QueryRowContext(ctx, ` - SELECT id FROM source_metadata - WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.Source.Host, req.Source.Path, req.Source.Query).Scan(&id) - if err != nil { - return 0, fmt.Errorf("failed to get source metadata ID: %w", err) - } - - return id, nil -} - -// GetSourceContent returns a reader for cached source content by its hash, +// GetSourceContent returns a reader for cached source content by its key, // and the content's size in bytes. func (c *Cache) GetSourceContent( - contentHash ContentHash, + key imagerequest.SourceKey, ) (io.ReadCloser, int64, error) { if c.disabled { return nil, 0, ErrNotFound } - return c.srcContent.LoadWithSize(contentHash) + return c.srcContent.LoadWithSize(key) } // CleanExpired removes the negative cache rows past their expiry, one @@ -687,36 +612,6 @@ func (c *Cache) loadVariantWithMeta( return reader, size, contentType, nil } -// writeMetadataSidecar writes the JSON metadata sidecar of a stored source. -// A failure is logged and is otherwise non-fatal; the metadata is in the -// database. -func (c *Cache) writeMetadataSidecar( - req *imagerequest.Request, - pathHash PathHash, - contentHash ContentHash, - result *httpfetcher.FetchResult, -) { - meta := &SourceMetadata{ - Host: req.Source.Host, - Path: req.Source.Path, - Query: req.Source.Query, - ContentHash: string(contentHash), - StatusCode: result.StatusCode, - ContentType: result.ContentType, - ContentLength: result.ContentLength, - ResponseHeaders: result.Headers, - FetchedAt: time.Now().UTC().Unix(), - FetchDurationMs: result.FetchDurationMs, - RemoteAddr: result.RemoteAddr, - } - - err := c.srcMetadata.Store(req.Source.Host, pathHash, meta) - if err != nil { - c.log.Warn("failed to write metadata sidecar", - "host", req.Source.Host, "path_hash", pathHash, "error", err) - } -} - // touchVariant updates the LRU timestamp of a variant, best-effort: // a failed touch only makes the entry look colder to eviction. func (c *Cache) touchVariant(ctx context.Context, cacheKey imagerequest.VariantKey) { @@ -730,16 +625,16 @@ func (c *Cache) touchVariant(ctx context.Context, cacheKey imagerequest.VariantK } } -// touchSourceContent updates the LRU timestamp of a source content -// blob, best-effort: a failed touch only makes the blob look colder. -func (c *Cache) touchSourceContent(ctx context.Context, contentHash ContentHash) { +// touchSourceContent updates the LRU timestamp of a cached source, +// best-effort: a failed touch only makes the source look colder. +func (c *Cache) touchSourceContent(ctx context.Context, key imagerequest.SourceKey) { _, err := c.db.ExecContext(ctx, ` UPDATE source_content SET last_accessed_at = CURRENT_TIMESTAMP - WHERE content_hash = ? - `, string(contentHash)) + WHERE source_key = ? + `, string(key)) if err != nil { c.log.Debug("failed to touch source content LRU timestamp", - "content_hash", contentHash, "error", err) + "source_key", key, "error", err) } } diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index 95966e2..0150e73 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -24,26 +24,14 @@ func setupTestDB(t *testing.T) *sql.DB { // Create schema schema := ` CREATE TABLE source_content ( - content_hash TEXT PRIMARY KEY, - content_type TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE TABLE source_metadata ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + source_key TEXT PRIMARY KEY, source_host TEXT NOT NULL, source_path TEXT NOT NULL, source_query TEXT NOT NULL DEFAULT '', - path_hash TEXT NOT NULL, - content_hash TEXT, - status_code INTEGER NOT NULL, - content_type TEXT, - response_headers TEXT, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, - expires_at DATETIME, - etag TEXT, - last_modified TEXT, - UNIQUE(source_host, source_path, source_query) + last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE output_content ( content_hash TEXT PRIMARY KEY, @@ -171,14 +159,14 @@ func TestCache_StoreAndLookup(t *testing.T) { Headers: map[string][]string{testHeaderContentType: {testContentTypeJPEG}}, } - contentHash, err := cache.StoreSource( + sourceKey, err := cache.StoreSource( ctx, req, bytes.NewReader(sourceContent), fetchResult) if err != nil { t.Fatalf("StoreSource() error = %v", err) } - if contentHash == "" { - t.Error("StoreSource() returned empty hash") + if sourceKey == "" { + t.Error("StoreSource() returned empty key") } // Store variant @@ -547,7 +535,6 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) { dirs := []string{ "cache/sources", "cache/variants", - "cache/metadata", } for _, dir := range dirs { diff --git a/internal/imgcache/cache_usage_internal_test.go b/internal/imgcache/cache_usage_internal_test.go index a4d9315..8471a77 100644 --- a/internal/imgcache/cache_usage_internal_test.go +++ b/internal/imgcache/cache_usage_internal_test.go @@ -31,10 +31,10 @@ func TestEvictionReadsTheUsageTotal(t *testing.T) { storeEvictionTestVariant(t, cache, testVariantKeyTwo, bytes.Repeat([]byte{0x75}, 200)) - // pixa never changes a source's size, but a statement that does must - // change the total too. + // Storing a source URL again changes its size, so the total must + // follow a statement that changes it. _, err := cache.db.ExecContext(ctx, - `UPDATE source_content SET size_bytes = 900 WHERE content_hash = ?`, + `UPDATE source_content SET size_bytes = 900 WHERE source_key = ?`, string(kept)) if err != nil { t.Fatalf("failed to change the source's size: %v", err) @@ -155,8 +155,9 @@ func TestUsageTotalNotCorrectedFromAnOlderSum(t *testing.T) { // pass. No read the pass makes of a content table, to check its rows or // to sum them, may cover more than two rows, so a request's query waits // for one page at most. Between them the reads must still cover every -// row of both tables twice, once to check it and once to sum it, and the -// sum must put a wrong total right. +// variant row twice, once to check it and once to sum it, and every +// source row once, as source rows are read only to sum them, and the sum +// must put a wrong total right. func TestReconciliationReadsAPageAtATime(t *testing.T) { t.Parallel() @@ -207,9 +208,10 @@ func TestReconciliationReadsAPageAtATime(t *testing.T) { coveredRows += rows } - // Ten rows, each read once to check it and once to sum it. - if coveredRows != 20 { - t.Errorf("the reads covered %d rows in all, want 20", coveredRows) + // Five variant rows, each read once to check it and once to sum it, + // and five source rows, read only to sum them. + if coveredRows != 15 { + t.Errorf("the reads covered %d rows in all, want 15", coveredRows) } usage, err := cache.UsageBytes(ctx) diff --git a/internal/imgcache/contentlock.go b/internal/imgcache/contentlock.go deleted file mode 100644 index 3dac7dc..0000000 --- a/internal/imgcache/contentlock.go +++ /dev/null @@ -1,64 +0,0 @@ -package imgcache - -import "sync" - -// contentLock provides per-key mutual exclusion for content-hash keyed -// operations. StoreSource and evictSourceBlob each hold a content -// hash's lock for the full duration of their file-plus-accounting-row -// work, so a store and an eviction racing on identical content bytes -// (the real SHA-256 content-addressed dedup case, not a contrived one) -// can never interleave: the unlink of an evicted blob's file can never -// race the creation of a fresh database row for a concurrently -// re-stored copy of the same content. Entries are removed once no -// goroutine holds or is waiting for them, so a long-running process -// does not accumulate memory proportional to the number of distinct -// content hashes it has ever seen. -type contentLock struct { - mu sync.Mutex - entries map[string]*contentLockEntry -} - -// contentLockEntry is one key's exclusion lock plus a count of -// goroutines currently holding or waiting to acquire it, used to know -// when it is safe to remove the entry from the map. -type contentLockEntry struct { - mu sync.Mutex - count int -} - -// newContentLock creates an empty contentLock. -func newContentLock() *contentLock { - return &contentLock{entries: make(map[string]*contentLockEntry)} -} - -// Lock acquires exclusive access for key, blocking until it is -// available, and returns a function that releases it. The caller must -// invoke the returned function exactly once to release the lock. -func (c *contentLock) Lock(key string) func() { - c.mu.Lock() - - entry, ok := c.entries[key] - if !ok { - entry = &contentLockEntry{} - c.entries[key] = entry - } - - entry.count++ - - c.mu.Unlock() - - entry.mu.Lock() - - return func() { - entry.mu.Unlock() - - c.mu.Lock() - - entry.count-- - if entry.count == 0 { - delete(c.entries, key) - } - - c.mu.Unlock() - } -} diff --git a/internal/imgcache/contentlock_internal_test.go b/internal/imgcache/contentlock_internal_test.go deleted file mode 100644 index ff94119..0000000 --- a/internal/imgcache/contentlock_internal_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package imgcache - -import ( - "sync" - "sync/atomic" - "testing" - "time" -) - -// TestContentLockExcludesSameKey verifies that two goroutines locking -// the same key never run their critical sections concurrently. -func TestContentLockExcludesSameKey(t *testing.T) { - t.Parallel() - - lock := newContentLock() - - var ( - active atomic.Int32 - maxSeen int32 - wg sync.WaitGroup - ) - - const goroutines = 20 - - wg.Add(goroutines) - - for range goroutines { - go func() { - defer wg.Done() - - unlock := lock.Lock("same-key") - defer unlock() - - n := active.Add(1) - - for { - seen := atomic.LoadInt32(&maxSeen) - if n <= seen || atomic.CompareAndSwapInt32(&maxSeen, seen, n) { - break - } - } - - time.Sleep(time.Millisecond) - - active.Add(-1) - }() - } - - wg.Wait() - - if maxSeen != 1 { - t.Errorf("max concurrent holders of the same key = %d, want 1", maxSeen) - } -} - -// TestContentLockAllowsDifferentKeys verifies that locking distinct -// keys does not serialize unrelated work: all goroutines must be able -// to enter their critical sections at once, proven by every one of -// them reaching the rendezvous point before any is allowed to -// proceed. -func TestContentLockAllowsDifferentKeys(t *testing.T) { - t.Parallel() - - lock := newContentLock() - - const goroutines = 20 - - var ( - wg sync.WaitGroup - inside atomic.Int32 - reached = make(chan struct{}, goroutines) - ) - - wg.Add(goroutines) - - release := make(chan struct{}) - - for i := range goroutines { - key := string(rune('a' + i)) - - go func() { - defer wg.Done() - - unlock := lock.Lock(key) - defer unlock() - - inside.Add(1) - - reached <- struct{}{} - - <-release - }() - } - - // Every goroutine must reach the rendezvous point (i.e. acquire its - // own key's lock) without needing any other to release first. If - // keys were incorrectly serialized onto one underlying lock, only - // one would get here and this would time out. - for i := range goroutines { - select { - case <-reached: - case <-time.After(2 * time.Second): - t.Fatalf("only %d/%d goroutines locking distinct keys made progress; "+ - "keys may be incorrectly serialized", i, goroutines) - } - } - - if n := inside.Load(); n != goroutines { - t.Errorf("goroutines inside their critical section = %d, want %d", - n, goroutines) - } - - close(release) - wg.Wait() -} - -// TestContentLockRemovesEntryAfterUnlock verifies that the internal -// entries map does not grow without bound: once no goroutine holds or -// awaits a key, its entry is removed. -func TestContentLockRemovesEntryAfterUnlock(t *testing.T) { - t.Parallel() - - lock := newContentLock() - - unlock := lock.Lock("k") - - lock.mu.Lock() - if _, ok := lock.entries["k"]; !ok { - lock.mu.Unlock() - t.Fatal("entry missing while lock is held") - } - lock.mu.Unlock() - - unlock() - - lock.mu.Lock() - defer lock.mu.Unlock() - - if _, ok := lock.entries["k"]; ok { - t.Error("entry for key still present after the last holder unlocked") - } -} diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 4c7a9f7..6470cbc 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -84,11 +84,11 @@ func (c *Cache) UsageBytes(ctx context.Context) (int64, error) { // evictionCandidate is one LRU eviction victim candidate: either a // processed variant (isVariant true, identified by cacheKey) or a -// source content blob (identified by contentHash). +// source content blob (identified by sourceKey). type evictionCandidate struct { isVariant bool cacheKey imagerequest.VariantKey - contentHash ContentHash + sourceKey imagerequest.SourceKey sizeBytes int64 lastAccessedAt string } @@ -160,7 +160,7 @@ func (c *Cache) evictBatch(ctx context.Context, excessBytes int64) (int64, error c.log.Warn("failed to evict cache entry", "cache_key", candidate.cacheKey, - "content_hash", candidate.contentHash, + "source_key", candidate.sourceKey, "error", err, ) @@ -179,7 +179,7 @@ func (c *Cache) evictCandidate(ctx context.Context, candidate evictionCandidate) return c.evictVariant(ctx, candidate.cacheKey) } - return c.evictSourceBlob(ctx, candidate.contentHash) + return c.evictSourceBlob(ctx, candidate.sourceKey) } // evictionCandidates returns up to evictionBatchSize variants and @@ -259,9 +259,9 @@ func (c *Cache) variantCandidates(ctx context.Context) ([]evictionCandidate, err // SQLite reads the rows in order from that column's index instead of // sorting the whole table. const sourceCandidatesQuery = ` - SELECT content_hash, size_bytes, last_accessed_at + SELECT source_key, size_bytes, last_accessed_at FROM source_content - ORDER BY last_accessed_at ASC, content_hash ASC + ORDER BY last_accessed_at ASC, source_key ASC LIMIT ?` // sourceCandidates returns the least recently used source blobs. @@ -278,14 +278,14 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro for rows.Next() { var candidate evictionCandidate - var hash string + var key string - err := rows.Scan(&hash, &candidate.sizeBytes, &candidate.lastAccessedAt) + err := rows.Scan(&key, &candidate.sizeBytes, &candidate.lastAccessedAt) if err != nil { return nil, fmt.Errorf("failed to scan source candidate: %w", err) } - candidate.contentHash = ContentHash(hash) + candidate.sourceKey = imagerequest.SourceKey(key) candidates = append(candidates, candidate) } @@ -321,117 +321,18 @@ func (c *Cache) evictVariant( return nil } -// sourceReference identifies one source_metadata row's JSON sidecar. -type sourceReference struct { - host string - pathHash PathHash -} - -// evictSourceBlob removes one source content blob. All source_metadata -// rows referencing the blob are deleted together with its -// source_content row in a single transaction BEFORE the file is -// unlinked: a blob referenced by multiple source paths is only ever -// removed together with all of its references, and database rows never -// point at deleted files. The JSON metadata sidecars for the removed -// rows are deleted afterwards. -// -// The whole operation holds the content hash's lock (the same one -// StoreSource holds for its full store), so a concurrent store of -// identical content bytes can never observe the file gone but a row -// still present, or insert a fresh row between this transaction's -// commit and the file unlink below: it either runs entirely before -// this eviction starts, or is blocked until this eviction (row -// deletion and unlink together) has fully completed. -func (c *Cache) evictSourceBlob(ctx context.Context, contentHash ContentHash) error { - unlock := c.contentLocks.Lock(string(contentHash)) - defer unlock() - - references, err := c.sourceReferences(ctx, contentHash) - if err != nil { - return err - } - - tx, err := c.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin eviction transaction: %w", err) - } - - defer func() { _ = tx.Rollback() }() - - _, err = tx.ExecContext(ctx, - `DELETE FROM source_metadata WHERE content_hash = ?`, string(contentHash)) - if err != nil { - return fmt.Errorf("failed to delete source metadata rows: %w", err) - } - - _, err = tx.ExecContext(ctx, - `DELETE FROM source_content WHERE content_hash = ?`, string(contentHash)) +// evictSourceBlob removes one source: its row first, then its file, as +// evictVariant does for a variant. It holds no lock: a store of the same +// source in between can leave the source's row with no file, which +// LookupSource treats as a miss. +func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error { + _, err := c.db.ExecContext(ctx, + `DELETE FROM source_content WHERE source_key = ?`, string(key)) if err != nil { return fmt.Errorf("failed to delete source content row: %w", err) } - err = tx.Commit() - if err != nil { - return fmt.Errorf("failed to commit eviction transaction: %w", err) - } - - if c.evictSourceBlobTestHook != nil { - c.evictSourceBlobTestHook(contentHash) - } - - // Only after the rows are gone may the files be removed. - for _, reference := range references { - err := c.srcMetadata.Delete(reference.host, reference.pathHash) - if err != nil { - c.log.Warn("failed to delete metadata sidecar", - "host", reference.host, "path_hash", reference.pathHash, "error", err) - } - } - - err = c.srcContent.Delete(contentHash) - if err != nil { - return err - } - - return nil -} - -// sourceReferences lists the metadata sidecar locations of every -// source_metadata row referencing the given blob. -func (c *Cache) sourceReferences( - ctx context.Context, contentHash ContentHash, -) ([]sourceReference, error) { - rows, err := c.db.QueryContext(ctx, ` - SELECT source_host, path_hash FROM source_metadata WHERE content_hash = ? - `, string(contentHash)) - if err != nil { - return nil, fmt.Errorf("failed to query source references: %w", err) - } - - defer func() { _ = rows.Close() }() - - var references []sourceReference - - for rows.Next() { - var reference sourceReference - - var pathHash string - - err := rows.Scan(&reference.host, &pathHash) - if err != nil { - return nil, fmt.Errorf("failed to scan source reference: %w", err) - } - - reference.pathHash = PathHash(pathHash) - references = append(references, reference) - } - - err = rows.Err() - if err != nil { - return nil, fmt.Errorf("source reference iteration failed: %w", err) - } - - return references, nil + return c.srcContent.Delete(key) } // notifyWritePressure wakes the background evictor after a store, so @@ -586,12 +487,12 @@ func (c *Cache) runCleanExpired(ctx context.Context) { // as long after the last pass ended as that pass took, off the request // hot path: it adopts variant files that predate the accounting table // (or whose accounting insert failed, e.g. StoreVariant's best-effort -// insert under transient DB contention), drops accounting rows whose -// files are missing, removes source blob files the database does not -// know (and rows whose files are gone), removes source and variant files -// that are empty or not the size their rows record, sweeps stale temp -// files left behind by crashed writes, and last checks the total cache -// usage against the tables. It reads the tables a page at a time. +// insert under transient DB contention), drops variant accounting rows +// whose files are missing, removes source blob files the database does +// not know, removes source and variant files that are empty or not the +// size their rows record, sweeps stale temp files left behind by crashed +// writes, and last checks the total cache usage against the tables. It +// reads the tables a page at a time. // Running it periodically, not just once, bounds how long such drift can // accumulate unaccounted for on a long-running process to about one // eviction interval plus ten times the last pass's time. @@ -617,11 +518,6 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error { return err } - err = c.reconcileSourceRows(ctx) - if err != nil { - return err - } - return c.reconcileUsageTotal(ctx) } @@ -842,19 +738,19 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error { return nil } - return c.reconcileSourceFile(ctx, path, entry, ContentHash(name)) + return c.reconcileSourceFile(ctx, path, entry, imagerequest.SourceKey(name)) }, ) } // reconcileSourceFile checks one source blob file against its -// source_content row. A file without a row is removed: it is -// unreachable, as source lookups always go through source_metadata. A -// file that is empty, or not the size its row records, as a crash or a -// restored snapshot can leave it, is removed with its rows and a warning -// naming it, so the source is fetched again. +// source_content row. A file without a row is removed, as no lookup +// serves it and no eviction removes it. A file that is empty, or not the +// size its row records, as a crash or a restored snapshot can leave it, +// is removed with its row and a warning naming it, so the source is +// fetched again. func (c *Cache) reconcileSourceFile( - ctx context.Context, path string, entry fs.DirEntry, contentHash ContentHash, + ctx context.Context, path string, entry fs.DirEntry, key imagerequest.SourceKey, ) error { info, err := entry.Info() if err != nil { @@ -864,13 +760,13 @@ func (c *Cache) reconcileSourceFile( var recordedSize int64 err = c.db.QueryRowContext(ctx, - `SELECT size_bytes FROM source_content WHERE content_hash = ?`, - string(contentHash), + `SELECT size_bytes FROM source_content WHERE source_key = ?`, + string(key), ).Scan(&recordedSize) switch { case errors.Is(err, sql.ErrNoRows): - return c.removeUntrackedSourceFile(ctx, path, contentHash) + return c.removeUntrackedSourceFile(path, key) case err != nil: return fmt.Errorf("failed to check source content row: %w", err) case info.Size() > 0 && info.Size() == recordedSize: @@ -881,96 +777,34 @@ func (c *Cache) reconcileSourceFile( c.log.Warn("removing damaged cached source file", "path", path, "file_bytes", info.Size(), "recorded_bytes", recordedSize) - return c.evictSourceBlob(ctx, contentHash) + return c.evictSourceBlob(ctx, key) } // removeUntrackedSourceFile deletes a source blob file that has no -// source_content row. Any source_metadata rows referencing the hash -// are removed first so no row ever points at a deleted file. +// source_content row. func (c *Cache) removeUntrackedSourceFile( - ctx context.Context, path string, contentHash ContentHash, + path string, key imagerequest.SourceKey, ) error { - _, err := c.db.ExecContext(ctx, - `DELETE FROM source_metadata WHERE content_hash = ?`, string(contentHash)) - if err != nil { - return fmt.Errorf("failed to delete metadata rows for untracked blob: %w", err) - } - - err = os.Remove(path) + err := os.Remove(path) if err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to remove untracked source file: %w", err) } - c.log.Info("removed untracked source content file", "content_hash", contentHash) + c.log.Info("removed untracked source content file", "source_key", key) return nil } -// reconcileSourceRows removes source_content rows (and their metadata -// references and sidecars) whose blob files are missing on disk. -func (c *Cache) reconcileSourceRows(ctx context.Context) error { - var after ContentHash - - for { - hashes, err := c.sourceContentHashesAfter(ctx, after) - if err != nil { - return err - } - - if c.reconciliationReadTestHook != nil { - c.reconciliationReadTestHook(len(hashes)) - } - - if len(hashes) == 0 { - return nil - } - - for _, hash := range hashes { - if ctx.Err() != nil { - return ctx.Err() - } - - if c.srcContent.Exists(hash) { - continue - } - - // The blob file is already gone; evictSourceBlob removes the - // rows and sidecars and tolerates the missing file. - err := c.evictSourceBlob(ctx, hash) - if err != nil { - return err - } - - c.log.Info("dropped rows for missing source content file", "content_hash", hash) - } - - after = hashes[len(hashes)-1] - } -} - -// sourceContentHashesAfter returns, in order, up to -// c.reconciliationPageSize tracked source content hashes that sort after -// the given one. -func (c *Cache) sourceContentHashesAfter( - ctx context.Context, after ContentHash, -) ([]ContentHash, error) { - return queryStringColumn[ContentHash](ctx, c.db, ` - SELECT content_hash FROM source_content - WHERE content_hash > ? ORDER BY content_hash LIMIT ? - `, "source content hashes", "content hash", string(after), - c.reconciliationPageSize) -} - // Each of these queries sums size_bytes over the next page of rows of // one content table, the rows that sort after a key, and returns the // page's last key, the sum and the number of rows in the page. Past the // last row the page has no rows and the key is NULL. const ( sourceSizePageQuery = ` - SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0), COUNT(*) + SELECT MAX(source_key), COALESCE(SUM(size_bytes), 0), COUNT(*) FROM ( - SELECT content_hash, size_bytes FROM source_content - WHERE content_hash > ? ORDER BY content_hash LIMIT ? + SELECT source_key, size_bytes FROM source_content + WHERE source_key > ? ORDER BY source_key LIMIT ? )` variantSizePageQuery = ` SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0), COUNT(*) diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index ce5087d..1655ee7 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -10,10 +10,11 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" - _ "modernc.org/sqlite" + "modernc.org/sqlite" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" @@ -83,10 +84,10 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) { } // storeEvictionTestSource stores content as a fetched source for -// host/path and returns the resulting content hash. +// host/path and returns the resulting source key. func storeEvictionTestSource( t *testing.T, cache *Cache, host, path string, content []byte, -) ContentHash { +) imagerequest.SourceKey { t.Helper() req := &imagerequest.Request{ @@ -110,14 +111,14 @@ func storeEvictionTestSource( }, } - hash, err := cache.StoreSource( + key, err := cache.StoreSource( context.Background(), req, bytes.NewReader(content), result, ) if err != nil { t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err) } - return hash + return key } // storeEvictionTestVariant stores content as a processed variant under @@ -162,13 +163,13 @@ func setVariantLastAccessed( // setSourceLastAccessed backdates the last access time of a tracked // source content blob. func setSourceLastAccessed( - t *testing.T, cache *Cache, hash ContentHash, when time.Time, + t *testing.T, cache *Cache, key imagerequest.SourceKey, when time.Time, ) { t.Helper() res, err := cache.db.ExecContext(t.Context(), - `UPDATE source_content SET last_accessed_at = ? WHERE content_hash = ?`, - when.UTC().Format(sqliteTimestampFormat), string(hash), + `UPDATE source_content SET last_accessed_at = ? WHERE source_key = ?`, + when.UTC().Format(sqliteTimestampFormat), string(key), ) if err != nil { t.Fatalf("failed to set source last_accessed_at: %v", err) @@ -180,7 +181,7 @@ func setSourceLastAccessed( } if affected != 1 { - t.Fatalf("source %s has no accounting row (affected=%d)", hash, affected) + t.Fatalf("source %s has no accounting row (affected=%d)", key, affected) } } @@ -205,32 +206,31 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) { t.Helper() rows, err := cache.db.QueryContext(t.Context(), - `SELECT content_hash FROM source_metadata - WHERE content_hash IS NOT NULL AND content_hash != ''`, + `SELECT source_key FROM source_content`, ) if err != nil { - t.Fatalf("failed to query source_metadata: %v", err) + t.Fatalf("failed to query source_content: %v", err) } defer func() { _ = rows.Close() }() for rows.Next() { - var hash string + var key string - err := rows.Scan(&hash) + err := rows.Scan(&key) if err != nil { - t.Fatalf("failed to scan content_hash: %v", err) + t.Fatalf("failed to scan source_key: %v", err) } - if !cache.srcContent.Exists(ContentHash(hash)) { - t.Errorf("source_metadata references content %s but the file is missing", - hash) + if !cache.srcContent.Exists(imagerequest.SourceKey(key)) { + t.Errorf("source_content references content %s but the file is missing", + key) } } err = rows.Err() if err != nil { - t.Fatalf("source_metadata iteration failed: %v", err) + t.Fatalf("source_content iteration failed: %v", err) } variantRows, err := cache.db.QueryContext(t.Context(), @@ -313,33 +313,6 @@ func TestUsageBytesAccountsSourceAndVariantBytes(t *testing.T) { } } -func TestUsageBytesCountsMultiReferencedBlobOnce(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - content := bytes.Repeat([]byte{0xCC}, 1200) - - hashOne := storeEvictionTestSource(t, cache, - "src.example.com", "/one.jpg", content) - hashTwo := storeEvictionTestSource(t, cache, - "src.example.com", "/two.jpg", content) - - if hashOne != hashTwo { - t.Fatalf("identical content produced different hashes: %s vs %s", - hashOne, hashTwo) - } - - usage, err := cache.UsageBytes(context.Background()) - if err != nil { - t.Fatalf("UsageBytes failed: %v", err) - } - - if usage != 1200 { - t.Errorf("UsageBytes = %d, want 1200 (deduplicated blob counted once)", usage) - } -} - func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) { t.Parallel() @@ -393,102 +366,15 @@ func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) { assertNoDanglingReferences(t, cache) } -func TestEvictionRemovesMultiReferencedBlobTogetherWithAllReferences(t *testing.T) { - t.Parallel() - - const limit = 1000 - - cache, _ := newEvictionTestCache(t, limit) - - now := time.Now() - - // One 800-byte blob referenced by two source paths. - sharedContent := bytes.Repeat([]byte{0xDD}, 800) - sharedHash := storeEvictionTestSource(t, cache, - "src.example.com", "/a.jpg", sharedContent) - - h := storeEvictionTestSource(t, cache, - "src.example.com", "/b.jpg", sharedContent) - if h != sharedHash { - t.Fatalf("identical content produced different hashes: %s vs %s", - h, sharedHash) - } - - // A newer 600-byte blob referenced by one source path. - recentHash := storeEvictionTestSource(t, cache, "src.example.com", "/c.jpg", - bytes.Repeat([]byte{0xEE}, 600)) - - setSourceLastAccessed(t, cache, sharedHash, now.Add(-2*time.Hour)) - setSourceLastAccessed(t, cache, recentHash, now.Add(-time.Minute)) - - err := cache.EvictToLimit(context.Background()) - if err != nil { - t.Fatalf("EvictToLimit failed: %v", err) - } - - usage, err := cache.UsageBytes(context.Background()) - if err != nil { - t.Fatalf("UsageBytes failed: %v", err) - } - - if usage > limit { - t.Errorf("usage after eviction = %d, want <= %d", usage, limit) - } - - // The multi-referenced blob must be gone from disk, from - // source_content, and from BOTH source_metadata rows: references - // are removed together with the blob, never left dangling. - if cache.srcContent.Exists(sharedHash) { - t.Errorf("evicted blob %s still exists on disk", sharedHash) - } - - if n := countRows(t, cache, - `SELECT COUNT(*) FROM source_content WHERE content_hash = ?`, string(sharedHash), - ); n != 0 { - t.Errorf("evicted blob %s still has %d source_content rows", sharedHash, n) - } - - if n := countRows(t, cache, - `SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(sharedHash), - ); n != 0 { - t.Errorf("evicted blob %s still has %d source_metadata references", sharedHash, n) - } - - // The JSON metadata sidecars for both referencing paths must be - // removed along with the rows. - for _, path := range []string{"/a.jpg", "/b.jpg"} { - pathHash := HashPath(path + "?") - if cache.srcMetadata.Exists("src.example.com", pathHash) { - t.Errorf("metadata sidecar for %s must be removed with its row", path) - } - } - - // The more recently used blob survives fully intact. - if !cache.srcContent.Exists(recentHash) { - t.Errorf("recently used blob %s must survive eviction", recentHash) - } - - if n := countRows(t, cache, - `SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(recentHash), - ); n != 1 { - t.Errorf("recently used blob %s has %d source_metadata rows, want 1", - recentHash, n) - } - - assertNoDanglingReferences(t, cache) -} - func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) content := bytes.Repeat([]byte{0xDF}, 800) - hash := storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content) - - h := storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content) - if h != hash { - t.Fatalf("identical content produced different hashes: %s vs %s", h, hash) + keys := []imagerequest.SourceKey{ + storeEvictionTestSource(t, cache, "src.example.com", "/a.jpg", content), + storeEvictionTestSource(t, cache, "src.example.com", "/b.jpg", content), } storeEvictionTestVariant(t, cache, testVariantKeyOne, @@ -499,14 +385,16 @@ func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) { t.Fatalf("EvictToLimit failed: %v", err) } - if !cache.srcContent.Exists(hash) { - t.Errorf("blob %s must not be evicted while usage is under the limit", hash) - } + for _, key := range keys { + if !cache.srcContent.Exists(key) { + t.Errorf("source %s must not be evicted while usage is under the limit", key) + } - if n := countRows(t, cache, - `SELECT COUNT(*) FROM source_metadata WHERE content_hash = ?`, string(hash), - ); n != 2 { - t.Errorf("blob %s has %d source_metadata rows, want 2", hash, n) + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_content WHERE source_key = ?`, string(key), + ); n != 1 { + t.Errorf("source %s has %d source_content rows, want 1", key, n) + } } if !cache.variants.Exists(testVariantKeyOne) { @@ -562,13 +450,13 @@ func assertDisabledCacheWritesAreNoOps( Headers: map[string][]string{}, } - hash, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result) + key, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result) if err != nil { t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err) } - if hash != "" { - t.Errorf("StoreSource on disabled cache returned hash %q, want empty", hash) + if key != "" { + t.Errorf("StoreSource on disabled cache returned key %q, want empty", key) } } @@ -590,14 +478,14 @@ func assertDisabledCacheReadsAlwaysMiss( t.Error("Lookup on disabled cache must always miss") } - srcHash, srcType, err := cache.LookupSource(ctx, req) + key, srcType, err := cache.LookupSource(ctx, req) if err != nil { t.Fatalf("LookupSource on disabled cache failed: %v", err) } - if srcHash != "" || srcType != "" { + if key != "" || srcType != "" { t.Errorf("LookupSource on disabled cache = (%q, %q), want empty", - srcHash, srcType) + key, srcType) } } @@ -618,10 +506,6 @@ func assertDisabledCacheTracksNothing(t *testing.T, cache *Cache) { if n := countRows(t, cache, `SELECT COUNT(*) FROM source_content`); n != 0 { t.Errorf("disabled cache wrote %d source_content rows, want 0", n) } - - if n := countRows(t, cache, `SELECT COUNT(*) FROM source_metadata`); n != 0 { - t.Errorf("disabled cache wrote %d source_metadata rows, want 0", n) - } } // assertDisabledCacheWroteNothingToDisk verifies that a disabled cache @@ -1064,9 +948,15 @@ func TestStopEvictionInterruptsPassInProgress(t *testing.T) { } // TestEvictToLimitStopsAtNextCandidateOnceCancelled cancels the context -// while the oldest of three source blobs is being evicted, and checks that -// EvictToLimit then returns context.Canceled without evicting the other -// two or logging a warning for either of them. +// once the oldest of three sources is evicted, while the next one waits +// for the database, and checks that EvictToLimit then returns +// context.Canceled, with that source gone and the other two kept, and +// logs no warning. The test database has one connection. When the +// deletion of the oldest source's row commits, a commit hook holds the +// deletion until the test waits for the connection, so the test takes it +// as soon as the deletion is done. The eviction of the next source then +// waits for the connection, and the test cancels the context while it +// waits. func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) { t.Parallel() @@ -1076,7 +966,7 @@ func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) { cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil)) - hashes := []ContentHash{ + keys := []imagerequest.SourceKey{ storeEvictionTestSource(t, cache, "cancel.example.com", "/a.jpg", bytes.Repeat([]byte{0x61}, 1000)), storeEvictionTestSource(t, cache, "cancel.example.com", "/b.jpg", @@ -1087,69 +977,155 @@ func TestEvictToLimitStopsAtNextCandidateOnceCancelled(t *testing.T) { base := time.Now().Add(-time.Hour) - for i, hash := range hashes { - setSourceLastAccessed(t, cache, hash, base.Add(time.Duration(i)*time.Minute)) + for i, key := range keys { + setSourceLastAccessed(t, cache, key, base.Add(time.Duration(i)*time.Minute)) } + waitsBefore := cache.db.Stats().WaitCount + oldestRowDeleted := make(chan struct{}) + + var oldestRowDeletedOnce sync.Once + + setCommitHook(t, cache, func() int32 { + oldestRowDeletedOnce.Do(func() { + close(oldestRowDeleted) + + deadline := time.Now().Add(5 * time.Second) + + for cache.db.Stats().WaitCount == waitsBefore && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + }) + + return 0 + }) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() - cache.evictSourceBlobTestHook = func(ContentHash) { cancel() } + evicted := make(chan error, 1) - err := cache.EvictToLimit(ctx) + go func() { evicted <- cache.EvictToLimit(ctx) }() + + select { + case <-oldestRowDeleted: + case err := <-evicted: + t.Fatalf("EvictToLimit() returned %v without deleting a row", err) + case <-time.After(5 * time.Second): + t.Fatal("EvictToLimit never deleted a row") + } + + releaseDatabase := holdDatabase(t, cache) + defer releaseDatabase() + + waitForConnectionWaits(t, cache.db, waitsBefore+2) + cancel() + + err := <-evicted t.Logf("EvictToLimit() error = %v", err) t.Logf("log output: %s", logBuf.String()) + // The checks below read the database, so the connection goes back. + releaseDatabase() + + setCommitHook(t, cache, nil) + if !errors.Is(err, context.Canceled) { t.Errorf("EvictToLimit() error = %v, want context.Canceled", err) } - if cache.srcContent.Exists(hashes[0]) { - t.Errorf("source blob %s, evicted when the context was cancelled, "+ - "is still on disk", hashes[0]) - } - - for _, hash := range hashes[1:] { - if !cache.srcContent.Exists(hash) { - t.Errorf("source blob %s was evicted after the context was cancelled", hash) - } - } + assertOnlyOldestSourceEvicted(t, cache, keys) if strings.Contains(logBuf.String(), `"level":"WARN"`) { t.Errorf("EvictToLimit logged a warning after the context was cancelled") } +} + +// setCommitHook makes the SQLite driver call hook each time a write on +// the test database's one connection commits, or stops it when hook is +// nil. +func setCommitHook(t *testing.T, cache *Cache, hook sqlite.CommitHookFn) { + t.Helper() + + conn, err := cache.db.Conn(t.Context()) + if err != nil { + t.Fatalf("failed to take the database connection: %v", err) + } + + defer func() { _ = conn.Close() }() + + err = conn.Raw(func(driverConn any) error { + hooks, ok := driverConn.(sqlite.HookRegisterer) + if !ok { + t.Fatalf("the database connection is a %T, which takes no hooks", + driverConn) + } + + hooks.RegisterCommitHook(hook) + + return nil + }) + if err != nil { + t.Fatalf("failed to set the commit hook: %v", err) + } +} + +// assertOnlyOldestSourceEvicted checks that the first of keys has neither +// its file nor its row, and that every other one has both. +func assertOnlyOldestSourceEvicted( + t *testing.T, cache *Cache, keys []imagerequest.SourceKey, +) { + t.Helper() + + if cache.srcContent.Exists(keys[0]) { + t.Errorf("source %s, evicted when the context was cancelled, "+ + "is still on disk", keys[0]) + } + + for _, key := range keys[1:] { + if !cache.srcContent.Exists(key) { + t.Errorf("source %s was evicted after the context was cancelled", key) + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_content WHERE source_key = ?`, string(key), + ); n != 1 { + t.Errorf("source %s, kept, has %d source_content rows, want 1", key, n) + } + } assertNoDanglingReferences(t, cache) } -// TestStopEvictionReturnsWhenItsContextEnds pauses an eviction pass where -// cancellation cannot reach it, after a source blob's rows are deleted and -// before its file is removed, and checks that StopEviction returns its -// context's error when that context ends instead of waiting for the pass. -// Once the pass goes on, the goroutine exits and no row points at a -// missing file. +// TestStopEvictionReturnsWhenItsContextEnds pauses the startup +// reconciliation pass in the hook called after each of its reads, where +// cancellation cannot reach it, and checks that StopEviction returns an +// error wrapping its context's error when that context ends, instead of +// waiting for the pass. Once the pass goes on, the goroutine exits, and a +// second StopEviction returns nil. func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) { t.Parallel() - cache, _ := newEvictionTestCache(t, 1) + cache, _ := newEvictionTestCache(t, 1<<30) paused := make(chan struct{}) resume := make(chan struct{}) - cache.evictSourceBlobTestHook = func(ContentHash) { - close(paused) - <-resume - } + var pauseOnce sync.Once - hash := storeEvictionTestSource(t, cache, "stop.example.com", "/a.jpg", - bytes.Repeat([]byte{0x61}, 1000)) + cache.reconciliationReadTestHook = func(int) { + pauseOnce.Do(func() { + close(paused) + <-resume + }) + } cache.StartEviction(time.Hour) select { case <-paused: case <-time.After(5 * time.Second): - t.Fatal("the eviction pass never reached the source blob") + t.Fatal("the reconciliation pass never made a read") } ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) @@ -1168,12 +1144,6 @@ func TestStopEvictionReturnsWhenItsContextEnds(t *testing.T) { if err != nil { t.Fatalf("second StopEviction() error = %v, want nil", err) } - - assertNoDanglingReferences(t, cache) - - if cache.srcContent.Exists(hash) { - t.Errorf("source blob %s is still on disk after its rows were deleted", hash) - } } // TestReconciliationWalksStopOnceCancelled checks that both directory @@ -1245,131 +1215,3 @@ func TestReconciliationPassLogsNoWarningOnceCancelled(t *testing.T) { t.Errorf("runReconciliationPass logged a warning with a cancelled context") } } - -// TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent exercises -// the exact TOCTOU window between evictSourceBlob's row-deletion -// transaction commit and its content file unlink: a concurrent -// StoreSource for a different source path whose content hashes to the -// same value (real SHA-256 dedup, not a contrived case) must not be -// able to insert a fresh row referencing the file while eviction is -// mid-unlink, and must not lose its own store once eviction has fully -// released the content hash. -func TestEvictSourceBlobExcludesConcurrentStoreOfIdenticalContent(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - ctx := context.Background() - - content := bytes.Repeat([]byte{0x55}, 400) - - hash := storeEvictionTestSource(t, cache, - "race.example.com", "/first.jpg", content) - - proceed := make(chan struct{}) - storeAttempted := make(chan struct{}) - - cache.evictSourceBlobTestHook = func(gotHash ContentHash) { - if gotHash != hash { - t.Errorf("test hook invoked for hash %s, want %s", gotHash, hash) - } - - close(storeAttempted) - <-proceed - } - - evictDone := make(chan error, 1) - - go func() { - evictDone <- cache.evictSourceBlob(ctx, hash) - }() - - // Wait until eviction has committed its delete transaction and is - // paused (inside the test hook) immediately before unlinking the - // content file: exactly the window the review flagged. - <-storeAttempted - - storeDone := storeIdenticalContentConcurrently(ctx, cache, content) - - // The concurrent store must not be able to complete while eviction - // still holds the content hash (i.e. before the file is unlinked): - // if it could, it would insert a row referencing a file about to be - // removed out from under it. - select { - case err := <-storeDone: - t.Fatalf("StoreSource for identical content completed (err=%v) while eviction "+ - "still held the content hash open between commit and unlink; the store and "+ - "the evict of identical content are not mutually exclusive", err) - case <-time.After(200 * time.Millisecond): - // Expected: the store is blocked behind eviction's exclusion. - } - - close(proceed) - - err := <-evictDone - if err != nil { - t.Fatalf("evictSourceBlob failed: %v", err) - } - - err = <-storeDone - if err != nil { - t.Fatalf("StoreSource failed: %v", err) - } - - assertNoDanglingReferences(t, cache) - - dupHash, _, err := cache.LookupSource(ctx, &imagerequest.Request{ - Source: imagerequest.Source{ - Host: "race.example.com", - Path: "/dup.jpg", - }, - }) - if err != nil { - t.Fatalf("LookupSource failed: %v", err) - } - - if dupHash == "" { - t.Fatal("re-stored blob was lost: the store legitimately ran after eviction " + - "released the content hash and must have recreated the file and row") - } - - if !cache.srcContent.Exists(dupHash) { - t.Errorf("source_content/source_metadata references %s but its file is missing", - dupHash) - } -} - -// storeIdenticalContentConcurrently starts a StoreSource for a second -// source path whose body hashes to the same content hash, returning the -// channel its error is delivered on. -func storeIdenticalContentConcurrently( - ctx context.Context, cache *Cache, content []byte, -) chan error { - storeDone := make(chan error, 1) - - go func() { - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: "race.example.com", - Path: "/dup.jpg", - }, - Transform: imagerequest.Transform{ - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - result := &httpfetcher.FetchResult{ - StatusCode: 200, - ContentType: testContentTypeJPEG, - ContentLength: int64(len(content)), - Headers: map[string][]string{ - testHeaderContentType: {testContentTypeJPEG}, - }, - } - - _, err := cache.StoreSource(ctx, req, bytes.NewReader(content), result) - storeDone <- err - }() - - return storeDone -} diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imgcache/max_concurrent_processing_internal_test.go index 69aeb31..dec7941 100644 --- a/internal/imgcache/max_concurrent_processing_internal_test.go +++ b/internal/imgcache/max_concurrent_processing_internal_test.go @@ -77,11 +77,11 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { _ = resp.Content.Close() - contentHash, _, err := svc.cache.LookupSource(t.Context(), + sourceKey, _, err := svc.cache.LookupSource(t.Context(), widthOnlyRequest(fixtures, 50)) - if err != nil || contentHash == "" { + if err != nil || sourceKey == "" { t.Fatalf("LookupSource() = %q, %v; want the cached source", - contentHash, err) + sourceKey, err) } release := holdProcessingSlot(t, svc.processor) @@ -103,7 +103,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { // before waiting, it would have read it by now. time.Sleep(100 * time.Millisecond) - err = os.WriteFile(svc.cache.srcContent.hashToPath(contentHash), + err = os.WriteFile(svc.cache.srcContent.keyToPath(sourceKey), generateTestJPEG(t, 100, 50, color.RGBA{0, 0, 255, 255}), 0o600) if err != nil { t.Fatalf("failed to rewrite the cached source: %v", err) diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index 1c3b2bf..c4c4269 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -147,13 +147,13 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { _ = resp.Content.Close() - contentHash, _, err := svc.cache.LookupSource(t.Context(), + sourceKey, _, err := svc.cache.LookupSource(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) - if err != nil || contentHash == "" { - t.Fatalf("LookupSource() = %q, %v, want the cached source", contentHash, err) + if err != nil || sourceKey == "" { + t.Fatalf("LookupSource() = %q, %v, want the cached source", sourceKey, err) } - err = svc.cache.evictSourceBlob(t.Context(), contentHash) + err = svc.cache.evictSourceBlob(t.Context(), sourceKey) if err != nil { t.Fatalf("evictSourceBlob() error = %v", err) } diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index 364d5cc..6f41ba6 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -2,8 +2,6 @@ package imgcache import ( "bytes" - "crypto/sha256" - "encoding/hex" "io" "log/slog" "os" @@ -67,12 +65,12 @@ func cachedSourcePath( ) string { t.Helper() - contentHash, _, err := svc.cache.LookupSource(t.Context(), req) - if err != nil || contentHash == "" { - t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err) + sourceKey, _, err := svc.cache.LookupSource(t.Context(), req) + if err != nil || sourceKey == "" { + t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err) } - return svc.cache.srcContent.hashToPath(contentHash) + return svc.cache.srcContent.keyToPath(sourceKey) } // reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does @@ -316,10 +314,10 @@ func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) { } } -// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a -// store over a file of another size at the content's path, such as a +// TestContentStorage_StoreReplacesFileOfAnotherSize checks that a +// store over a file of another size at the source's path, such as a // partial file a crash left, replaces it with the content. -func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) { +func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) { t.Parallel() storage, err := NewContentStorage(t.TempDir()) @@ -328,13 +326,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) { } content := []byte("the whole source image") - sum := sha256.Sum256(content) - hash := ContentHash(hex.EncodeToString(sum[:])) - path := storage.hashToPath(hash) + key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() + path := storage.keyToPath(key) err = os.MkdirAll(filepath.Dir(path), StorageDirPerm) if err != nil { - t.Fatalf("creating the content's directory: %v", err) + t.Fatalf("creating the source's directory: %v", err) } err = os.WriteFile(path, content[:5], StorageFilePerm) @@ -342,12 +339,12 @@ func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) { t.Fatalf("writing a partial file: %v", err) } - _, err = storage.StoreHashed(hash, content) + _, err = storage.Store(key, content) if err != nil { - t.Fatalf("StoreHashed() error = %v", err) + t.Fatalf("Store() error = %v", err) } - stored, err := storage.Load(hash) + stored, err := storage.Load(key) if err != nil { t.Fatalf("Load() error = %v", err) } diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 25d19fa..d0e345d 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -359,9 +359,9 @@ func withoutCancelKeepingDeadline( // returns it with its size; nil if the cached data is unavailable, empty or // exceeds maxResponseSize. func (s *Service) loadCachedSource( - contentHash ContentHash, + sourceKey imagerequest.SourceKey, ) (io.ReadCloser, int64) { - reader, size, err := s.cache.GetSourceContent(contentHash) + reader, size, err := s.cache.GetSourceContent(sourceKey) if err != nil { s.log.Warn("failed to load cached source, fetching", "error", err) @@ -372,7 +372,7 @@ func (s *Service) loadCachedSource( _ = reader.Close() s.log.Warn("cached source exceeds max response size, discarding", - "hash", contentHash, + "source_key", sourceKey, "max_bytes", s.maxResponseSize, ) @@ -396,7 +396,7 @@ func (s *Service) processFromSourceOrFetch( cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { // Check if we have cached source content - contentHash, _, err := s.cache.LookupSource(ctx, req) + sourceKey, _, err := s.cache.LookupSource(ctx, req) if err != nil { s.log.Warn("source lookup failed", "error", err) } @@ -406,9 +406,9 @@ func (s *Service) processFromSourceOrFetch( sourceSize int64 ) - if contentHash != "" { - s.log.Debug("using cached source", "hash", contentHash) - source, sourceSize = s.loadCachedSource(contentHash) + if sourceKey != "" { + s.log.Debug("using cached source", "source_key", sourceKey) + source, sourceSize = s.loadCachedSource(sourceKey) } // Fetch from upstream if we don't have source data or it's empty diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index 712955c..92915ca 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -550,21 +550,17 @@ func TestService_Get_Caching(t *testing.T) { // TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and // a variant of it and sets back two days every time pixa stored for them. -// For the source: fetched_at in source_metadata, fetched_at and -// last_accessed_at in source_content, the fetched_at in its metadata file, -// and the access and modification times of its file and of its metadata -// file. For the variant: created_at and last_accessed_at in variant_content, -// the created_at in its .meta file, and the access and modification times of -// its file and of its .meta file. It then checks that the variant is still -// served from the cache and that a new variant is made from the cached source -// without fetching it again. +// For the source: fetched_at and last_accessed_at in source_content, and the +// access and modification times of its file. For the variant: created_at and +// last_accessed_at in variant_content, the created_at in its .meta file, and +// the access and modification times of its file and of its .meta file. It +// then checks that the variant is still served from the cache and that a new +// variant is made from the cached source without fetching it again. // // Left at the present: the files' status change and birth times, which a // program cannot set, and the times of the directories holding the files, -// which other entries share. The source's stored response headers are empty, -// as the mock fetcher sends none. expires_at, etag and last_modified in -// source_metadata and in the metadata file are never written, nor are the -// request_cache and output_content tables. +// which other entries share. The request_cache and output_content tables are +// never written. func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { t.Parallel() @@ -615,39 +611,19 @@ func backdateSource( ) { t.Helper() - pathHash := HashPath(req.Source.Path + "?" + req.Source.Query) - - meta, err := cache.srcMetadata.Load(req.Source.Host, pathHash) - if err != nil { - t.Fatalf("reading the source's metadata file: %v", err) - } - - meta.FetchedAt = when.Unix() - - err = cache.srcMetadata.Store(req.Source.Host, pathHash, meta) - if err != nil { - t.Fatalf("writing the source's metadata file: %v", err) - } - + key := req.Source.Key() dbTime := when.UTC().Format(sqliteTimestampFormat) - updateOneRow(t, cache, ` - UPDATE source_metadata SET fetched_at = ? - WHERE source_host = ? AND source_path = ? AND source_query = ? - `, dbTime, req.Source.Host, req.Source.Path, req.Source.Query) updateOneRow(t, cache, - `UPDATE source_content SET fetched_at = ? WHERE content_hash = ?`, - dbTime, meta.ContentHash) - setSourceLastAccessed(t, cache, ContentHash(meta.ContentHash), when) + `UPDATE source_content SET fetched_at = ? WHERE source_key = ?`, + dbTime, string(key)) + setSourceLastAccessed(t, cache, key, when) - for _, path := range []string{ - cache.srcContent.hashToPath(ContentHash(meta.ContentHash)), - cache.srcMetadata.metaPath(req.Source.Host, pathHash), - } { - err = os.Chtimes(path, when, when) - if err != nil { - t.Fatalf("setting back the times of %s: %v", path, err) - } + path := cache.srcContent.keyToPath(key) + + err := os.Chtimes(path, when, when) + if err != nil { + t.Fatalf("setting back the times of %s: %v", path, err) } } diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go new file mode 100644 index 0000000..8d7be09 --- /dev/null +++ b/internal/imgcache/source_content_internal_test.go @@ -0,0 +1,146 @@ +package imgcache + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" + + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// TestStoreSource_StoresIdenticalBytesOfTwoURLsTwice stores the same bytes +// for two source URLs and checks that each URL has a source_content row of +// its own and a file of its own, named by its source key under +// cache/sources///. +func TestStoreSource_StoresIdenticalBytesOfTwoURLsTwice(t *testing.T) { + t.Parallel() + + cache, stateDir := newEvictionTestCache(t, 1<<30) + content := bytes.Repeat([]byte{0x5A}, 400) + + keys := map[string]imagerequest.SourceKey{ + "/a.jpg": storeEvictionTestSource(t, cache, testHostCDN, "/a.jpg", content), + "/b.jpg": storeEvictionTestSource(t, cache, testHostCDN, "/b.jpg", content), + } + t.Logf("source keys: %v", keys) + + if keys["/a.jpg"] == keys["/b.jpg"] { + t.Fatalf("both source URLs got the key %s", keys["/a.jpg"]) + } + + for sourcePath, key := range keys { + rows := countRows(t, cache, ` + SELECT COUNT(*) FROM source_content + WHERE source_key = ? AND source_host = ? AND source_path = ? + `, string(key), testHostCDN, sourcePath) + if rows != 1 { + t.Errorf("%s has %d source_content rows, want 1", sourcePath, rows) + } + + name := string(key) + file := filepath.Join(stateDir, "cache", "sources", name[0:2], name[2:4], name) + + stored, err := os.ReadFile(file) //nolint:gosec // path in the test's state dir + if err != nil || !bytes.Equal(stored, content) { + t.Errorf("file of %s at %s holds %d bytes (error %v), want the %d stored", + sourcePath, file, len(stored), err, len(content)) + } + } + + usage, err := cache.UsageBytes(t.Context()) + if err != nil { + t.Fatalf("UsageBytes() error = %v", err) + } + + if usage != 800 { + t.Errorf("UsageBytes() = %d, want 800, the 400 bytes once for each URL", usage) + } +} + +// TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed stores the same +// bytes for two source URLs and evicts the first. Its row and file must be +// gone, and the second must still be found and read from the cache. +func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + content := bytes.Repeat([]byte{0x5B}, 400) + + evicted := storeEvictionTestSource(t, cache, testHostCDN, "/evicted.jpg", content) + storeEvictionTestSource(t, cache, testHostCDN, "/kept.jpg", content) + + err := cache.evictSourceBlob(t.Context(), evicted) + if err != nil { + t.Fatalf("evictSourceBlob() error = %v", err) + } + + if cache.srcContent.Exists(evicted) { + t.Error("the evicted source's file is still on disk") + } + + if n := countRows(t, cache, + `SELECT COUNT(*) FROM source_content WHERE source_key = ?`, string(evicted), + ); n != 0 { + t.Errorf("the evicted source still has %d source_content rows", n) + } + + kept, contentType, err := cache.LookupSource(t.Context(), &imagerequest.Request{ + Source: imagerequest.Source{Host: testHostCDN, Path: "/kept.jpg"}, + }) + t.Logf("LookupSource() of the kept source = %q, %q, error %v", kept, contentType, err) + + if err != nil || kept == "" { + t.Fatalf("LookupSource() of the kept source = %q, %v; want it cached", kept, err) + } + + reader, _, err := cache.GetSourceContent(kept) + if err != nil { + t.Fatalf("GetSourceContent() of the kept source error = %v", err) + } + + defer func() { _ = reader.Close() }() + + served, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("reading the kept source: %v", err) + } + + if !bytes.Equal(served, content) { + t.Errorf("the kept source reads as %d bytes, want the %d stored", + len(served), len(content)) + } +} + +// TestStoreSource_FailedRowWriteLeavesNoFile drops source_content, so that +// StoreSource cannot write the source's row, and checks that the store fails +// and leaves no file under the source's key. +func TestStoreSource_FailedRowWriteLeavesNoFile(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + _, err := cache.db.ExecContext(t.Context(), `DROP TABLE source_content`) + if err != nil { + t.Fatalf("failed to drop source_content: %v", err) + } + + req := &imagerequest.Request{ + Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, + } + result := &httpfetcher.FetchResult{ContentType: testContentTypeJPEG} + + key, err := cache.StoreSource(t.Context(), req, + bytes.NewReader([]byte("source bytes")), result) + t.Logf("StoreSource() = %q, error = %v", key, err) + + if err == nil { + t.Error("StoreSource() error = nil, want the failed row write") + } + + if cache.srcContent.Exists(req.Source.Key()) { + t.Error("the source's file is on disk although its row was not written") + } +} diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index a243fab..9af86fd 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -1,8 +1,6 @@ package imgcache import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -30,14 +28,8 @@ var ( ErrNotFound = errors.New("content not found") ) -// ContentHash is a SHA256 hash of file content (hex-encoded). -type ContentHash string - -// PathHash is a SHA256 hash of a URL path (hex-encoded). -type PathHash string - -// ContentStorage handles content-addressable file storage. -// Files are stored at: /// +// ContentStorage handles source image file storage, one file per source +// URL. Files are stored at: /// type ContentStorage struct { baseDir string } @@ -52,38 +44,19 @@ func NewContentStorage(baseDir string) (*ContentStorage, error) { return &ContentStorage{baseDir: baseDir}, nil } -// Store writes content to storage and returns its SHA256 hash. -// The content is read fully into memory to compute the hash before writing. -func (s *ContentStorage) Store(r io.Reader) (ContentHash, int64, error) { - // Read all content to compute hash - data, err := io.ReadAll(r) - if err != nil { - return "", 0, fmt.Errorf("failed to read content: %w", err) - } - - // Compute hash - h := sha256.Sum256(data) - hash := ContentHash(hex.EncodeToString(h[:])) - size := int64(len(data)) - - err = s.writeIfAbsent(hash, data) - if err != nil { - return "", 0, err - } - - return hash, size, nil -} - -// StoreHashed writes pre-hashed content to storage at the path derived -// from hash, without recomputing it. Callers that already know the -// hash before writing (e.g. because they must hold a hash-keyed lock -// across the whole store operation) use this instead of Store. Like -// Store, it is idempotent: a file of the content's size already at that -// path is left untouched, and a file of another size is replaced. -func (s *ContentStorage) StoreHashed( - hash ContentHash, data []byte, +// Store writes data with writeFileSynced to the path derived from key, +// replacing any file already there, and returns its size. +func (s *ContentStorage) Store( + key imagerequest.SourceKey, data []byte, ) (int64, error) { - err := s.writeIfAbsent(hash, data) + path := s.keyToPath(key) + + err := os.MkdirAll(filepath.Dir(path), StorageDirPerm) + if err != nil { + return 0, fmt.Errorf("failed to create directory: %w", err) + } + + err = writeFileSynced(path, data) if err != nil { return 0, err } @@ -91,11 +64,11 @@ func (s *ContentStorage) StoreHashed( return int64(len(data)), nil } -// Load returns a reader for the content with the given hash. -func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) { - path := s.hashToPath(hash) +// Load returns a reader for the content with the given key. +func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) { + path := s.keyToPath(key) - f, err := os.Open(path) //nolint:gosec // path derived from content hash + f, err := os.Open(path) //nolint:gosec // path derived from source key if err != nil { if os.IsNotExist(err) { return nil, ErrNotFound @@ -107,11 +80,13 @@ func (s *ContentStorage) Load(hash ContentHash) (io.ReadCloser, error) { return f, nil } -// LoadWithSize returns a reader and file size for the content with the given hash. -func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, error) { - path := s.hashToPath(hash) +// LoadWithSize returns a reader and file size for the content with the given key. +func (s *ContentStorage) LoadWithSize( + key imagerequest.SourceKey, +) (io.ReadCloser, int64, error) { + path := s.keyToPath(key) - f, err := os.Open(path) //nolint:gosec // path derived from content hash + f, err := os.Open(path) //nolint:gosec // path derived from source key if err != nil { if os.IsNotExist(err) { return nil, 0, ErrNotFound @@ -130,9 +105,9 @@ func (s *ContentStorage) LoadWithSize(hash ContentHash) (io.ReadCloser, int64, e return f, stat.Size(), nil } -// Delete removes content with the given hash. -func (s *ContentStorage) Delete(hash ContentHash) error { - path := s.hashToPath(hash) +// Delete removes content with the given key. +func (s *ContentStorage) Delete(key imagerequest.SourceKey) error { + path := s.keyToPath(key) err := os.Remove(path) if err != nil && !os.IsNotExist(err) { @@ -142,34 +117,14 @@ func (s *ContentStorage) Delete(hash ContentHash) error { return nil } -// Exists checks if content with the given hash exists. -func (s *ContentStorage) Exists(hash ContentHash) bool { - path := s.hashToPath(hash) +// Exists checks if content with the given key exists. +func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool { + path := s.keyToPath(key) _, err := os.Stat(path) return err == nil } -// writeIfAbsent writes data with writeFileSynced to the path derived -// from hash, unless a file of the data's size is already there. -func (s *ContentStorage) writeIfAbsent(hash ContentHash, data []byte) error { - // Build path: /// - path := s.hashToPath(hash) - - // A file of another size there is damaged, and is replaced - info, statErr := os.Stat(path) - if statErr == nil && info.Size() == int64(len(data)) { - return nil - } - - err := os.MkdirAll(filepath.Dir(path), StorageDirPerm) - if err != nil { - return fmt.Errorf("failed to create directory: %w", err) - } - - return writeFileSynced(path, data) -} - // writeFileSynced writes data to path through a temp file in the same // directory: the temp file is synced to disk, renamed to path, and the // directory synced after, so that neither a concurrent reader nor a @@ -244,167 +199,19 @@ func syncDirectory(dir string) error { return nil } -// hashToPath converts a hash to a file path: /// -func (s *ContentStorage) hashToPath(hash ContentHash) string { - h := string(hash) - if len(h) < MinHashLength { - return filepath.Clean(filepath.Join(s.baseDir, h)) +// keyToPath converts a key to a file path: /// +func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string { + k := string(key) + if len(k) < MinHashLength { + return filepath.Clean(filepath.Join(s.baseDir, k)) } - return filepath.Clean(filepath.Join(s.baseDir, h[0:2], h[2:4], h)) -} - -// MetadataStorage handles JSON metadata file storage. -// Files are stored at: //.json -type MetadataStorage struct { - baseDir string -} - -// NewMetadataStorage creates a new metadata storage at the given base directory. -func NewMetadataStorage(baseDir string) (*MetadataStorage, error) { - err := os.MkdirAll(baseDir, StorageDirPerm) - if err != nil { - return nil, fmt.Errorf("failed to create metadata directory: %w", err) - } - - return &MetadataStorage{baseDir: baseDir}, nil -} - -// SourceMetadata represents cached metadata about a source URL. -// -//nolint:tagliatelle // stored metadata format uses snake_case -type SourceMetadata struct { - Host string `json:"host"` - Path string `json:"path"` - Query string `json:"query,omitempty"` - ContentHash string `json:"content_hash,omitempty"` - StatusCode int `json:"status_code"` - ContentType string `json:"content_type,omitempty"` - ContentLength int64 `json:"content_length,omitempty"` - ResponseHeaders map[string][]string `json:"response_headers,omitempty"` - FetchedAt int64 `json:"fetched_at"` - FetchDurationMs int64 `json:"fetch_duration_ms,omitempty"` - ExpiresAt int64 `json:"expires_at,omitempty"` - ETag string `json:"etag,omitempty"` - LastModified string `json:"last_modified,omitempty"` - RemoteAddr string `json:"remote_addr,omitempty"` -} - -// Store writes metadata to storage. -func (s *MetadataStorage) Store( - host string, pathHash PathHash, meta *SourceMetadata, -) error { - path := s.metaPath(host, pathHash) - - // Create directory structure - dir := filepath.Dir(path) - - err := os.MkdirAll(dir, StorageDirPerm) - if err != nil { - return fmt.Errorf("failed to create directory: %w", err) - } - - // Marshal to JSON - data, err := json.MarshalIndent(meta, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal metadata: %w", err) - } - - // Write to temp file first, then rename for atomicity - tmpFile, err := os.CreateTemp(dir, ".tmp-*.json") - if err != nil { - return fmt.Errorf("failed to create temp file: %w", err) - } - - tmpPath := tmpFile.Name() - - _, err = tmpFile.Write(data) - if err != nil { - _ = tmpFile.Close() - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to write metadata: %w", err) - } - - err = tmpFile.Close() - if err != nil { - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to close temp file: %w", err) - } - - // Atomic rename - err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path)) - if err != nil { - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to rename temp file: %w", err) - } - - return nil -} - -// Load reads metadata from storage. -func (s *MetadataStorage) Load( - host string, pathHash PathHash, -) (*SourceMetadata, error) { - path := s.metaPath(host, pathHash) - - data, err := os.ReadFile(path) //nolint:gosec // path derived from host+hash - if err != nil { - if os.IsNotExist(err) { - return nil, ErrNotFound - } - - return nil, fmt.Errorf("failed to read metadata: %w", err) - } - - var meta SourceMetadata - - err = json.Unmarshal(data, &meta) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal metadata: %w", err) - } - - return &meta, nil -} - -// Delete removes metadata for the given host and path hash. -func (s *MetadataStorage) Delete(host string, pathHash PathHash) error { - path := s.metaPath(host, pathHash) - - err := os.Remove(path) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete metadata: %w", err) - } - - return nil -} - -// Exists checks if metadata exists for the given host and path hash. -func (s *MetadataStorage) Exists(host string, pathHash PathHash) bool { - path := s.metaPath(host, pathHash) - _, err := os.Stat(path) - - return err == nil -} - -// metaPath returns the file path for metadata: //.json -func (s *MetadataStorage) metaPath(host string, pathHash PathHash) string { - return filepath.Clean(filepath.Join(s.baseDir, host, string(pathHash)+".json")) -} - -// HashPath computes the SHA256 hash of a path string. -func HashPath(path string) PathHash { - h := sha256.Sum256([]byte(path)) - - return PathHash(hex.EncodeToString(h[:])) + return filepath.Clean(filepath.Join(s.baseDir, k[0:2], k[2:4], k)) } // VariantStorage handles key-based file storage for processed image variants. // Files are stored at: /// // Metadata is stored at: ///.meta -// Unlike ContentStorage, the key is provided by the caller (not computed from content). type VariantStorage struct { baseDir string log *slog.Logger diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go index 0819b9f..98314bf 100644 --- a/internal/imgcache/storage_internal_test.go +++ b/internal/imgcache/storage_internal_test.go @@ -24,8 +24,9 @@ func TestContentStorage_StoreAndLoad(t *testing.T) { } content := []byte("hello world") + key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() - hash, size, err := storage.Store(bytes.NewReader(content)) + size, err := storage.Store(key, content) if err != nil { t.Fatalf("Store() error = %v", err) } @@ -34,14 +35,10 @@ func TestContentStorage_StoreAndLoad(t *testing.T) { t.Errorf("Store() size = %d, want %d", size, len(content)) } - if hash == "" { - t.Error("Store() returned empty hash") - } - // Verify file exists at expected path - hashStr := string(hash) + keyStr := string(key) - expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr) + expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr) _, err = os.Stat(expectedPath) if err != nil { @@ -49,7 +46,7 @@ func TestContentStorage_StoreAndLoad(t *testing.T) { } // Load and verify content - r, err := storage.Load(hash) + r, err := storage.Load(key) if err != nil { t.Fatalf("Load() error = %v", err) } @@ -66,33 +63,6 @@ func TestContentStorage_StoreAndLoad(t *testing.T) { } } -func TestContentStorage_StoreIdempotent(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewContentStorage(tmpDir) - if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) - } - - content := []byte("same content") - - hash1, _, err := storage.Store(bytes.NewReader(content)) - if err != nil { - t.Fatalf("Store() first error = %v", err) - } - - hash2, _, err := storage.Store(bytes.NewReader(content)) - if err != nil { - t.Fatalf("Store() second error = %v", err) - } - - if hash1 != hash2 { - t.Errorf("Store() hashes differ: %s vs %s", hash1, hash2) - } -} - func TestContentStorage_LoadNotFound(t *testing.T) { t.Parallel() @@ -103,7 +73,7 @@ func TestContentStorage_LoadNotFound(t *testing.T) { t.Fatalf("NewContentStorage() error = %v", err) } - _, err = storage.Load(ContentHash("nonexistent")) + _, err = storage.Load(imagerequest.SourceKey("nonexistent")) if !errors.Is(err, ErrNotFound) { t.Errorf("Load() error = %v, want ErrNotFound", err) } @@ -120,22 +90,23 @@ func TestContentStorage_Delete(t *testing.T) { } content := []byte("to be deleted") + key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() - hash, _, err := storage.Store(bytes.NewReader(content)) + _, err = storage.Store(key, content) if err != nil { t.Fatalf("Store() error = %v", err) } - if !storage.Exists(hash) { + if !storage.Exists(key) { t.Error("Exists() = false, want true") } - err = storage.Delete(hash) + err = storage.Delete(key) if err != nil { t.Fatalf("Delete() error = %v", err) } - if storage.Exists(hash) { + if storage.Exists(key) { t.Error("Exists() = true after delete, want false") } } @@ -151,13 +122,13 @@ func TestContentStorage_DeleteNonexistent(t *testing.T) { } // Should not error - err = storage.Delete(ContentHash("nonexistent")) + err = storage.Delete(imagerequest.SourceKey("nonexistent")) if err != nil { t.Errorf("Delete() error = %v, want nil", err) } } -func TestContentStorage_HashToPath(t *testing.T) { +func TestContentStorage_KeyToPath(t *testing.T) { t.Parallel() tmpDir := t.TempDir() @@ -169,15 +140,16 @@ func TestContentStorage_HashToPath(t *testing.T) { // Test by storing and verifying the resulting path structure content := []byte("test content for path verification") + key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() - hash, _, err := storage.Store(bytes.NewReader(content)) + _, err = storage.Store(key, content) if err != nil { t.Fatalf("Store() error = %v", err) } - hashStr := string(hash) + keyStr := string(key) - expectedPath := filepath.Join(tmpDir, hashStr[0:2], hashStr[2:4], hashStr) + expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr) _, err = os.Stat(expectedPath) if err != nil { @@ -185,145 +157,6 @@ func TestContentStorage_HashToPath(t *testing.T) { } } -func TestMetadataStorage_StoreAndLoad(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewMetadataStorage(tmpDir) - if err != nil { - t.Fatalf("NewMetadataStorage() error = %v", err) - } - - meta := &SourceMetadata{ - Host: testHostCDN, - Path: testPathCat, - ContentHash: "abc123", - StatusCode: 200, - ContentType: testContentTypeJPEG, - FetchedAt: 1704067200, - ETag: `"etag123"`, - } - - pathHash := HashPath(testPathCat) - - err = storage.Store(testHostCDN, pathHash, meta) - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - // Verify file exists at expected path - expectedPath := filepath.Join(tmpDir, testHostCDN, string(pathHash)+".json") - - _, err = os.Stat(expectedPath) - if err != nil { - t.Errorf("File not at expected path %s: %v", expectedPath, err) - } - - // Load and verify - loaded, err := storage.Load(testHostCDN, pathHash) - if err != nil { - t.Fatalf("Load() error = %v", err) - } - - if loaded.Host != meta.Host { - t.Errorf("Host = %q, want %q", loaded.Host, meta.Host) - } - - if loaded.Path != meta.Path { - t.Errorf("Path = %q, want %q", loaded.Path, meta.Path) - } - - if loaded.ContentHash != meta.ContentHash { - t.Errorf("ContentHash = %q, want %q", loaded.ContentHash, meta.ContentHash) - } - - if loaded.StatusCode != meta.StatusCode { - t.Errorf("StatusCode = %d, want %d", loaded.StatusCode, meta.StatusCode) - } - - if loaded.ETag != meta.ETag { - t.Errorf("ETag = %q, want %q", loaded.ETag, meta.ETag) - } -} - -func TestMetadataStorage_LoadNotFound(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewMetadataStorage(tmpDir) - if err != nil { - t.Fatalf("NewMetadataStorage() error = %v", err) - } - - _, err = storage.Load(testHostExample, PathHash("nonexistent")) - if !errors.Is(err, ErrNotFound) { - t.Errorf("Load() error = %v, want ErrNotFound", err) - } -} - -func TestMetadataStorage_Delete(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewMetadataStorage(tmpDir) - if err != nil { - t.Fatalf("NewMetadataStorage() error = %v", err) - } - - meta := &SourceMetadata{ - Host: testHostExample, - Path: "/test.jpg", - StatusCode: 200, - } - - pathHash := HashPath("/test.jpg") - - err = storage.Store(testHostExample, pathHash, meta) - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - if !storage.Exists(testHostExample, pathHash) { - t.Error("Exists() = false, want true") - } - - err = storage.Delete(testHostExample, pathHash) - if err != nil { - t.Fatalf("Delete() error = %v", err) - } - - if storage.Exists(testHostExample, pathHash) { - t.Error("Exists() = true after delete, want false") - } -} - -func TestHashPath(t *testing.T) { - t.Parallel() - - // Same input should produce same hash - hash1 := HashPath(testPathCat) - hash2 := HashPath(testPathCat) - - if hash1 != hash2 { - t.Errorf("HashPath() not deterministic: %s vs %s", hash1, hash2) - } - - // Different input should produce different hash - hash3 := HashPath("/photos/dog.jpg") - - if hash1 == hash3 { - t.Error("HashPath() produced same hash for different inputs") - } - - // Hash should be 64 hex chars (256 bits) - if len(string(hash1)) != 64 { - t.Errorf("HashPath() length = %d, want 64", len(string(hash1))) - } -} - // TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write // that fails is logged at warn and the store still succeeds. func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go index ba56e4f..cf0f4b5 100644 --- a/internal/server/image_proxy_internal_test.go +++ b/internal/server/image_proxy_internal_test.go @@ -3,9 +3,7 @@ package server import ( "bytes" "context" - "crypto/sha256" "database/sql" - "encoding/hex" "image" "image/color" "image/jpeg" @@ -30,6 +28,7 @@ import ( "sneak.berlin/go/pixa/internal/handlers" "sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/middleware" ) @@ -254,39 +253,26 @@ func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder return rec } -// checkSourceCached checks that source is stored under its SHA-256 in -// cache/sources, recorded in source_content, and that the source URL's row in -// source_metadata points at it. +// checkSourceCached checks that source is stored in cache/sources under the +// source key of upstreamHost and /photo.png, and recorded under that key in +// source_content. func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) { t.Helper() - sum := sha256.Sum256(source) - hash := hex.EncodeToString(sum[:]) + key := string(imagerequest.Source{Host: upstreamHost, Path: "/photo.png"}.Key()) - checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash), + checkFile(t, filepath.Join(stateDir, "cache", "sources", key[0:2], key[2:4], key), source) var rows int err := db.QueryRowContext(t.Context(), - "SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash, + "SELECT COUNT(*) FROM source_content WHERE source_key = ?", key, ).Scan(&rows) if err != nil || rows != 1 { t.Errorf("source_content rows for the source = %d (error %v), want 1", rows, err) } - - var metadataHash string - - err = db.QueryRowContext(t.Context(), - `SELECT content_hash FROM source_metadata - WHERE source_host = ? AND source_path = ?`, - upstreamHost, "/photo.png", - ).Scan(&metadataHash) - if err != nil || metadataHash != hash { - t.Errorf("source_metadata content_hash = %q (error %v), want %q", - metadataHash, err, hash) - } } // checkVariantCached checks that the converted image served is recorded in From 5f9d624d74a5b1c962929309ba7e56372608ad22 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 21 Oct 2026 18:12:55 +0200 Subject: [PATCH 099/131] Serve a cache hit with no database read or write (closes #309) A hit opens the variant file and nothing else: its content type comes from the file's first bytes, and a file in no output format is a miss. Its access time and count, and the access time of a cached source a miss uses, go without waiting to one background goroutine that writes them once a second; each kind of access time waits in a channel of 10,000 that drops the rest. Expired negative cache rows are left to CleanExpired. The .meta files, metaCache, Lookup and golang-lru go, with their tests; the two reconciliation walks, identical without the .meta skip, share one. make bench runs a parallel cache hit benchmark on the host, outside make check. Model: opus-5-5 --- Makefile | 8 +- README.md | 70 ++-- TODO.md | 17 + go.mod | 1 - go.sum | 2 - internal/db/migrations/001_schema.sql | 10 +- internal/handlers/cache_hit_internal_test.go | 265 +++++++++++++ internal/handlers/handlers.go | 7 +- internal/imgcache/cache.go | 285 ++++---------- internal/imgcache/cache_hit_internal_test.go | 299 +++++++++++++++ internal/imgcache/cache_internal_test.go | 350 +---------------- .../concurrent_misses_internal_test.go | 2 + internal/imgcache/eviction.go | 105 ++--- internal/imgcache/eviction_internal_test.go | 15 +- internal/imgcache/metacache_internal_test.go | 361 ------------------ .../partial_cache_files_internal_test.go | 60 +-- internal/imgcache/pendingcounts.go | 150 ++++++-- .../imgcache/pendingcounts_internal_test.go | 168 ++++++-- internal/imgcache/service.go | 51 +-- internal/imgcache/service_internal_test.go | 42 +- internal/imgcache/stats_internal_test.go | 45 +-- internal/imgcache/storage.go | 121 +++--- internal/imgcache/storage_internal_test.go | 108 ------ script/bench | 19 + 24 files changed, 1115 insertions(+), 1446 deletions(-) create mode 100644 internal/handlers/cache_hit_internal_test.go create mode 100644 internal/imgcache/cache_hit_internal_test.go delete mode 100644 internal/imgcache/metacache_internal_test.go create mode 100755 script/bench diff --git a/Makefile b/Makefile index 188f643..22ac9c9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest +.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest bench VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") LDFLAGS := -X main.Version=$(VERSION) @@ -64,6 +64,12 @@ docker-smoke: loadtest: @script/loadtest +# Serve cache hits in parallel through the image handler and report hits per +# second (needs Go on the host after `script/bootstrap --cgo`, not Docker; a +# benchmark, not part of check) +bench: + @script/bench + # Build Docker image as `make docker` does, and also tag it pixa:$(VERSION) docker-versioned: @script/docker diff --git a/README.md b/README.md index cc09929..8c190de 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,9 @@ database and the disk cache: pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user must be able to write the directory. - `cache_max_bytes` limits the source and transformed images together. The - database, the `.meta` file beside each transformed image and files still being - written come on top, and eviction runs in the background, so the cache can - pass the limit for a while: leave room on the volume beyond it. + database and files still being written come on top, and eviction runs in the + background, so the cache can pass the limit for a while: leave room on the + volume beyond it. - Set `cache_max_bytes` for a lasting deployment. Its default, worked out each time pixa starts, is 75% of the sum of the space free on the volume and the space the cached images already take, so a restart keeps the limit the cache @@ -90,11 +90,12 @@ which answers 200 whenever pixa is running, in maintenance mode too (see On SIGTERM or SIGINT pixa stops accepting connections, gives the requests in progress and the images being processed 5 seconds to finish, then writes to the -database the counts of cache hits, misses, fetches and conversions that requests -could not write by their deadline, and exits: with 0, or with 1 when images were -still being processed after those 5 seconds, some of those counts could not be -written, or another part of pixa failed to stop. A request not finished by then -is cut off. `docker stop` waits 10 seconds before it kills the container. +database the counts of cache hits, misses, fetches and conversions and the times +cached images were last used that are not written yet (see Storage), and exits: +with 0, or with 1 when images were still being processed after those 5 seconds, +some of those counts or times could not be written, or another part of pixa +failed to stop. A request not finished by then is cut off. `docker stop` waits +10 seconds before it kills the container. Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL @@ -181,8 +182,7 @@ allowlisted source hosts for open access. file for each source URL - **Database**: `/state.sqlite3` (SQLite) - **Transformed images**: - `/cache/variants///`, - each with a `.meta` file beside it holding its content type + `/cache/variants///` `` and `` are the first and second pairs of characters of the file's name. @@ -193,22 +193,30 @@ bytes are stored twice, once under each URL. A source image is stored by writing its file and then its row; when the row cannot be written, the file is removed again. -Source and transformed image files, and the `.meta` file beside each transformed -image, are written under a temporary name, synced to disk, and only then renamed -into place, so a crash leaves no file under its own name with part of its data. -When pixa starts, and every five minutes after, it removes each cached source or -transformed image whose file is empty, or not the size the database records for -it, with a warning naming the file; the next request for it fetches the source -or converts the image again. Looking for those files reads the database once for -every cached file, so pixa skips each five-minute look that would start less -than ten times as long after the last look ended as that look took. After a look -that took one minute, the next one starts 10 to 15 minutes after that look -ended. +Source and transformed image files are written under a temporary name, synced to +disk, and only then renamed into place, so a crash leaves no file under its own +name with part of its data. When pixa starts, and every five minutes after, it +removes each cached source or transformed image whose file is empty, or not the +size the database records for it, with a warning naming the file; the next +request for it fetches the source or converts the image again. Looking for those +files reads the database once for every cached file, so pixa skips each +five-minute look that would start less than ten times as long after the last +look ended as that look took. After a look that took one minute, the next one +starts 10 to 15 minutes after that look ended. pixa's target is 1-5k r/s, which has not been measured at that rate (see Load -Test). Toward it, pixa keeps in memory the content types of the 10,000 -transformed images most recently cached or served, so a cache hit on one of them -reads only the image file from disk and not the metadata file stored beside it. +Test). Toward it, a cache hit opens the transformed image's file and nothing +else, and makes no database call. The content type comes from the file's first +bytes, which are those of the format pixa wrote it in; a file whose first bytes +are not those of a format pixa writes is not served, and the image is converted +again. The time the image was last served, which eviction orders by, and the hit +count are handed to a background writer without waiting for it, as are the +counts of misses, fetches and conversions, and the time a miss last used a +cached source image. Once a second that writer writes the waiting times in one +transaction and the counts in one statement. While 10,000 times of transformed +images, or 10,000 of source images, wait to be written, a further one is +dropped, and that image only looks less recently used to eviction. +`script/bench` measures how many hits pixa serves per second (see Load Test). ### Routes @@ -846,6 +854,9 @@ provide: - `script/docker-smoke` — build the image, start it, wait for it to be healthy - `script/loadtest` — measure pixad's throughput, latency and peak memory; a benchmark, not part of `script/check` (see Load Test) +- `script/bench` — run the Go benchmark that serves cache hits in parallel + through the image handler, with Go on the host after `script/bootstrap --cgo`; + a benchmark, not part of `script/check` (see Load Test) - `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`), then `script/check`, then build the image as `script/docker` does - `script/precommit` — pre-commit checks (`go mod tidy` guard, then @@ -926,6 +937,17 @@ The numbers depend on the machine and on whatever else runs on it. The first measurement, made on a shared machine with few clients, is in `TODO.md`; it says nothing about the target. +`script/bench` (or `make bench`) measures cache hits alone, without the network +and with no pixad process: a Go benchmark in `internal/handlers` serves one +cached image through the `/v1/image/` handler from as many goroutines at once as +the machine has CPUs, and reports the hits served per second as `hits/s`. Each +run starts with an empty cache and an empty database, a SQLite file of its own, +and puts the image in the cache before it measures. The benchmark fails if +serving the hits ran any statement on the database. Like the load test it is not +part of `script/check`. It runs Go on the host, as the load test does, and +compiles pixa there, so it needs the C compiler and libraries that +`script/bootstrap --cgo` installs, but not Docker. + ## TODO See [TODO.md](TODO.md) for the full prioritized task list. diff --git a/TODO.md b/TODO.md index 049f2c8..ad36d3e 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,23 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-21 a cache hit makes no database call (closes #309): `Service.Get` + opens the variant file with `Cache.GetVariant` and nothing else, with no + existence check first. The content type comes from the file's first bytes; a + file whose first bytes are not those of an output format, SVG included, is a + miss, and the variant is made again. A hit hands its access time to a channel + that holds up to 10,000 and drops the rest, and its hit count to the pending + counts, without waiting; every count is now a pending count. A miss that finds + a cached source hands that source's access time to a second such channel. The + goroutine `StartPendingCountWrites` starts writes, once a second, the access + times in one transaction and the counts in one `UPDATE`; + `StopPendingCountWrites` writes what is left once at shutdown and logs + separately the access times and the counts it could not write. + `checkNegativeCache` no longer deletes an expired row; `CleanExpired` in the + eviction goroutine does. The `.meta` files, `VariantMeta`, `LoadWithMeta`, + `variantContentTypeFromSidecar`, `metaCache`, `Lookup` and `golang-lru` are + gone. `make bench` (`script/bench`) runs `BenchmarkHandleImage_CacheHits` with + Go on the host after `script/bootstrap --cgo`, outside `make check`. - 2026-10-21 a source is stored the way a variant is (closes #308): one `source_content` row and one file per source URL, the file named by the source's `SourceKey`, the SHA-256 of its host, path and query made as the diff --git a/go.mod b/go.mod index ae8006a..618e2e9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/go-chi/httprate v0.16.0 github.com/gorilla/csrf v1.7.3 github.com/gorilla/securecookie v1.1.2 - github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/prometheus/client_golang v1.23.2 github.com/slok/go-http-metrics v0.13.0 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index c4542f6..f9a4f3d 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,6 @@ github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index 71d9ec9..c046293 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -18,10 +18,12 @@ CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed ON source_content(last_accessed_at); -- Processed variant blobs --- Files stored at: cache/variants/// (plus a .meta --- sidecar with the content type). Tracked here (like source content --- blobs above) so total cache usage is known without a directory scan, --- and so LRU eviction has a timestamp to order on. +-- Files stored at: cache/variants///. Tracked here +-- (like source content blobs above) so total cache usage is known +-- without a directory scan, and so LRU eviction has a timestamp to order +-- on. A cache hit does not read this table: it takes the content type +-- from the file's first bytes, and its last_accessed_at is written +-- later, in a batch. CREATE TABLE IF NOT EXISTS variant_content ( cache_key TEXT PRIMARY KEY, size_bytes INTEGER NOT NULL, diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go new file mode 100644 index 0000000..9592ce2 --- /dev/null +++ b/internal/handlers/cache_hit_internal_test.go @@ -0,0 +1,265 @@ +package handlers + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "image" + "image/jpeg" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "sync/atomic" + "testing" + "testing/fstest" + "time" + + "github.com/go-chi/chi/v5" + + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// countingHost is the allowlisted host whose photoPath the server of +// newCountingDatabaseServer serves. +const countingHost = "counting.example.com" + +// countingConnector opens connections to the SQLite database file name with +// sqliteDriver, each one counting in statements the statements run on it. +type countingConnector struct { + sqliteDriver driver.Driver + name string + statements *atomic.Int64 +} + +func (c countingConnector) Connect(context.Context) (driver.Conn, error) { + conn, err := c.sqliteDriver.Open(c.name) + if err != nil { + return nil, err + } + + return countingConn{Conn: conn, statements: c.statements}, nil +} + +func (c countingConnector) Driver() driver.Driver { + return c.sqliteDriver +} + +// countingConn counts the statements run on its connection. It has only the +// methods of driver.Conn, so database/sql prepares every query and every +// other statement with Prepare, which counts it. +type countingConn struct { + driver.Conn + + statements *atomic.Int64 +} + +func (c countingConn) Prepare(query string) (driver.Stmt, error) { + c.statements.Add(1) + + return c.Conn.Prepare(query) +} + +// newCountingDatabase returns a database with pixa's schema, in a file of +// its own and with one connection, and the count of the statements run on +// it once the schema is in place. +func newCountingDatabase(tb testing.TB) (*sql.DB, *atomic.Int64) { + tb.Helper() + + path := filepath.Join(tb.TempDir(), "state.sqlite3") + + schemaDB, err := sql.Open("sqlite", path) + if err != nil { + tb.Fatalf("opening the database: %v", err) + } + + defer func() { _ = schemaDB.Close() }() + + err = database.ApplyMigrations(tb.Context(), schemaDB, nil) + if err != nil { + tb.Fatalf("applying the migrations: %v", err) + } + + statements := new(atomic.Int64) + + db := sql.OpenDB(countingConnector{ + sqliteDriver: schemaDB.Driver(), + name: path, + statements: statements, + }) + db.SetMaxOpenConns(1) + + tb.Cleanup(func() { _ = db.Close() }) + + return db, statements +} + +// newCountingDatabaseServer returns a router for both image routes whose +// image cache uses newCountingDatabase, the count of the statements run on +// that database, and a URL on each route for a different image of +// photoPath on countingHost. Nothing writes the cache's pending counts and +// access times in the background, so every statement counted comes from a +// request. +func newCountingDatabaseServer( + tb testing.TB, +) (http.Handler, *atomic.Int64, []string) { + tb.Helper() + + db, statements := newCountingDatabase(tb) + + var photo bytes.Buffer + + err := jpeg.Encode(&photo, image.NewRGBA(image.Rect(0, 0, 100, 100)), nil) + if err != nil { + tb.Fatalf("encoding the photo: %v", err) + } + + cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ + StateDir: tb.TempDir(), + NegativeTTL: 5 * time.Minute, + }) + if err != nil { + tb.Fatalf("imgcache.NewCache() error = %v", err) + } + + svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + Cache: cache, + Fetcher: newMockFetcher(fstest.MapFS{ + countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()}, + }), + SigningKey: testSigningKey, + Allowlist: []string{countingHost}, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + tb.Fatalf("imgcache.NewService() error = %v", err) + } + + encGen, err := encurl.NewGenerator(testSigningKey) + if err != nil { + tb.Fatalf("encurl.NewGenerator() error = %v", err) + } + + token, err := encGen.Generate(&imagerequest.Request{ + Source: imagerequest.Source{Host: countingHost, Path: photoPath}, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 40, Height: 40}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + }) + if err != nil { + tb.Fatalf("encGen.Generate() error = %v", err) + } + + h := &Handlers{ + log: slog.New(slog.DiscardHandler), + imgSvc: svc, + encGen: encGen, + } + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + r.Get("/v1/e/{token}/*", h.HandleImageEnc()) + + return r, statements, []string{ + "/v1/image/" + countingHost + photoPath + "/50x50.jpeg", + "/v1/e/" + token + "/img.jpg", + } +} + +// serveImage sends srv a GET for target and returns the response, failing +// the test unless it is a 200 with an image. +func serveImage( + tb testing.TB, srv http.Handler, target string, +) *httptest.ResponseRecorder { + tb.Helper() + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, + httptest.NewRequestWithContext(tb.Context(), http.MethodGet, target, nil)) + + if rec.Code != http.StatusOK || rec.Body.Len() == 0 { + tb.Errorf("GET %s: %d with %d bytes, want 200 with an image", + target, rec.Code, rec.Body.Len()) + } + + return rec +} + +// TestImageRoutes_CacheHitsMakeNoDatabaseCall serves an image on each image +// route once, which makes and caches it, then serves it again from the cache. +// Serving those cache hits must run no statement on the database. +func TestImageRoutes_CacheHitsMakeNoDatabaseCall(t *testing.T) { + t.Parallel() + + srv, statements, targets := newCountingDatabaseServer(t) + + for _, target := range targets { + before := statements.Load() + + serveImage(t, srv, target) + + afterMiss := statements.Load() + t.Logf("GET %s, a miss: %d statements", target, afterMiss-before) + + if afterMiss == before { + t.Fatalf("GET %s, a miss: no statement counted, want some", target) + } + + for range 3 { + rec := serveImage(t, srv, target) + + cacheStatus := rec.Header().Get("X-Pixa-Cache") + if cacheStatus != string(imgcache.CacheHit) { + t.Errorf("GET %s: X-Pixa-Cache = %q, want %s", + target, cacheStatus, imgcache.CacheHit) + } + } + + if hitStatements := statements.Load() - afterMiss; hitStatements != 0 { + t.Errorf("GET %s: 3 cache hits ran %d statements, want 0", + target, hitStatements) + } + } +} + +// BenchmarkHandleImage_CacheHits serves one cached image through the +// /v1/image/ route from b.RunParallel's goroutines, and reports the cache +// hits served per second. It fails if serving them ran any statement on the +// database. script/bench runs it. +func BenchmarkHandleImage_CacheHits(b *testing.B) { + srv, statements, targets := newCountingDatabaseServer(b) + target := targets[0] + + serveImage(b, srv, target) + + before := statements.Load() + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + rec := serveImage(b, srv, target) + + cacheStatus := rec.Header().Get("X-Pixa-Cache") + if cacheStatus != string(imgcache.CacheHit) { + b.Errorf("GET %s: X-Pixa-Cache = %q, want %s", + target, cacheStatus, imgcache.CacheHit) + + return + } + } + }) + + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "hits/s") + + if hitStatements := statements.Load() - before; hitStatements != 0 { + b.Fatalf("%d cache hits ran %d statements, want 0", b.N, hitStatements) + } +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 9cfc3bb..3285c2b 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -76,8 +76,8 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { OnStart: func(_ context.Context) error { return s.initImageService() }, - // The pending counts are written here, before the database's stop - // hook closes it. + // The pending counts and access times are written here, before the + // database's stop hook closes it. OnStop: func(ctx context.Context) error { if s.imgCache == nil { return nil @@ -128,7 +128,8 @@ func (s *Handlers) initImageService() error { // rows, which is all it does when the disk cache is disabled. cache.StartEviction(imgcache.DefaultEvictionInterval) - // Writes the counts requests could not write by their deadline + // Writes the counts and the access times of cache hits that requests + // hand over, so that no request waits for the database to record them cache.StartPendingCountWrites() // Create the fetcher config diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index abefe36..3cb1f2d 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -12,7 +12,6 @@ import ( "sync/atomic" "time" - lru "github.com/hashicorp/golang-lru/v2" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -23,9 +22,11 @@ var ( ErrNegativeCache = errors.New("negative cache hit") ) -// metaCacheSize is how many variants' content types metaCache holds. A -// variant not among them is served as before, reading its .meta file. -const metaCacheSize = 10000 +// maxPendingAccessTimes is how many access times of variants served from +// the cache, and how many of cached sources a miss used, may each wait to be +// written, at about 80 bytes each. An access time handed over while that +// many of its kind wait is dropped. +const maxPendingAccessTimes = 10000 // cleanExpiredBatchSize is the most expired negative cache rows one // statement of CleanExpired deletes. Each batch is a statement of its @@ -82,19 +83,23 @@ type Cache struct { evictionCancel context.CancelFunc // The pending counts: hits, misses, upstream fetches and transforms - // whose write to the database missed its request's deadline, kept - // here until they are written by the goroutine that - // StartPendingCountWrites starts. As for eviction, the channels are - // created in NewCache: pendingCountsAdded wakes that goroutine, and - // pendingCountsDone is closed when it returns. pendingCountsCancel, - // set by StartPendingCountWrites, cancels its context, which tells it - // to finish. + // that requests counted, kept here until they are written by the + // goroutine that StartPendingCountWrites starts, so that no request + // waits for the database to count it. pendingVariantAccessTimes holds + // the variants served from the cache, and pendingSourceAccessTimes the + // cached sources a miss used, whose access time is still to be written + // by that goroutine. As for eviction, the channels are created in + // NewCache, so that requests can hand over access times before that + // goroutine starts; pendingCountsDone is closed when it returns. + // pendingCountsCancel, set by StartPendingCountWrites, cancels its + // context, which tells it to finish. pendingHits atomic.Int64 pendingMisses atomic.Int64 pendingUpstreamFetches atomic.Int64 pendingUpstreamFetchBytes atomic.Int64 pendingTransforms atomic.Int64 - pendingCountsAdded chan struct{} + pendingVariantAccessTimes chan imagerequest.VariantKey + pendingSourceAccessTimes chan imagerequest.SourceKey pendingCountsDone chan struct{} pendingCountsCancel context.CancelFunc @@ -102,11 +107,6 @@ type Cache struct { // readHitAndMissCounts to read them pendingCountsWriteMutex sync.Mutex - // metaCache holds the content types of the variants most recently - // stored or served, so a hit does not read the variant's .meta file. - // It never stands in for the variant file, which is always opened. - metaCache *lru.Cache[imagerequest.VariantKey, string] - // reconciliationPageSize is the most rows one read of a content table // returns in the reconciliation pass and in Stats. newCache sets it to // defaultReconciliationPageSize; tests set it smaller. @@ -134,11 +134,6 @@ func newCache( log = slog.Default() } - metaCache, err := lru.New[imagerequest.VariantKey, string](metaCacheSize) - if err != nil { - return nil, fmt.Errorf("failed to create variant content type cache: %w", err) - } - c := &Cache{ db: db, config: config, @@ -146,10 +141,10 @@ func newCache( disabled: config.DisableDiskCache, evictionPressure: make(chan struct{}, 1), evictionDone: make(chan struct{}), - metaCache: metaCache, - pendingCountsAdded: make(chan struct{}, 1), - pendingCountsDone: make(chan struct{}), + pendingVariantAccessTimes: make(chan imagerequest.VariantKey, maxPendingAccessTimes), + pendingSourceAccessTimes: make(chan imagerequest.SourceKey, maxPendingAccessTimes), + pendingCountsDone: make(chan struct{}), reconciliationPageSize: defaultReconciliationPageSize, } @@ -166,7 +161,7 @@ func newCache( } variants, err := NewVariantStorage( - filepath.Join(config.StateDir, "cache", "variants"), log, + filepath.Join(config.StateDir, "cache", "variants"), ) if err != nil { return nil, fmt.Errorf("failed to create variant storage: %w", err) @@ -192,45 +187,11 @@ func newCache( return c, nil } -// LookupResult contains the result of a cache lookup. -type LookupResult struct { - Hit bool - CacheKey imagerequest.VariantKey - ContentType string - SizeBytes int64 - CacheStatus CacheStatus -} - -// Lookup checks if a processed variant exists on disk. Hits touch the -// variant's LRU timestamp; a disabled cache always misses. -func (c *Cache) Lookup( - ctx context.Context, req *imagerequest.Request, -) (*LookupResult, error) { - cacheKey := imagerequest.CacheKey(req) - - // Check variant storage directly - no DB needed for cache hits - if !c.disabled && c.variants.Exists(cacheKey) { - c.touchVariant(ctx, cacheKey) - - return &LookupResult{ - Hit: true, - CacheKey: cacheKey, - CacheStatus: CacheHit, - }, nil - } - - return &LookupResult{ - Hit: false, - CacheKey: cacheKey, - CacheStatus: CacheMiss, - }, nil -} - -// GetVariant returns a reader, size, and content type for a cached -// variant. The content type comes from metaCache, or else from the -// variant's .meta file and is then kept in metaCache. A variant with -// no .meta file is served as application/octet-stream, which is not -// kept. +// GetVariant opens a cached variant and returns a reader, its size and its +// content type, read from the variant file's first bytes. It opens that one +// file and makes no database call. A variant not in the cache, or any +// variant of a disabled cache, is ErrNotFound; a variant file whose first +// bytes are not those of an output format is another error. func (c *Cache) GetVariant( cacheKey imagerequest.VariantKey, ) (io.ReadCloser, int64, string, error) { @@ -238,20 +199,7 @@ func (c *Cache) GetVariant( return nil, 0, "", ErrNotFound } - contentType, known := c.metaCache.Get(cacheKey) - if !known { - return c.loadVariantWithMeta(cacheKey) - } - - reader, size, err := c.variants.LoadWithSize(cacheKey) - if err != nil { - // The file is gone, e.g. deleted outside pixa - c.metaCache.Remove(cacheKey) - - return nil, 0, "", err - } - - return reader, size, contentType, nil + return c.variants.LoadWithContentType(cacheKey) } // StoreSource stores fetched source content: it writes the file named by @@ -321,13 +269,11 @@ func (c *Cache) StoreVariant( return nil } - size, err := c.variants.Store(cacheKey, content, contentType) + size, err := c.variants.Store(cacheKey, content) if err != nil { return err } - c.metaCache.Add(cacheKey, contentType) - _, err = c.db.ExecContext(ctx, ` INSERT INTO variant_content (cache_key, size_bytes, content_type) VALUES (?, ?, ?) @@ -348,9 +294,9 @@ func (c *Cache) StoreVariant( // LookupSource checks if we have cached source content for a request. // Returns the source key and content type if found, or empty values -// if not; a row whose file is missing is not found. Hits touch the -// source's LRU timestamp; a disabled cache always reports no cached -// source. +// if not; a row whose file is missing is not found. A source found has +// its access time handed to the background writer without waiting (see +// touchSourceContent); a disabled cache always reports no cached source. func (c *Cache) LookupSource( ctx context.Context, req *imagerequest.Request, ) (imagerequest.SourceKey, string, error) { @@ -379,7 +325,7 @@ func (c *Cache) LookupSource( return "", "", nil } - c.touchSourceContent(ctx, key) + c.touchSourceContent(key) return key, contentType, nil } @@ -480,92 +426,33 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { // IncrementStats counts a cache hit or miss, and an upstream fetch that read // fetchBytes bytes, as IncrementUpstreamFetch does. Like the other Increment -// methods, it writes to the database with ctx's deadline but not its -// cancellation, so an ended request is still counted but does not wait for -// the database past its deadline; a count not written by then becomes a -// pending count (see StartPendingCountWrites). -func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) { - countCtx, cancel := withoutCancelKeepingDeadline(ctx) - defer cancel() - - var err error - - pendingCount := &c.pendingMisses - +// methods, it only adds to the pending counts, which the goroutine +// StartPendingCountWrites starts writes to the database, so it never waits +// for the database or fails. +func (c *Cache) IncrementStats(hit bool, fetchBytes int64) { if hit { - pendingCount = &c.pendingHits - - _, err = c.db.ExecContext(countCtx, ` - UPDATE cache_stats - SET hit_count = hit_count + 1, - last_updated_at = CURRENT_TIMESTAMP - WHERE id = 1 - `) + c.pendingHits.Add(1) } else { - _, err = c.db.ExecContext(countCtx, ` - UPDATE cache_stats - SET miss_count = miss_count + 1, - last_updated_at = CURRENT_TIMESTAMP - WHERE id = 1 - `) + c.pendingMisses.Add(1) } - switch { - case errors.Is(err, context.DeadlineExceeded): - c.addPendingCount(pendingCount, 1) - case err != nil: - c.log.Warn("failed to count cache hit or miss", "hit", hit, "error", err) - } - - c.IncrementUpstreamFetch(ctx, fetchBytes) + c.IncrementUpstreamFetch(fetchBytes) } // IncrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes. // A fetch that read no bytes is not counted. -func (c *Cache) IncrementUpstreamFetch(ctx context.Context, fetchBytes int64) { +func (c *Cache) IncrementUpstreamFetch(fetchBytes int64) { if fetchBytes <= 0 { return } - countCtx, cancel := withoutCancelKeepingDeadline(ctx) - defer cancel() - - _, err := c.db.ExecContext(countCtx, ` - UPDATE cache_stats - SET upstream_fetch_count = upstream_fetch_count + 1, - upstream_fetch_bytes = upstream_fetch_bytes + ?, - last_updated_at = CURRENT_TIMESTAMP - WHERE id = 1 - `, fetchBytes) - - switch { - case errors.Is(err, context.DeadlineExceeded): - c.addPendingCount(&c.pendingUpstreamFetches, 1) - c.addPendingCount(&c.pendingUpstreamFetchBytes, fetchBytes) - case err != nil: - c.log.Warn("failed to count upstream fetch", - "fetch_bytes", fetchBytes, "error", err) - } + c.pendingUpstreamFetches.Add(1) + c.pendingUpstreamFetchBytes.Add(fetchBytes) } // IncrementTransformCount counts one image transcoded by the image processor. -func (c *Cache) IncrementTransformCount(ctx context.Context) { - countCtx, cancel := withoutCancelKeepingDeadline(ctx) - defer cancel() - - _, err := c.db.ExecContext(countCtx, ` - UPDATE cache_stats - SET transform_count = transform_count + 1, - last_updated_at = CURRENT_TIMESTAMP - WHERE id = 1 - `) - - switch { - case errors.Is(err, context.DeadlineExceeded): - c.addPendingCount(&c.pendingTransforms, 1) - case err != nil: - c.log.Warn("failed to count transform", "error", err) - } +func (c *Cache) IncrementTransformCount() { + c.pendingTransforms.Add(1) } // readHitAndMissCounts sets the hit and miss counts of stats to those of the @@ -587,58 +474,33 @@ func (c *Cache) readHitAndMissCounts(ctx context.Context, stats *CacheStats) err return err } -// loadVariantWithMeta is GetVariant for a variant metaCache does not -// hold: it reads the content type from the variant's .meta file and -// keeps it in metaCache, unless a StoreVariant has put one there -// meanwhile, as the store's is newer. A read that finds no .meta file, -// as one can when the store's .meta write failed or an eviction removed -// the file after the read opened the variant, serves -// application/octet-stream and keeps nothing, so metaCache only ever -// holds a type read from a .meta file or passed to StoreVariant. -func (c *Cache) loadVariantWithMeta( - cacheKey imagerequest.VariantKey, -) (io.ReadCloser, int64, string, error) { - reader, size, contentType, err := c.variants.LoadWithMeta(cacheKey) - if err != nil { - return nil, 0, "", err - } - - if contentType == "" { - return reader, size, fallbackContentType, nil - } - - c.metaCache.ContainsOrAdd(cacheKey, contentType) - - return reader, size, contentType, nil -} - -// touchVariant updates the LRU timestamp of a variant, best-effort: -// a failed touch only makes the entry look colder to eviction. -func (c *Cache) touchVariant(ctx context.Context, cacheKey imagerequest.VariantKey) { - _, err := c.db.ExecContext(ctx, ` - UPDATE variant_content SET last_accessed_at = CURRENT_TIMESTAMP - WHERE cache_key = ? - `, string(cacheKey)) - if err != nil { - c.log.Debug("failed to touch variant LRU timestamp", - "cache_key", cacheKey, "error", err) +// touchVariant hands the access time of a variant served from the cache to +// the goroutine StartPendingCountWrites starts, which writes it in its next +// batch. It never waits: while maxPendingAccessTimes access times of +// variants wait to be written, this one is dropped, and the variant only +// looks colder to eviction. +func (c *Cache) touchVariant(cacheKey imagerequest.VariantKey) { + select { + case c.pendingVariantAccessTimes <- cacheKey: + default: } } -// touchSourceContent updates the LRU timestamp of a cached source, -// best-effort: a failed touch only makes the source look colder. -func (c *Cache) touchSourceContent(ctx context.Context, key imagerequest.SourceKey) { - _, err := c.db.ExecContext(ctx, ` - UPDATE source_content SET last_accessed_at = CURRENT_TIMESTAMP - WHERE source_key = ? - `, string(key)) - if err != nil { - c.log.Debug("failed to touch source content LRU timestamp", - "source_key", key, "error", err) +// touchSourceContent hands the access time of a cached source a miss used to +// the goroutine StartPendingCountWrites starts, which writes it in its next +// batch. It never waits: while maxPendingAccessTimes access times of sources +// wait to be written, this one is dropped, and the source only looks colder +// to eviction. +func (c *Cache) touchSourceContent(key imagerequest.SourceKey) { + select { + case c.pendingSourceAccessTimes <- key: + default: } } -// checkNegativeCache checks if a request is in the negative cache. +// checkNegativeCache checks if a request is in the negative cache. It only +// reads: a row past its expiry is not a hit, and the eviction goroutine +// deletes it with the other expired rows (see CleanExpired). func (c *Cache) checkNegativeCache( ctx context.Context, req *imagerequest.Request, ) (bool, error) { @@ -657,22 +519,9 @@ func (c *Cache) checkNegativeCache( return false, fmt.Errorf("failed to check negative cache: %w", err) } - // Check if expired - if time.Now().After(expiresAt) { - // Clean up expired entry - _, err = c.db.ExecContext(ctx, ` - DELETE FROM negative_cache - WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.Source.Host, req.Source.Path, req.Source.Query) - if err != nil { - c.log.Warn("failed to delete expired negative cache entry", - "host", req.Source.Host, "path", req.Source.Path, "error", err) - } + expired := time.Now().After(expiresAt) - return false, nil - } - - return true, nil + return !expired, nil } // cleanExpiredBatch deletes up to cleanExpiredBatchSize negative cache diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go new file mode 100644 index 0000000..b24ea08 --- /dev/null +++ b/internal/imgcache/cache_hit_internal_test.go @@ -0,0 +1,299 @@ +package imgcache + +import ( + "bytes" + "errors" + "fmt" + "image/color" + "io" + "os" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/magic" +) + +// TestGetVariant_ContentTypeFromFirstBytes stores an image in each output +// format as a variant, recorded with the content type text/plain, and checks +// that GetVariant serves it whole with the content type of its format, read +// from the file. +func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + processor := imageprocessor.New(imageprocessor.Params{}) + source := generateTestJPEG(t, 40, 30, color.RGBA{200, 40, 40, 255}) + + for _, format := range []imagerequest.Format{ + imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, + imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, + } { + result, err := processor.Process(t.Context(), bytes.NewReader(source), + &imagerequest.Transform{ + Size: imagerequest.Size{Width: 20, Height: 15}, + Format: format, + Quality: 85, + FitMode: imagerequest.FitCover, + }) + if err != nil { + t.Fatalf("making a %s image: %v", format, err) + } + + image, err := io.ReadAll(result.Content) + if err != nil { + t.Fatalf("reading the %s image: %v", format, err) + } + + key := imagerequest.VariantKey("aabbccdd10" + string(format)) + + err = cache.StoreVariant(t.Context(), key, bytes.NewReader(image), "text/plain") + if err != nil { + t.Fatalf("StoreVariant(%s) error = %v", format, err) + } + + reader, size, contentType, err := cache.GetVariant(key) + if err != nil { + t.Fatalf("GetVariant(%s) error = %v", format, err) + } + + served, err := io.ReadAll(reader) + _ = reader.Close() + + if err != nil { + t.Fatalf("reading the %s variant: %v", format, err) + } + + t.Logf("%s: %d bytes served as %s", format, size, contentType) + + if want := imageprocessor.FormatToMIME(format); contentType != want { + t.Errorf("%s variant content type = %q, want %q", format, contentType, want) + } + + if size != int64(len(image)) || !bytes.Equal(served, image) { + t.Errorf("%s variant served %d bytes, size %d, want the %d bytes stored", + format, len(served), size, len(image)) + } + } +} + +// TestGetVariant_RefusesFileOfNoOutputFormat checks that GetVariant refuses +// a variant file whose first bytes are not those of an output format, with +// an error other than ErrNotFound, and that a variant with no file, or any +// variant of a disabled cache, is ErrNotFound. +func TestGetVariant_RefusesFileOfNoOutputFormat(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + jpegImage := generateTestJPEG(t, 10, 10, color.RGBA{0, 0, 255, 255}) + + tests := []struct { + name string + content []byte + wantErr error + }{ + {"text", []byte("this is text, not an image"), magic.ErrUnknownFormat}, + {"SVG", []byte(``), + errNotAnOutputFormat}, + {"JPEG cut to 10 bytes", jpegImage[:10], io.EOF}, + {"empty", nil, io.EOF}, + } + + for i, tt := range tests { + key := imagerequest.VariantKey(fmt.Sprintf("aabbccdd20%02d", i)) + + _, err := cache.variants.Store(key, bytes.NewReader(tt.content)) + if err != nil { + t.Fatalf("%s: storing the variant file: %v", tt.name, err) + } + + reader, _, contentType, err := cache.GetVariant(key) + t.Logf("%s: GetVariant() error = %v", tt.name, err) + + if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) { + t.Errorf("%s: GetVariant() = reader %v, content type %q, error %v; "+ + "want no reader and %v", tt.name, reader, contentType, err, tt.wantErr) + } + } + + reader, _, _, err := cache.GetVariant(testVariantKeyFour) + if reader != nil || !errors.Is(err, ErrNotFound) { + t.Errorf("GetVariant() of a variant with no file = reader %v, error %v; "+ + "want no reader and %v", reader, err, ErrNotFound) + } + + disabled, _ := newEvictionTestCache(t, 0) + + reader, _, _, err = disabled.GetVariant(testVariantKeyOne) + if reader != nil || !errors.Is(err, ErrNotFound) { + t.Errorf("GetVariant() on a disabled cache = reader %v, error %v; "+ + "want no reader and %v", reader, err, ErrNotFound) + } +} + +// TestService_Get_MakesAgainVariantOfNoOutputFormat replaces a cached +// variant's file with text, and checks that the next request is a miss +// answered with the variant made again, and the one after a hit. +func TestService_Get_MakesAgainVariantOfNoOutputFormat(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + req := photoVariant(fixtures, 85, imagerequest.FitCover) + + want, _ := getImage(t, svc, req) + + path := svc.cache.variants.keyToPath(imagerequest.CacheKey(req)) + + err := os.WriteFile(path, []byte("this is text, not an image"), StorageFilePerm) + if err != nil { + t.Fatalf("replacing the variant file: %v", err) + } + + got, status := getImage(t, svc, req) + t.Logf("after the replacement: %d bytes, %s", len(got), status) + + if status != CacheMiss || !bytes.Equal(got, want) { + t.Errorf("after the replacement: Get() = %d bytes, %s; want the %d bytes "+ + "made again, %s", len(got), status, len(want), CacheMiss) + } + + got, status = getImage(t, svc, req) + + if status != CacheHit || !bytes.Equal(got, want) { + t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s", + len(got), status, len(want), CacheHit) + } +} + +// TestService_Get_HitWaitsForNoDatabaseWrite holds the database's one +// connection while a cached variant is requested. The hit must be answered +// while the connection is still held, and its access time and hit count +// must reach the database once it is free. +func TestService_Get_HitWaitsForNoDatabaseWrite(t *testing.T) { + t.Parallel() + + svc, fixtures := SetupTestService(t) + req := photoVariant(fixtures, 85, imagerequest.FitCover) + key := imagerequest.CacheKey(req) + + getImage(t, svc, req) + waitForPendingCountsWritten(t, svc.cache) + + anHourAgo := time.Now().Add(-time.Hour) + setVariantLastAccessed(t, svc.cache, key, anHourAgo) + + want := readCacheStatsCounters(t, svc.cache) + want.hitCount++ + + releaseDatabase := holdDatabase(t, svc.cache) + defer releaseDatabase() + + results := startGet(t.Context(), svc, req) + + select { + case got := <-results: + if got.err != nil { + t.Fatalf("Get() of the cached variant error = %v", got.err) + } + case <-time.After(pendingCountsWait): + t.Fatal("the cache hit waited for the database") + } + + releaseDatabase() + + waitForCounters(t, svc.cache, want) + + deadline := time.Now().Add(pendingCountsWait) + + for countRows(t, svc.cache, ` + SELECT COUNT(*) FROM variant_content + WHERE cache_key = ? AND last_accessed_at > ? + `, string(key), anHourAgo.UTC().Format(sqliteTimestampFormat)) == 0 { + if time.Now().After(deadline) { + t.Fatal("the hit's access time was not written") + } + + time.Sleep(10 * time.Millisecond) + } +} + +// TestTouchVariantAndTouchSourceContent_DropAccessTimesPastTheLimit hands one +// more access time of a variant, and one more of a source, than may wait to +// be written. touchVariant and touchSourceContent must return each time +// without waiting, and the last of each must be dropped. +func TestTouchVariantAndTouchSourceContent_DropAccessTimesPastTheLimit(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + done := make(chan struct{}) + + go func() { + defer close(done) + + for range maxPendingAccessTimes + 1 { + cache.touchVariant(testVariantKeyOne) + cache.touchSourceContent("aabbccdd3001") + } + }() + + select { + case <-done: + case <-time.After(pendingCountsWait): + t.Fatal("touchVariant or touchSourceContent waited once the access " + + "times reached the limit") + } + + if waiting := len(cache.pendingVariantAccessTimes); waiting != maxPendingAccessTimes { + t.Errorf("%d access times of variants wait to be written, want %d", + waiting, maxPendingAccessTimes) + } + + if waiting := len(cache.pendingSourceAccessTimes); waiting != maxPendingAccessTimes { + t.Errorf("%d access times of sources wait to be written, want %d", + waiting, maxPendingAccessTimes) + } +} + +// TestStopPendingCountWrites_WritesPendingAccessTimes hands the access time +// of one of two variants, three times, then stops the goroutine that writes +// them, which never started. The stop must write that access time and +// leave the other variant's. +func TestStopPendingCountWrites_WritesPendingAccessTimes(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + anHourAgo := time.Now().Add(-time.Hour) + + for _, key := range []imagerequest.VariantKey{testVariantKeyOne, testVariantKeyTwo} { + storeEvictionTestVariant(t, cache, key, []byte("variant bytes")) + setVariantLastAccessed(t, cache, key, anHourAgo) + } + + for range 3 { + cache.touchVariant(testVariantKeyOne) + } + + err := cache.StopPendingCountWrites(t.Context()) + if err != nil { + t.Fatalf("StopPendingCountWrites() error = %v", err) + } + + for key, wantTouched := range map[imagerequest.VariantKey]int{ + testVariantKeyOne: 1, testVariantKeyTwo: 0, + } { + touched := countRows(t, cache, ` + SELECT COUNT(*) FROM variant_content + WHERE cache_key = ? AND last_accessed_at > ? + `, string(key), anHourAgo.UTC().Format(sqliteTimestampFormat)) + if touched != wantTouched { + t.Errorf("variant %s: %d rows with a new access time, want %d", + key, touched, wantTouched) + } + } + + if waiting := len(cache.pendingVariantAccessTimes); waiting != 0 { + t.Errorf("%d access times still wait to be written, want 0", waiting) + } +} diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index 0150e73..a05f049 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -1,7 +1,6 @@ package imgcache import ( - "bytes" "context" "database/sql" "os" @@ -9,8 +8,6 @@ import ( "time" _ "modernc.org/sqlite" - "sneak.berlin/go/pixa/internal/httpfetcher" - "sneak.berlin/go/pixa/internal/imagerequest" ) func setupTestDB(t *testing.T) *sql.DB { @@ -100,347 +97,6 @@ func setupTestCache(t *testing.T) *Cache { return cache } -func TestCache_LookupMiss(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: testPathCat, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - result, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() error = %v", err) - } - - if result.Hit { - t.Error("Lookup() hit = true, want false for cache miss") - } - - if result.CacheStatus != CacheMiss { - t.Errorf("Lookup() status = %v, want %v", result.CacheStatus, CacheMiss) - } -} - -func TestCache_StoreAndLookup(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: testPathCat, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Store source content - sourceContent := []byte("fake jpeg data") - fetchResult := &httpfetcher.FetchResult{ - ContentType: testContentTypeJPEG, - Headers: map[string][]string{testHeaderContentType: {testContentTypeJPEG}}, - } - - sourceKey, err := cache.StoreSource( - ctx, req, bytes.NewReader(sourceContent), fetchResult) - if err != nil { - t.Fatalf("StoreSource() error = %v", err) - } - - if sourceKey == "" { - t.Error("StoreSource() returned empty key") - } - - // Store variant - cacheKey := imagerequest.CacheKey(req) - outputContent := []byte("fake webp data") - - err = cache.StoreVariant( - t.Context(), cacheKey, bytes.NewReader(outputContent), "image/webp") - if err != nil { - t.Fatalf("StoreVariant() error = %v", err) - } - - // Now lookup should hit - result, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() error = %v", err) - } - - if !result.Hit { - t.Error("Lookup() hit = false, want true after store") - } - - if result.CacheStatus != CacheHit { - t.Errorf("Lookup() status = %v, want %v", result.CacheStatus, CacheHit) - } - - if result.CacheKey == "" { - t.Error("Lookup() returned empty cache key") - } -} - -func TestCache_NegativeCache(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: "/photos/notfound.jpg", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - }, - } - - // Store negative cache entry - err := cache.StoreNegative(ctx, req, 404, "not found") - if err != nil { - t.Fatalf("StoreNegative() error = %v", err) - } - - // Lookup should return miss (negative cache is checked at service level) - result, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() error = %v", err) - } - - if result.Hit { - t.Error("Lookup() hit = true, want false for negative cache entry") - } -} - -func TestCache_NegativeCacheExpiry(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - db := setupTestDB(t) - - // Very short negative TTL for testing - cache, err := NewCache(db, CacheConfig{ - StateDir: tmpDir, - NegativeTTL: 1 * time.Millisecond, - }) - if err != nil { - t.Fatalf("failed to create cache: %v", err) - } - - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: "/photos/expired.jpg", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - }, - } - - // Store negative cache entry - err = cache.StoreNegative(ctx, req, 404, "not found") - if err != nil { - t.Fatalf("StoreNegative() error = %v", err) - } - - // Wait for expiry - time.Sleep(10 * time.Millisecond) - - // Lookup should return miss after expiry - result, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() error = %v, want nil after expiry", err) - } - - if result.Hit { - t.Error("Lookup() hit = true, want false after negative cache expiry") - } -} - -func TestCache_VariantLookup(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: "/photos/variant.jpg", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Store variant - cacheKey := imagerequest.CacheKey(req) - outputContent := []byte("output data") - - err := cache.StoreVariant( - t.Context(), cacheKey, bytes.NewReader(outputContent), "image/webp") - if err != nil { - t.Fatalf("StoreVariant() error = %v", err) - } - - // First lookup - result1, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() first error = %v", err) - } - - if !result1.Hit { - t.Error("Lookup() first hit = false") - } - - // Second lookup should also hit (from disk) - result2, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() second error = %v", err) - } - - if !result2.Hit { - t.Error("Lookup() second hit = false") - } - - if result1.CacheKey != result2.CacheKey { - t.Error("Lookup() returned different cache keys") - } -} - -func TestCache_GetVariant_ReturnsContentType(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: "/photos/variantct.jpg", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Store variant - cacheKey := imagerequest.CacheKey(req) - outputContent := []byte("output webp data") - - err := cache.StoreVariant( - t.Context(), cacheKey, bytes.NewReader(outputContent), "image/webp") - if err != nil { - t.Fatalf("StoreVariant() error = %v", err) - } - - // Lookup - result, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() error = %v", err) - } - - if !result.Hit { - t.Fatal("Lookup() hit = false, want true") - } - - // GetVariant should return content type - reader, size, contentType, err := cache.GetVariant(result.CacheKey) - if err != nil { - t.Fatalf("GetVariant() error = %v", err) - } - - defer func() { _ = reader.Close() }() - - if contentType != "image/webp" { - t.Errorf("GetVariant() ContentType = %q, want %q", contentType, "image/webp") - } - - if size != int64(len(outputContent)) { - t.Errorf("GetVariant() size = %d, want %d", size, len(outputContent)) - } -} - -func TestCache_GetVariant(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: "/photos/output.jpg", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: imagerequest.FormatWebP, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Store variant - cacheKey := imagerequest.CacheKey(req) - outputContent := []byte("the actual output content") - - err := cache.StoreVariant( - t.Context(), cacheKey, bytes.NewReader(outputContent), "image/webp") - if err != nil { - t.Fatalf("StoreVariant() error = %v", err) - } - - // Lookup to get cache key - result, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup() error = %v", err) - } - - // Get variant content - reader, _, _, err := cache.GetVariant(result.CacheKey) - if err != nil { - t.Fatalf("GetVariant() error = %v", err) - } - - defer func() { _ = reader.Close() }() - - buf := make([]byte, 100) - n, _ := reader.Read(buf) - - if !bytes.Equal(buf[:n], outputContent) { - t.Errorf("GetVariant() content = %q, want %q", buf[:n], outputContent) - } -} - func TestCache_Stats(t *testing.T) { t.Parallel() @@ -448,9 +104,9 @@ func TestCache_Stats(t *testing.T) { ctx := context.Background() // Increment some stats - cache.IncrementStats(ctx, true, 0) - cache.IncrementStats(ctx, true, 0) - cache.IncrementStats(ctx, false, 1024) + cache.IncrementStats(true, 0) + cache.IncrementStats(true, 0) + cache.IncrementStats(false, 1024) stats, err := cache.Stats(ctx) if err != nil { diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index cb1570f..f46b346 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -193,6 +193,8 @@ func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { want := cacheStatsCounters{0, requests, 1, int64(len(photo)), 1} + waitForPendingCountsWritten(t, svc.cache) + if got := readCacheStatsCounters(t, svc.cache); got != want { t.Errorf("counters = %+v, want %+v", got, want) } diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 6470cbc..509e1ac 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -3,7 +3,6 @@ package imgcache import ( "context" "database/sql" - "encoding/json" "errors" "fmt" "io/fs" @@ -52,12 +51,9 @@ const sqliteTimestampLayout = "2006-01-02 15:04:05" // writes (".tmp-*" patterns in the storage layer). const tempFilePrefix = ".tmp-" -// variantMetaSuffix is the sidecar suffix VariantStorage writes next -// to each variant file. -const variantMetaSuffix = ".meta" - -// fallbackContentType is the content type given to a variant file that -// has no readable .meta sidecar, when it is served or reconciled. +// fallbackContentType is the content type reconciliation records for a +// variant file it adopts whose first bytes are not those of an output +// format. Such a file is never served. const fallbackContentType = "application/octet-stream" // UsageBytes returns the total number of bytes of cache content @@ -298,10 +294,7 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro } // evictVariant removes one variant: accounting row first, then the -// content and .meta files, so the database never references a deleted -// file. The metaCache entry goes before the files; a GetVariant that -// read them just before may put it back, and the next GetVariant then -// fails to open the file and removes it again. +// file, so the database never references a deleted file. func (c *Cache) evictVariant( ctx context.Context, cacheKey imagerequest.VariantKey, ) error { @@ -311,8 +304,6 @@ func (c *Cache) evictVariant( return fmt.Errorf("failed to delete variant accounting row: %w", err) } - c.metaCache.Remove(cacheKey) - err = c.variants.Delete(cacheKey) if err != nil { return err @@ -521,12 +512,16 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error { return c.reconcileUsageTotal(ctx) } -// reconcileVariantFiles walks the variant storage directory, checking -// each variant file against its accounting row (reconcileVariantFile) -// and sweeping stale temp files. -func (c *Cache) reconcileVariantFiles(ctx context.Context) error { +// reconcileFiles walks baseDir, one of the cache's storage directories, +// sweeping stale temp files and passing each other file, with its name, to +// reconcileFile. Once ctx is cancelled it stops at the next file and +// returns ctx's error. +func (c *Cache) reconcileFiles( + ctx context.Context, baseDir string, + reconcileFile func(path string, entry fs.DirEntry, name string) error, +) error { return filepath.WalkDir( - c.variants.baseDir, + baseDir, func(path string, entry fs.DirEntry, err error) error { if ctx.Err() != nil { return ctx.Err() @@ -544,15 +539,21 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error { return nil } - if strings.HasSuffix(name, variantMetaSuffix) { - return nil - } - - return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name)) + return reconcileFile(path, entry, name) }, ) } +// reconcileVariantFiles walks the variant storage directory, checking +// each variant file against its accounting row (reconcileVariantFile) +// and sweeping stale temp files. +func (c *Cache) reconcileVariantFiles(ctx context.Context) error { + return c.reconcileFiles(ctx, c.variants.baseDir, + func(path string, entry fs.DirEntry, name string) error { + return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name)) + }) +} + // reconcileVariantFile checks one variant file against its accounting // row. A file without a row is adopted into the accounting, unless it is // empty. A file that is empty, or not the size its row records, as a @@ -574,7 +575,7 @@ func (c *Cache) reconcileVariantFile( switch { case errors.Is(err, sql.ErrNoRows) && info.Size() > 0: - return c.adoptVariantFile(ctx, path, info, cacheKey) + return c.adoptVariantFile(ctx, info, cacheKey) case err != nil && !errors.Is(err, sql.ErrNoRows): return fmt.Errorf("failed to check variant accounting row: %w", err) case info.Size() > 0 && info.Size() == recordedSize: @@ -589,14 +590,21 @@ func (c *Cache) reconcileVariantFile( } // adoptVariantFile inserts an accounting row for a variant file that -// has none, using the file's size and modification time. +// has none, using the file's size and modification time, and the content +// type its first bytes show, as a hit reads it. func (c *Cache) adoptVariantFile( - ctx context.Context, path string, info fs.FileInfo, cacheKey imagerequest.VariantKey, + ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey, ) error { modTime := info.ModTime().UTC().Format(sqliteTimestampLayout) - contentType := c.variantContentTypeFromSidecar(path) + contentType := fallbackContentType - _, err := c.db.ExecContext(ctx, ` + content, _, detectedType, err := c.variants.LoadWithContentType(cacheKey) + if err == nil { + _ = content.Close() + contentType = detectedType + } + + _, err = c.db.ExecContext(ctx, ` INSERT INTO variant_content (cache_key, size_bytes, content_type, created_at, last_accessed_at) VALUES (?, ?, ?, ?, ?) @@ -611,23 +619,6 @@ func (c *Cache) adoptVariantFile( return nil } -// variantContentTypeFromSidecar reads the content type from a variant -// .meta sidecar, falling back to application/octet-stream. -func (c *Cache) variantContentTypeFromSidecar(variantPath string) string { - //nolint:gosec // path from cache walk - metaData, err := os.ReadFile(variantPath + variantMetaSuffix) - if err != nil { - return fallbackContentType - } - - var meta VariantMeta - if json.Unmarshal(metaData, &meta) != nil || meta.ContentType == "" { - return fallbackContentType - } - - return meta.ContentType -} - // reconcileVariantRows drops accounting rows whose variant files are // missing, so the database never references deleted content. func (c *Cache) reconcileVariantRows(ctx context.Context) error { @@ -719,28 +710,10 @@ func queryStringColumn[T ~string]( // each blob file against its source_content row (reconcileSourceFile) // and sweeping stale temp files. func (c *Cache) reconcileSourceFiles(ctx context.Context) error { - return filepath.WalkDir( - c.srcContent.baseDir, - func(path string, entry fs.DirEntry, err error) error { - if ctx.Err() != nil { - return ctx.Err() - } - - if err != nil || entry.IsDir() { - return err - } - - name := entry.Name() - - if strings.HasPrefix(name, tempFilePrefix) { - c.sweepStaleTempFile(path, entry) - - return nil - } - + return c.reconcileFiles(ctx, c.srcContent.baseDir, + func(path string, entry fs.DirEntry, name string) error { return c.reconcileSourceFile(ctx, path, entry, imagerequest.SourceKey(name)) - }, - ) + }) } // reconcileSourceFile checks one source blob file against its diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 1655ee7..5cbc8a0 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -469,15 +469,6 @@ func assertDisabledCacheReadsAlwaysMiss( ctx := t.Context() - lookup, err := cache.Lookup(ctx, req) - if err != nil { - t.Fatalf("Lookup on disabled cache failed: %v", err) - } - - if lookup.Hit { - t.Error("Lookup on disabled cache must always miss") - } - key, srcType, err := cache.LookupSource(ctx, req) if err != nil { t.Fatalf("LookupSource on disabled cache failed: %v", err) @@ -616,7 +607,7 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) { for i, key := range keys { content := bytes.Repeat([]byte{fills[i]}, 1000) - _, err = cache.variants.Store(key, bytes.NewReader(content), "image/webp") + _, err = cache.variants.Store(key, bytes.NewReader(content)) if err != nil { t.Fatalf("failed to store variant file: %v", err) } @@ -668,7 +659,7 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { untracked := bytes.Repeat([]byte{0x31}, 1000) _, err := cache.variants.Store( - testVariantKeyOne, bytes.NewReader(untracked), "image/webp", + testVariantKeyOne, bytes.NewReader(untracked), ) if err != nil { t.Fatalf("failed to store untracked variant file: %v", err) @@ -770,7 +761,7 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { untracked := bytes.Repeat([]byte{0x41}, 900) _, err = cache.variants.Store( - "aabbccdd0099", bytes.NewReader(untracked), "image/webp", + "aabbccdd0099", bytes.NewReader(untracked), ) if err != nil { t.Fatalf("failed to store untracked variant file: %v", err) diff --git a/internal/imgcache/metacache_internal_test.go b/internal/imgcache/metacache_internal_test.go deleted file mode 100644 index 1eecbad..0000000 --- a/internal/imgcache/metacache_internal_test.go +++ /dev/null @@ -1,361 +0,0 @@ -package imgcache - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "sync" - "testing" - "time" - - "sneak.berlin/go/pixa/internal/imagerequest" -) - -// webpRequest returns a request for a 100x100 WebP variant of path. -func webpRequest(path string) *imagerequest.Request { - return &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: path, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 100, Height: 100}, - Format: imagerequest.FormatWebP, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } -} - -// assertVariantServed checks that GetVariant serves key with the given -// content and the image/webp content type storeEvictionTestVariant stores. -func assertVariantServed( - t *testing.T, cache *Cache, key imagerequest.VariantKey, content []byte, -) { - t.Helper() - - reader, size, contentType, err := cache.GetVariant(key) - if err != nil { - t.Fatalf("GetVariant(%s) error = %v", key, err) - } - - defer func() { _ = reader.Close() }() - - got, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("reading variant %s: %v", key, err) - } - - if !bytes.Equal(got, content) { - t.Errorf("GetVariant(%s) content = %q, want %q", key, got, content) - } - - if size != int64(len(content)) { - t.Errorf("GetVariant(%s) size = %d, want %d", key, size, len(content)) - } - - if contentType != testContentTypeWebP { - t.Errorf("GetVariant(%s) content type = %q, want %q", - key, contentType, testContentTypeWebP) - } -} - -// assertVariantNotFound checks that GetVariant refuses key with -// ErrNotFound. -func assertVariantNotFound( - t *testing.T, cache *Cache, key imagerequest.VariantKey, -) { - t.Helper() - - reader, _, _, err := cache.GetVariant(key) - if err == nil { - _ = reader.Close() - } - - if !errors.Is(err, ErrNotFound) { - t.Errorf("GetVariant(%s) error = %v, want ErrNotFound", key, err) - } -} - -// assertLookupMisses checks that Lookup reports request as a miss. -func assertLookupMisses( - t *testing.T, cache *Cache, request *imagerequest.Request, -) { - t.Helper() - - lookup, err := cache.Lookup(t.Context(), request) - if err != nil { - t.Fatalf("Lookup(%s) error = %v", request.Source.Path, err) - } - - if lookup.Hit { - t.Errorf("Lookup(%s) is a hit, want a miss", request.Source.Path) - } -} - -// renameFile renames the file at from to to. -func renameFile(t *testing.T, from, to string) { - t.Helper() - - err := os.Rename(from, to) - if err != nil { - t.Fatalf("renaming %s: %v", from, err) - } -} - -// TestSecondHitDoesNotReadMetaFile checks that once a variant has been -// stored or read, a hit takes its content type from memory: with the -// .meta file deleted, GetVariant must still return the stored content -// type rather than the application/octet-stream it uses without one. -func TestSecondHitDoesNotReadMetaFile(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<20) - content := []byte("webp variant bytes") - - storeEvictionTestVariant(t, cache, testVariantKeyOne, content) - - // A second Cache on the same state directory starts with nothing in - // memory, as pixad does after a restart, so its first read uses the - // .meta file. - restarted, err := NewCache(cache.db, cache.config) - if err != nil { - t.Fatalf("NewCache() error = %v", err) - } - - assertVariantServed(t, restarted, testVariantKeyOne, content) - - err = os.Remove(cache.variants.keyToPath(testVariantKeyOne) + ".meta") - if err != nil { - t.Fatalf("removing .meta file: %v", err) - } - - assertVariantServed(t, cache, testVariantKeyOne, content) - assertVariantServed(t, restarted, testVariantKeyOne, content) -} - -// TestReadDuringStoreKeepsStoredContentType checks that a GetVariant -// which began before StoreVariant finished cannot replace the content -// type the store kept in memory. Such a read can find the variant file -// but not yet its .meta file, and so gets application/octet-stream. The -// test deletes the .meta file after the store, then runs the part of -// GetVariant that comes after its check of memory. -func TestReadDuringStoreKeepsStoredContentType(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<20) - content := []byte("webp variant bytes") - - storeEvictionTestVariant(t, cache, testVariantKeyOne, content) - - err := os.Remove(cache.variants.keyToPath(testVariantKeyOne) + ".meta") - if err != nil { - t.Fatalf("removing .meta file: %v", err) - } - - reader, _, contentType, err := cache.loadVariantWithMeta(testVariantKeyOne) - if err != nil { - t.Fatalf("loadVariantWithMeta(%s) error = %v", testVariantKeyOne, err) - } - - _ = reader.Close() - - t.Logf("the read without a .meta file got content type %q", contentType) - - assertVariantServed(t, cache, testVariantKeyOne, content) -} - -// TestReadOfOlderMetaFileKeepsStoredContentType checks that a read -// which got its content type from a .meta file that StoreVariant had -// not yet rewritten cannot replace the type the store kept in memory. -// The test writes such a .meta file, with a different content type, -// after the store, then runs the part of GetVariant that comes after -// its check of memory. -func TestReadOfOlderMetaFileKeepsStoredContentType(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<20) - content := []byte("webp variant bytes") - - storeEvictionTestVariant(t, cache, testVariantKeyOne, content) - - olderMeta, err := json.Marshal(VariantMeta{ - ContentType: testContentTypeJPEG, - Size: int64(len(content)), - }) - if err != nil { - t.Fatalf("encoding .meta file: %v", err) - } - - metaPath := cache.variants.keyToPath(testVariantKeyOne) + ".meta" - - err = os.WriteFile(metaPath, olderMeta, StorageFilePerm) - if err != nil { - t.Fatalf("writing .meta file: %v", err) - } - - reader, _, contentType, err := cache.loadVariantWithMeta(testVariantKeyOne) - if err != nil { - t.Fatalf("loadVariantWithMeta(%s) error = %v", testVariantKeyOne, err) - } - - _ = reader.Close() - - if contentType != testContentTypeJPEG { - t.Fatalf("loadVariantWithMeta(%s) content type = %q, want %q from the .meta file", - testVariantKeyOne, contentType, testContentTypeJPEG) - } - - kept, _ := cache.metaCache.Get(testVariantKeyOne) - if kept != testContentTypeWebP { - t.Errorf("content type in memory = %q, want the stored %q", - kept, testContentTypeWebP) - } -} - -// TestFailedReadDuringStoreKeepsStoredContentType checks that a read -// which found no .meta file cannot leave application/octet-stream in -// memory, even when another read has removed the content type -// StoreVariant kept there. In this order: the store; a read that finds -// the variant in memory but cannot open its file, and so removes it -// from memory; a read that opened the variant file before the store -// wrote its .meta file. Later hits must get the stored content type. -func TestFailedReadDuringStoreKeepsStoredContentType(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<20) - content := []byte("webp variant bytes") - variantPath := cache.variants.keyToPath(testVariantKeyOne) - metaPath := variantPath + ".meta" - - storeEvictionTestVariant(t, cache, testVariantKeyOne, content) - - renameFile(t, variantPath, variantPath+".hidden") - assertVariantNotFound(t, cache, testVariantKeyOne) - renameFile(t, variantPath+".hidden", variantPath) - - renameFile(t, metaPath, metaPath+".hidden") - - reader, _, contentType, err := cache.GetVariant(testVariantKeyOne) - if err != nil { - t.Fatalf("GetVariant(%s) error = %v", testVariantKeyOne, err) - } - - _ = reader.Close() - - t.Logf("the read without a .meta file got content type %q", contentType) - - renameFile(t, metaPath+".hidden", metaPath) - - assertVariantServed(t, cache, testVariantKeyOne, content) -} - -// TestEvictedVariantIsNotServed checks that a variant the evictor -// removed is a miss and cannot be read, although it had been stored -// and served before. -func TestEvictedVariantIsNotServed(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1500) - - oldRequest := webpRequest("/old.jpg") - newRequest := webpRequest("/new.jpg") - oldKey := imagerequest.CacheKey(oldRequest) - newKey := imagerequest.CacheKey(newRequest) - oldContent := bytes.Repeat([]byte{0x01}, 1000) - newContent := bytes.Repeat([]byte{0x02}, 1000) - - storeEvictionTestVariant(t, cache, oldKey, oldContent) - storeEvictionTestVariant(t, cache, newKey, newContent) - assertVariantServed(t, cache, oldKey, oldContent) - setVariantLastAccessed(t, cache, oldKey, time.Now().Add(-time.Hour)) - - err := cache.EvictToLimit(t.Context()) - if err != nil { - t.Fatalf("EvictToLimit() error = %v", err) - } - - assertLookupMisses(t, cache, oldRequest) - assertVariantNotFound(t, cache, oldKey) - assertVariantServed(t, cache, newKey, newContent) -} - -// TestVariantDeletedFromDiskIsNotServed checks that a variant whose -// file was deleted by something other than the evictor cannot be read, -// and is a miss afterwards. -func TestVariantDeletedFromDiskIsNotServed(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<20) - - request := webpRequest("/deleted.jpg") - key := imagerequest.CacheKey(request) - content := []byte("webp variant bytes") - - storeEvictionTestVariant(t, cache, key, content) - assertVariantServed(t, cache, key, content) - - err := os.Remove(cache.variants.keyToPath(key)) - if err != nil { - t.Fatalf("removing variant file: %v", err) - } - - assertVariantNotFound(t, cache, key) - assertLookupMisses(t, cache, request) -} - -// TestConcurrentVariantStoreReadAndEvict stores, reads and evicts -// variants from several goroutines at once, for the race detector. -func TestConcurrentVariantStoreReadAndEvict(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<20) - ctx := t.Context() - - var wg sync.WaitGroup - - for goroutine := range 8 { - wg.Go(func() { - key := imagerequest.VariantKey(fmt.Sprintf("aabbccdd01%02d", goroutine)) - content := []byte(key) - - for range 20 { - err := cache.StoreVariant( - ctx, key, bytes.NewReader(content), testContentTypeWebP) - if err != nil { - t.Errorf("StoreVariant(%s) error = %v", key, err) - - return - } - - reader, _, contentType, err := cache.GetVariant(key) - if err != nil { - t.Errorf("GetVariant(%s) error = %v", key, err) - - return - } - - _ = reader.Close() - - if contentType != testContentTypeWebP { - t.Errorf("GetVariant(%s) content type = %q, want %q", - key, contentType, testContentTypeWebP) - } - - err = cache.evictVariant(ctx, key) - if err != nil { - t.Errorf("evictVariant(%s) error = %v", key, err) - - return - } - - assertVariantNotFound(t, cache, key) - } - }) - } - - wg.Wait() -} diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index 6f41ba6..c8ff377 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -128,6 +128,8 @@ func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) { getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) + waitForPendingCountsWritten(t, svc.cache) + fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount t.Logf("upstream fetches = %d", fetches) @@ -164,6 +166,8 @@ func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) { getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) + waitForPendingCountsWritten(t, svc.cache) + fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount t.Logf("upstream fetches = %d", fetches) @@ -239,8 +243,7 @@ func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) { cache, _ := newEvictionTestCache(t, 1<<30) - _, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil), - testContentTypeWebP) + _, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil)) if err != nil { t.Fatalf("failed to store an empty variant file: %v", err) } @@ -261,59 +264,6 @@ func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) { } } -// TestVariantStorage_StoreReplacesMetaFile stores a variant over one -// already stored and checks that Store puts a new .meta file in place of -// the old one, as it does the image, rather than writing into the old -// file, which a crash could leave empty. The old .meta file is given a -// second name before the store, so that it can be told apart from the new -// one after it. -func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) { - t.Parallel() - - storage, key := storeTestVariant(t) - metaPath := storage.keyToPath(key) + variantMetaSuffix - oldMetaPath := metaPath + ".old" - - err := os.Link(metaPath, oldMetaPath) - if err != nil { - t.Fatalf("giving the .meta file a second name: %v", err) - } - - _, err = storage.Store(key, bytes.NewReader([]byte("new variant data")), - testContentTypeJPEG) - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - oldMeta, err := os.Stat(oldMetaPath) - if err != nil { - t.Fatalf("stat of the old .meta file: %v", err) - } - - newMeta, err := os.Stat(metaPath) - if err != nil { - t.Fatalf("stat of the new .meta file: %v", err) - } - - if os.SameFile(oldMeta, newMeta) { - t.Error("Store() wrote into the old .meta file instead of putting " + - "a new one in its place") - } - - reader, _, contentType, err := storage.LoadWithMeta(key) - if err != nil { - t.Fatalf("LoadWithMeta() error = %v", err) - } - - _ = reader.Close() - - t.Logf("content type after the store: %s", contentType) - - if contentType != testContentTypeJPEG { - t.Errorf("content type = %q, want %q", contentType, testContentTypeJPEG) - } -} - // TestContentStorage_StoreReplacesFileOfAnotherSize checks that a // store over a file of another size at the source's path, such as a // partial file a crash left, replaces it with the content. diff --git a/internal/imgcache/pendingcounts.go b/internal/imgcache/pendingcounts.go index c967b01..5a7eef0 100644 --- a/internal/imgcache/pendingcounts.go +++ b/internal/imgcache/pendingcounts.go @@ -2,17 +2,26 @@ package imgcache import ( "context" + "errors" "fmt" - "sync/atomic" + "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) -// StartPendingCountWrites starts the goroutine that writes the pending -// counts to the database whenever there are some, one UPDATE at a time: -// however many requests pass their deadline before their counts are -// written, at most this one write waits for the database. It is a no-op -// when already started. The goroutine outlives the caller, so it runs with -// its own context, which StopPendingCountWrites cancels to tell it to -// finish. +// pendingWriteInterval is how often the goroutine StartPendingCountWrites +// starts writes the pending access times and counts. SQLite's +// CURRENT_TIMESTAMP, which an access time is written as, counts whole +// seconds. +const pendingWriteInterval = time.Second + +// StartPendingCountWrites starts the goroutine that writes, every +// pendingWriteInterval, the pending access times in one transaction and then +// the pending counts in one UPDATE. Requests only hand these to it, so none +// waits for the database to record what it did, and only this goroutine's +// write ever waits for it. It is a no-op when already started. The goroutine +// outlives the caller, so it runs with its own context, which +// StopPendingCountWrites cancels to tell it to finish. func (c *Cache) StartPendingCountWrites() { if c.pendingCountsCancel != nil { return @@ -28,12 +37,12 @@ func (c *Cache) StartPendingCountWrites() { // StopPendingCountWrites tells the goroutine StartPendingCountWrites // started, if it did, to finish, and waits for its write under way to end -// and for it to return. It then writes the pending counts left, so that -// they are written before the database closes. It waits for all of this at -// most until ctx ends, then logs the counts not written and returns an -// error. +// and for it to return. It then writes the pending access times and counts +// left, once, so that they are written before the database closes. It waits +// for all of this at most until ctx ends. It logs what was not written, +// access times or counts, and returns an error. func (c *Cache) StopPendingCountWrites(ctx context.Context) error { - var err error + var accessTimesErr, countsErr error if c.pendingCountsCancel != nil { c.pendingCountsCancel() @@ -41,71 +50,134 @@ func (c *Cache) StopPendingCountWrites(ctx context.Context) error { select { case <-c.pendingCountsDone: case <-ctx.Done(): - // The write given up on here may not reach the database: - // database/sql closes without waiting for a query, and a - // write still waiting for the connection fails when the - // database closes. - err = fmt.Errorf("pending counts still being written: %w", ctx.Err()) + // The write given up on here, of access times or of counts, + // may not reach the database: database/sql closes without + // waiting for a query, and a write still waiting for the + // connection fails when the database closes. Neither is + // written here. + accessTimesErr = fmt.Errorf("access times still being written: %w", ctx.Err()) + countsErr = fmt.Errorf("pending counts still being written: %w", ctx.Err()) } } - if err == nil { - err = c.writePendingCounts(ctx) + if accessTimesErr == nil && countsErr == nil { + accessTimesErr = c.writePendingAccessTimes(ctx) + countsErr = c.writePendingCounts(ctx) } - if err != nil { + if accessTimesErr != nil { + c.log.Error("access times not written at shutdown", "error", accessTimesErr) + } + + if countsErr != nil { c.log.Error("counts not written at shutdown", "hits", c.pendingHits.Load(), "misses", c.pendingMisses.Load(), "upstream_fetches", c.pendingUpstreamFetches.Load(), "upstream_fetch_bytes", c.pendingUpstreamFetchBytes.Load(), "transforms", c.pendingTransforms.Load(), - "error", err, + "error", countsErr, ) - - return err } - return nil + return errors.Join(accessTimesErr, countsErr) } // pendingCountWriteLoop is the body of the goroutine // StartPendingCountWrites starts. Once ctx is cancelled it starts no write -// and returns, but does not cut off a write under way then. A write that -// fails leaves the counts pending, for the next write. +// and returns, but does not cut off a write under way then. Counts whose +// write fails stay pending, for the next write; access times whose write +// fails are dropped. func (c *Cache) pendingCountWriteLoop(ctx context.Context) { defer close(c.pendingCountsDone) + ticker := time.NewTicker(pendingWriteInterval) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - case <-c.pendingCountsAdded: + case <-ticker.C: } - // select picks at random when ctx is cancelled and a wakeup - // waits too. + // select picks at random when ctx is cancelled and a tick waits + // too. if ctx.Err() != nil { return } - err := c.writePendingCounts(context.WithoutCancel(ctx)) + err := c.writePendingAccessTimes(context.WithoutCancel(ctx)) + if err != nil { + c.log.Warn("failed to write access times", "error", err) + } + + err = c.writePendingCounts(context.WithoutCancel(ctx)) if err != nil { c.log.Warn("failed to write pending counts", "error", err) } } } -// addPendingCount adds n to pendingCount, one of the pending counts, and -// wakes the goroutine that writes them. pendingCountsAdded has capacity one, -// so a wakeup already waiting is enough. -func (c *Cache) addPendingCount(pendingCount *atomic.Int64, n int64) { - pendingCount.Add(n) +// writePendingAccessTimes takes out the variants waiting in +// pendingVariantAccessTimes and the sources waiting in +// pendingSourceAccessTimes, and sets their last_accessed_at to now in one +// transaction, once for each however often it was used. Nothing else takes +// from those channels while it runs: only the goroutine +// StartPendingCountWrites starts calls it, or StopPendingCountWrites once +// that goroutine has returned. +func (c *Cache) writePendingAccessTimes(ctx context.Context) error { + variants := make(map[imagerequest.VariantKey]struct{}) - select { - case c.pendingCountsAdded <- struct{}{}: - default: + for range len(c.pendingVariantAccessTimes) { + variants[<-c.pendingVariantAccessTimes] = struct{}{} } + + sources := make(map[imagerequest.SourceKey]struct{}) + + for range len(c.pendingSourceAccessTimes) { + sources[<-c.pendingSourceAccessTimes] = struct{}{} + } + + accessTimes := len(variants) + len(sources) + if accessTimes == 0 { + return nil + } + + tx, err := c.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + + // Does nothing once the transaction is committed + defer func() { _ = tx.Rollback() }() + + for cacheKey := range variants { + _, err = tx.ExecContext(ctx, ` + UPDATE variant_content SET last_accessed_at = CURRENT_TIMESTAMP + WHERE cache_key = ? + `, string(cacheKey)) + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + } + + for sourceKey := range sources { + _, err = tx.ExecContext(ctx, ` + UPDATE source_content SET last_accessed_at = CURRENT_TIMESTAMP + WHERE source_key = ? + `, string(sourceKey)) + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + + return nil } // writePendingCounts adds the pending counts to the cache_stats row in one diff --git a/internal/imgcache/pendingcounts_internal_test.go b/internal/imgcache/pendingcounts_internal_test.go index e8a559a..aaa63cd 100644 --- a/internal/imgcache/pendingcounts_internal_test.go +++ b/internal/imgcache/pendingcounts_internal_test.go @@ -1,9 +1,12 @@ package imgcache import ( + "bytes" "context" "database/sql" "errors" + "log/slog" + "strings" "testing" "time" @@ -73,6 +76,35 @@ func waitForCounters(t *testing.T, cache *Cache, want cacheStatsCounters) { } } +// waitForPendingCountsWritten starts the goroutine that writes cache's +// pending counts, unless it is running, and waits until it has written them +// all, so that the database holds every count made so far. The goroutine is +// stopped when the test ends. The database is limited to one connection +// first: each connection to an in-memory SQLite database opens a new, empty +// one, so the goroutine must share the connection that holds the schema. +func waitForPendingCountsWritten(t *testing.T, cache *Cache) { + t.Helper() + + if cache.pendingCountsCancel == nil { + cache.db.SetMaxOpenConns(1) + cache.StartPendingCountWrites() + + t.Cleanup(func() { _ = cache.StopPendingCountWrites(context.Background()) }) + } + + deadline := time.Now().Add(pendingCountsWait) + + for cache.pendingHits.Load()+cache.pendingMisses.Load()+ + cache.pendingUpstreamFetches.Load()+cache.pendingUpstreamFetchBytes.Load()+ + cache.pendingTransforms.Load() > 0 { + if time.Now().After(deadline) { + t.Fatal("the pending counts were not written") + } + + time.Sleep(10 * time.Millisecond) + } +} + // TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the // database's one connection while a request whose fetch is held reaches its // deadline. The request must return with the deadline's error while the @@ -123,11 +155,12 @@ func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { waitForCounters(t, svc.cache, cacheStatsCounters{missCount: 1}) } -// TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting holds the -// database's one connection while many misses are counted past their -// deadline. At most one write may then wait for the connection, and every -// miss must reach the database once it is free. -func TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting(t *testing.T) { +// TestIncrementStats_ManyCountsLeaveOneWriteWaiting holds the database's one +// connection while many misses are added to the pending counts, which the +// goroutine StartPendingCountWrites starts writes to the database. At most one +// write may then wait for the connection, and every miss must reach the +// database once it is free. +func TestIncrementStats_ManyCountsLeaveOneWriteWaiting(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) @@ -140,14 +173,10 @@ func TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting(t *testi waitsBefore := cache.db.Stats().WaitCount - // Writes given a context past its deadline fail at once. - ended, cancel := context.WithDeadline(t.Context(), time.Now()) - defer cancel() - const misses = 20 for range misses { - cache.IncrementStats(ended, false, 0) + cache.IncrementStats(false, 0) } // Once one write waits for the connection, any others start waiting too. @@ -166,9 +195,9 @@ func TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting(t *testi waitForCounters(t, cache, cacheStatsCounters{missCount: misses}) } -// TestStats_IncludesPendingCounts counts hits and misses whose writes miss -// their deadline, and checks that Stats adds them to the database's counts -// before they are written. +// TestStats_IncludesPendingCounts adds hits and misses to the pending counts, +// with no goroutine started to write them, and checks that Stats adds them to +// the database's counts before they are written. func TestStats_IncludesPendingCounts(t *testing.T) { t.Parallel() @@ -180,13 +209,9 @@ func TestStats_IncludesPendingCounts(t *testing.T) { t.Fatal(err) } - // Writes given a context past its deadline fail at once. - ended, cancel := context.WithDeadline(t.Context(), time.Now()) - defer cancel() - - cache.IncrementStats(ended, true, 0) - cache.IncrementStats(ended, true, 0) - cache.IncrementStats(ended, false, 0) + cache.IncrementStats(true, 0) + cache.IncrementStats(true, 0) + cache.IncrementStats(false, 0) stats, err := cache.Stats(t.Context()) if err != nil { @@ -222,19 +247,15 @@ func TestStopPendingCountWrites_WritesPendingCounts(t *testing.T) { waitsBefore := cache.db.Stats().WaitCount - // Writes given a context past its deadline fail at once. - ended, cancel := context.WithDeadline(t.Context(), time.Now()) - defer cancel() - - cache.IncrementStats(ended, true, 0) + cache.IncrementStats(true, 0) // The goroutine's write waits for the connection. waitForConnectionWaits(t, cache.db, waitsBefore+1) // Counted after that write read the pending counts - cache.IncrementStats(ended, false, 0) - cache.IncrementUpstreamFetch(ended, 1024) - cache.IncrementTransformCount(ended) + cache.IncrementStats(false, 0) + cache.IncrementUpstreamFetch(1024) + cache.IncrementTransformCount() // The goroutine is told to finish here as well as by the stop, so that it // is told before its write under way ends, however late the stop runs. @@ -295,11 +316,7 @@ func TestStopPendingCountWrites_ReturnsWhenItsContextEnds(t *testing.T) { waitsBefore := cache.db.Stats().WaitCount - // Writes given a context past its deadline fail at once. - ended, cancel := context.WithDeadline(t.Context(), time.Now()) - defer cancel() - - cache.IncrementStats(ended, true, 0) + cache.IncrementStats(true, 0) // The goroutine's write waits for the connection. waitForConnectionWaits(t, cache.db, waitsBefore+1) @@ -327,3 +344,88 @@ func TestStopPendingCountWrites_ReturnsWhenItsContextEnds(t *testing.T) { waitForCounters(t, cache, cacheStatsCounters{hitCount: 1}) } + +// TestLookupSource_HandsAccessTimeToTheBackgroundWriter finds a cached source +// whose access time is an hour old. LookupSource must not write the access +// time itself but hand it over to be written, and StopPendingCountWrites must +// then write it. +func TestLookupSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg", + []byte("source bytes")) + anHourAgo := time.Now().Add(-time.Hour) + setSourceLastAccessed(t, cache, key, anHourAgo) + + touchedQuery := ` + SELECT COUNT(*) FROM source_content + WHERE source_key = ? AND last_accessed_at > ? + ` + anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat) + + found, _, err := cache.LookupSource(t.Context(), &imagerequest.Request{ + Source: imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}, + }) + if err != nil || found != key { + t.Fatalf("LookupSource() = %q, %v; want %q", found, err, key) + } + + if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 0 { + t.Error("LookupSource wrote the source's access time itself") + } + + if waiting := len(cache.pendingSourceAccessTimes); waiting != 1 { + t.Errorf("%d access times of sources wait to be written, want 1", waiting) + } + + err = cache.StopPendingCountWrites(t.Context()) + if err != nil { + t.Fatalf("StopPendingCountWrites() error = %v", err) + } + + if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 1 { + t.Error("the source's access time was not written") + } +} + +// TestStopPendingCountWrites_LogsWhatWasNotWritten makes the write of the +// access times fail at shutdown, by dropping the table they are written to, +// while a hit count is written. The stop must return an error and log that +// access times were not written, and must not log the counts as not written. +func TestStopPendingCountWrites_LogsWhatWasNotWritten(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + var logs bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logs, nil)) + + cache.touchVariant(testVariantKeyOne) + cache.IncrementStats(true, 0) + + _, err := cache.db.ExecContext(t.Context(), `DROP TABLE variant_content`) + if err != nil { + t.Fatalf("dropping variant_content: %v", err) + } + + err = cache.StopPendingCountWrites(t.Context()) + t.Logf("StopPendingCountWrites() error = %v; logged:\n%s", err, logs.String()) + + if err == nil { + t.Error("StopPendingCountWrites() error = nil, want the access times' error") + } + + if !strings.Contains(logs.String(), "access times not written at shutdown") { + t.Error("the access times not written were not logged") + } + + if strings.Contains(logs.String(), "counts not written at shutdown") { + t.Error("the counts, which were written, were logged as not written") + } + + if got := readCacheStatsCounters(t, cache); got.hitCount != 1 { + t.Errorf("hit count in the database = %d, want 1", got.hitCount) + } +} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index d0e345d..37bf41a 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -140,36 +140,37 @@ var ErrProcessingTimedOut = errors.New( // While its source URL is in the negative cache, a cached variant is still // served and a variant not in the cache is still made from a cached source; // only a request that needs the source fetched is refused. +// +// A cache hit opens the variant file and makes no database call: its access +// time and its hit count are handed to the goroutine that +// StartPendingCountWrites starts, without waiting for it. A variant file +// that cannot be served, such as one whose first bytes are not those of an +// output format, is a miss, and the variant is made again. func (s *Service) Get( ctx context.Context, req *imagerequest.Request, ) (*ImageResponse, error) { // Propagate AllowHTTP setting to the request req.AllowHTTP = s.allowHTTP - // Check variant cache first (disk only, no DB) - result, err := s.cache.Lookup(ctx, req) - if err != nil { - s.log.Warn("cache lookup failed", "error", err) + cacheKey := imagerequest.CacheKey(req) + + reader, size, contentType, err := s.cache.GetVariant(cacheKey) + if err == nil { + s.cache.touchVariant(cacheKey) + s.cache.IncrementStats(true, 0) + + return &ImageResponse{ + Content: reader, + ContentLength: size, + ContentType: contentType, + CacheStatus: CacheHit, + ETag: formatETag(cacheKey), + }, nil } - // Cache hit - serve directly from disk - if result != nil && result.Hit { - reader, size, contentType, err := s.cache.GetVariant(result.CacheKey) - if err != nil { - s.log.Error("failed to get cached variant", "key", result.CacheKey, "error", err) - // Fall through to re-process - } else { - // Counted also when the request context has ended meanwhile - s.cache.IncrementStats(ctx, true, 0) - - return &ImageResponse{ - Content: reader, - ContentLength: size, - ContentType: contentType, - CacheStatus: CacheHit, - ETag: formatETag(result.CacheKey), - }, nil - } + if !errors.Is(err, ErrNotFound) { + s.log.Warn("cached variant not served, making it again", + "cache_key", cacheKey, "error", err) } // Cache miss - get the variant, processed once for all the requests that @@ -177,7 +178,7 @@ func (s *Service) Get( // failed or the request context has ended meanwhile response, err := s.processOrWait(ctx, req) - s.cache.IncrementStats(ctx, false, 0) + s.cache.IncrementStats(false, 0) if err != nil { return nil, err @@ -474,7 +475,7 @@ func (s *Service) fetchAndProcess( fetchBytes := int64(len(sourceData)) // Counted also when the request context has ended meanwhile - s.cache.IncrementUpstreamFetch(ctx, fetchBytes) + s.cache.IncrementUpstreamFetch(fetchBytes) if err != nil { return nil, fmt.Errorf("failed to read upstream response: %w", err) @@ -575,7 +576,7 @@ func (s *Service) processAndStore( processDuration := time.Since(processStart) // Counted also when the request context has ended meanwhile - s.cache.IncrementTransformCount(ctx) + s.cache.IncrementTransformCount() // Read processed content processedData, err := io.ReadAll(processResult.Content) diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index 92915ca..2183d30 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -2,7 +2,6 @@ package imgcache import ( "context" - "encoding/json" "io" "os" "testing" @@ -552,10 +551,10 @@ func TestService_Get_Caching(t *testing.T) { // a variant of it and sets back two days every time pixa stored for them. // For the source: fetched_at and last_accessed_at in source_content, and the // access and modification times of its file. For the variant: created_at and -// last_accessed_at in variant_content, the created_at in its .meta file, and -// the access and modification times of its file and of its .meta file. It -// then checks that the variant is still served from the cache and that a new -// variant is made from the cached source without fetching it again. +// last_accessed_at in variant_content, and the access and modification times +// of its file. It then checks that the variant is still served from the +// cache and that a new variant is made from the cached source without +// fetching it again. // // Left at the present: the files' status change and birth times, which a // program cannot set, and the times of the directories holding the files, @@ -598,6 +597,8 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { _ = resp.Content.Close() + waitForPendingCountsWritten(t, svc.cache) + fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount if fetches != 1 { t.Errorf("upstream fetches = %d, want 1", fetches) @@ -636,41 +637,16 @@ func backdateVariant( t.Helper() key := imagerequest.CacheKey(req) - - content, size, contentType, err := cache.variants.LoadWithMeta(key) - if err != nil || contentType == "" { - t.Fatalf("reading the variant's .meta file: content type %q, error %v", - contentType, err) - } - - _ = content.Close() - - meta, err := json.Marshal(VariantMeta{ - ContentType: contentType, - Size: size, - CreatedAt: when.Unix(), - }) - if err != nil { - t.Fatalf("encoding the variant's .meta file: %v", err) - } - path := cache.variants.keyToPath(key) - err = os.WriteFile(path+variantMetaSuffix, meta, StorageFilePerm) - if err != nil { - t.Fatalf("writing the variant's .meta file: %v", err) - } - updateOneRow(t, cache, `UPDATE variant_content SET created_at = ? WHERE cache_key = ?`, when.UTC().Format(sqliteTimestampFormat), string(key)) setVariantLastAccessed(t, cache, key, when) - for _, file := range []string{path, path + variantMetaSuffix} { - err = os.Chtimes(file, when, when) - if err != nil { - t.Fatalf("setting back the times of %s: %v", file, err) - } + err := os.Chtimes(path, when, when) + if err != nil { + t.Fatalf("setting back the times of %s: %v", path, err) } } diff --git a/internal/imgcache/stats_internal_test.go b/internal/imgcache/stats_internal_test.go index 0ff7dae..d1da15b 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imgcache/stats_internal_test.go @@ -148,43 +148,6 @@ func TestStats_LogsFailedCountQueries(t *testing.T) { } } -// TestIncrementStats_LogsFailedUpdates verifies that a failed hit or miss -// count update and a failed upstream fetch count update are each logged at -// warn. -func TestIncrementStats_LogsFailedUpdates(t *testing.T) { - t.Parallel() - - db := setupStatsTestDB(t) - - var logBuf bytes.Buffer - - cache, err := NewCache(db, CacheConfig{ - StateDir: t.TempDir(), - NegativeTTL: 5 * time.Minute, - Logger: slog.New(slog.NewJSONHandler(&logBuf, nil)), - }) - if err != nil { - t.Fatal(err) - } - - _, err = db.ExecContext(t.Context(), `DROP TABLE cache_stats`) - if err != nil { - t.Fatal(err) - } - - cache.IncrementStats(t.Context(), false, 1024) - - for _, msg := range []string{ - "failed to count cache hit or miss", - "failed to count upstream fetch", - } { - want := `"level":"WARN","msg":"` + msg + `"` - if !strings.Contains(logBuf.String(), want) { - t.Errorf("log missing %s; got %q", want, logBuf.String()) - } - } -} - // TestStats_TotalsCountSourcesAndVariants verifies that TotalItems and // TotalSizeBytes cover the stored source images and processed variants. func TestStats_TotalsCountSourcesAndVariants(t *testing.T) { @@ -340,6 +303,8 @@ func TestService_Get_CountsStats(t *testing.T) { _ = resp.Content.Close() } + waitForPendingCountsWritten(t, svc.cache) + got := readCacheStatsCounters(t, svc.cache) if got != step.want { t.Fatalf("after the %s: counters = %+v, want %+v", step.name, got, step.want) @@ -446,6 +411,8 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { _ = resp.Content.Close() } + waitForPendingCountsWritten(t, svc.cache) + got := readCacheStatsCounters(t, svc.cache) if got != tc.want { t.Errorf("counters = %+v, want %+v", got, tc.want) @@ -483,6 +450,8 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { _ = resp.Content.Close() + waitForPendingCountsWritten(t, svc.cache) + want := readCacheStatsCounters(t, svc.cache) want.hitCount++ @@ -500,6 +469,8 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { t.Fatalf("CacheStatus = %v, want %v", resp.CacheStatus, CacheHit) } + waitForPendingCountsWritten(t, svc.cache) + got := readCacheStatsCounters(t, svc.cache) if got != want { t.Errorf("counters = %+v, want %+v", got, want) diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index 9af86fd..4cfe4cd 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -1,16 +1,14 @@ package imgcache import ( - "encoding/json" "errors" "fmt" "io" - "log/slog" "os" "path/filepath" - "time" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/magic" ) // Storage constants. @@ -211,77 +209,46 @@ func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string { // VariantStorage handles key-based file storage for processed image variants. // Files are stored at: /// -// Metadata is stored at: ///.meta type VariantStorage struct { baseDir string - log *slog.Logger } -// VariantMeta contains metadata about a cached variant. -// -//nolint:tagliatelle // stored metadata format uses snake_case -type VariantMeta struct { - ContentType string `json:"content_type"` - Size int64 `json:"size"` - CreatedAt int64 `json:"created_at"` -} +// errNotAnOutputFormat is returned for a variant file whose first bytes are +// those of a format pixa reads but never writes, such as SVG. +var errNotAnOutputFormat = errors.New("variant file is not in an output format") // NewVariantStorage creates a new variant storage at the given base directory. -// A failed .meta write is logged to log. -func NewVariantStorage(baseDir string, log *slog.Logger) (*VariantStorage, error) { +func NewVariantStorage(baseDir string) (*VariantStorage, error) { err := os.MkdirAll(baseDir, StorageDirPerm) if err != nil { return nil, fmt.Errorf("failed to create variant storage directory: %w", err) } - return &VariantStorage{baseDir: baseDir, log: log}, nil + return &VariantStorage{baseDir: baseDir}, nil } -// Store writes content and metadata to storage at the given key, each -// with writeFileSynced. The .meta file is written first, so that a crash -// cannot leave the content in place without it. +// Store writes content to storage at the given key with writeFileSynced. func (s *VariantStorage) Store( - key imagerequest.VariantKey, r io.Reader, contentType string, + key imagerequest.VariantKey, r io.Reader, ) (int64, error) { data, err := io.ReadAll(r) if err != nil { return 0, fmt.Errorf("failed to read content: %w", err) } - size := int64(len(data)) path := s.keyToPath(key) - metaPath := path + ".meta" err = os.MkdirAll(filepath.Dir(path), StorageDirPerm) if err != nil { return 0, fmt.Errorf("failed to create directory: %w", err) } - meta := VariantMeta{ - ContentType: contentType, - Size: size, - CreatedAt: time.Now().UTC().Unix(), - } - - metaData, err := json.Marshal(meta) - if err != nil { - return 0, fmt.Errorf("failed to marshal metadata: %w", err) - } - - // Metadata write failure is non-fatal: the content is then served as - // application/octet-stream. - err = writeFileSynced(metaPath, metaData) - if err != nil { - s.log.Warn("failed to write variant metadata sidecar", - "path", metaPath, "error", err) - } - err = writeFileSynced(path, data) if err != nil { return 0, err } - return size, nil + return int64(len(data)), nil } // Load returns a reader for the content at the given key. @@ -300,56 +267,63 @@ func (s *VariantStorage) Load(key imagerequest.VariantKey) (io.ReadCloser, error return f, nil } -// LoadWithSize returns a reader and file size for the content at the -// given key. -func (s *VariantStorage) LoadWithSize( +// LoadWithContentType opens the content at the given key and returns a +// reader, the file size and the content type, which is read from the +// file's first bytes. A missing file is ErrNotFound. A file whose first +// bytes are not those of an output format, an empty one included, is +// closed again and the reason returned. +func (s *VariantStorage) LoadWithContentType( key imagerequest.VariantKey, -) (io.ReadCloser, int64, error) { +) (io.ReadCloser, int64, string, error) { path := s.keyToPath(key) f, err := os.Open(path) //nolint:gosec // path derived from cache key if err != nil { if os.IsNotExist(err) { - return nil, 0, ErrNotFound + return nil, 0, "", ErrNotFound } - return nil, 0, fmt.Errorf("failed to open content: %w", err) + return nil, 0, "", fmt.Errorf("failed to open content: %w", err) } stat, err := f.Stat() if err != nil { _ = f.Close() - return nil, 0, fmt.Errorf("failed to stat content: %w", err) + return nil, 0, "", fmt.Errorf("failed to stat content: %w", err) } - return f, stat.Size(), nil -} - -// LoadWithMeta returns a reader, size, and content type for the content at -// the given key. The content type is read from the .meta file, and is -// empty when that file is missing or unreadable. -func (s *VariantStorage) LoadWithMeta( - key imagerequest.VariantKey, -) (io.ReadCloser, int64, string, error) { - f, size, err := s.LoadWithSize(key) + contentType, err := outputContentType(f) if err != nil { + _ = f.Close() + return nil, 0, "", err } - var contentType string + return f, stat.Size(), contentType, nil +} - metaPath := s.keyToPath(key) + ".meta" +// outputContentType reads the first bytes of content, without moving its +// offset, and returns the content type of the output format they show. +func outputContentType(content io.ReaderAt) (string, error) { + firstBytes := make([]byte, magic.MinMagicBytes) - metaData, err := os.ReadFile(metaPath) //nolint:gosec // path derived from cache key - if err == nil { - var meta VariantMeta - if json.Unmarshal(metaData, &meta) == nil && meta.ContentType != "" { - contentType = meta.ContentType - } + _, err := content.ReadAt(firstBytes, 0) + if err != nil { + return "", fmt.Errorf("failed to read the first bytes: %w", err) } - return f, size, contentType, nil + detected, err := magic.DetectFormat(firstBytes) + if err != nil { + return "", err + } + + _, isOutputFormat := magic.MIMEToImageFormat(string(detected)) + if !isOutputFormat { + return "", fmt.Errorf("%w: %s", errNotAnOutputFormat, detected) + } + + return string(detected), nil } // Exists checks if content exists at the given key. @@ -360,8 +334,8 @@ func (s *VariantStorage) Exists(key imagerequest.VariantKey) bool { return err == nil } -// Delete removes the content at the given key together with its .meta -// sidecar file. A missing file is not an error. +// Delete removes the content at the given key. A missing file is not an +// error. func (s *VariantStorage) Delete(key imagerequest.VariantKey) error { path := s.keyToPath(key) @@ -370,13 +344,6 @@ func (s *VariantStorage) Delete(key imagerequest.VariantKey) error { return fmt.Errorf("failed to delete content: %w", err) } - metaPath := path + ".meta" - - err = os.Remove(metaPath) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete variant metadata: %w", err) - } - return nil } diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go index 98314bf..dc88cc9 100644 --- a/internal/imgcache/storage_internal_test.go +++ b/internal/imgcache/storage_internal_test.go @@ -4,10 +4,8 @@ import ( "bytes" "errors" "io" - "log/slog" "os" "path/filepath" - "strings" "testing" "sneak.berlin/go/pixa/internal/imagerequest" @@ -156,109 +154,3 @@ func TestContentStorage_KeyToPath(t *testing.T) { t.Errorf("File not at expected path %s: %v", expectedPath, err) } } - -// TestVariantStorage_StoreLogsFailedMetaWrite verifies that a .meta write -// that fails is logged at warn and the store still succeeds. -func TestVariantStorage_StoreLogsFailedMetaWrite(t *testing.T) { - t.Parallel() - - var logBuf bytes.Buffer - - storage, err := NewVariantStorage( - t.TempDir(), slog.New(slog.NewJSONHandler(&logBuf, nil))) - if err != nil { - t.Fatalf("NewVariantStorage() error = %v", err) - } - - key := imagerequest.CacheKey(&imagerequest.Request{ - Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, - }) - - // A directory where the .meta file goes makes the .meta write fail. - err = os.MkdirAll(storage.keyToPath(key)+".meta", StorageDirPerm) - if err != nil { - t.Fatalf("failed to create directory: %v", err) - } - - _, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp") - if err != nil { - t.Fatalf("Store() error = %v, want nil", err) - } - - want := `"level":"WARN","msg":"failed to write variant metadata sidecar"` - if !strings.Contains(logBuf.String(), want) { - t.Errorf("log missing %s; got %q", want, logBuf.String()) - } -} - -// storeTestVariant stores one variant, with its .meta file, in a new -// VariantStorage and returns the storage and the variant's key. -func storeTestVariant(t *testing.T) (*VariantStorage, imagerequest.VariantKey) { - t.Helper() - - storage, err := NewVariantStorage(t.TempDir(), slog.New(slog.DiscardHandler)) - if err != nil { - t.Fatalf("NewVariantStorage() error = %v", err) - } - - key := imagerequest.CacheKey(&imagerequest.Request{ - Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, - }) - - _, err = storage.Store(key, bytes.NewReader([]byte("variant data")), "image/webp") - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - return storage, key -} - -// TestVariantStorage_DeleteRemovesMeta verifies that Delete removes the -// variant's .meta file along with the variant file. -func TestVariantStorage_DeleteRemovesMeta(t *testing.T) { - t.Parallel() - - storage, key := storeTestVariant(t) - metaPath := storage.keyToPath(key) + ".meta" - - _, err := os.Stat(metaPath) - if err != nil { - t.Fatalf("Store() wrote no .meta file: %v", err) - } - - err = storage.Delete(key) - if err != nil { - t.Fatalf("Delete() error = %v", err) - } - - if storage.Exists(key) { - t.Error("Exists() = true after delete, want false") - } - - _, err = os.Stat(metaPath) - if !os.IsNotExist(err) { - t.Errorf(".meta file left after Delete() (stat err=%v)", err) - } -} - -// TestVariantStorage_DeleteWithoutMeta verifies that Delete succeeds for a -// variant whose .meta file is missing. -func TestVariantStorage_DeleteWithoutMeta(t *testing.T) { - t.Parallel() - - storage, key := storeTestVariant(t) - - err := os.Remove(storage.keyToPath(key) + ".meta") - if err != nil { - t.Fatalf("removing .meta file: %v", err) - } - - err = storage.Delete(key) - if err != nil { - t.Fatalf("Delete() error = %v, want nil", err) - } - - if storage.Exists(key) { - t.Error("Exists() = true after delete, want false") - } -} diff --git a/script/bench b/script/bench new file mode 100755 index 0000000..4e015e5 --- /dev/null +++ b/script/bench @@ -0,0 +1,19 @@ +#!/bin/sh +# script/bench: run the Go benchmarks of internal/handlers, which serve +# cache hits in parallel through the image handler and report the hits +# served per second. A benchmark, not a check: script/check does not run +# it. It runs Go on the host, as script/loadtest does, and compiles pixa +# there with CGO, so it needs the C compiler and libraries that +# `script/bootstrap --cgo` installs. -run '^$' runs no test, only the +# benchmarks. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + CGO_ENABLED=1 go test -run '^$' -bench . ./internal/handlers +} + +main "$@" From 6b19e4c96c09488986075836cddea17492646675 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 09:29:26 +0100 Subject: [PATCH 100/131] Declare only the tables and columns pixa writes and reads (closes #310) Removes output_content and request_cache, which nothing writes, and every column that was written but never read: the host, path, query, content_type and fetched_at of source_content, the content_type and created_at of variant_content, and the status_code, error_message and fetched_at of negative_cache. StoreSource no longer takes the fetch result and LookupSource returns only the source key. StoreVariant no longer takes a content type and StoreNegative no longer takes a status code or message, so extractStatusCode and the status constants, which only fed status_code, are gone. Reconciliation adopts a variant file without opening it, as its row no longer records a content type. Tests that set or checked a removed column follow. Model: opus-5-5 --- TODO.md | 9 +++ .../concurrent_writes_internal_test.go | 5 +- internal/database/database_internal_test.go | 5 +- internal/db/migrations/001_schema.sql | 45 +------------ internal/imgcache/cache.go | 66 +++++++------------ internal/imgcache/cache_hit_internal_test.go | 7 +- internal/imgcache/cache_internal_test.go | 31 +-------- internal/imgcache/cachesize_internal_test.go | 5 +- internal/imgcache/eviction.go | 24 ++----- internal/imgcache/eviction_internal_test.go | 37 +++-------- ...max_concurrent_processing_internal_test.go | 2 +- .../imgcache/negative_cache_internal_test.go | 23 ++++--- .../partial_cache_files_internal_test.go | 2 +- .../imgcache/pendingcounts_internal_test.go | 2 +- internal/imgcache/service.go | 43 ++++-------- internal/imgcache/service_internal_test.go | 38 ++--------- .../imgcache/source_content_internal_test.go | 15 ++--- internal/imgcache/testutil_internal_test.go | 10 ++- internal/server/image_proxy_internal_test.go | 20 ++---- 19 files changed, 107 insertions(+), 282 deletions(-) diff --git a/TODO.md b/TODO.md index ad36d3e..97da3b8 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-21 the schema file declares only the tables and columns pixa writes + and reads (closes #310): `output_content` and `request_cache` are gone, and so + are the columns nothing read: the host, path, query, `content_type` and + `fetched_at` of `source_content`, the `content_type` and `created_at` of + `variant_content`, and the `status_code`, `error_message` and `fetched_at` of + `negative_cache`. `StoreSource` no longer takes the fetch result, + `LookupSource` returns only the source key, `StoreVariant` no longer takes a + content type, `StoreNegative` no longer takes a status code or message, and + reconciliation adopts a variant file without reading its first bytes. - 2026-10-21 a cache hit makes no database call (closes #309): `Service.Get` opens the variant file with `Cache.GetVariant` and nothing else, with no existence check first. The content type comes from the file's first bytes; a diff --git a/internal/database/concurrent_writes_internal_test.go b/internal/database/concurrent_writes_internal_test.go index 9644aea..c067853 100644 --- a/internal/database/concurrent_writes_internal_test.go +++ b/internal/database/concurrent_writes_internal_test.go @@ -131,14 +131,13 @@ func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error { } _, err = db.ExecContext(ctx, `INSERT INTO source_content - (source_key, source_host, source_path, content_type, size_bytes) - VALUES (?, 'example.com', '/image.png', 'image/png', 1)`, key) + (source_key, size_bytes) VALUES (?, 1)`, key) if err != nil { return fmt.Errorf("storing a source: %w", err) } _, err = db.ExecContext(ctx, `INSERT INTO variant_content - (cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key) + (cache_key, size_bytes) VALUES (?, 1)`, key) if err != nil { return fmt.Errorf("recording a transformed image: %w", err) } diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go index c19511a..5cc6186 100644 --- a/internal/database/database_internal_test.go +++ b/internal/database/database_internal_test.go @@ -159,10 +159,7 @@ func TestApplyMigrations_CreatesSchemaAndTables(t *testing.T) { } // Verify that the application tables created by 001.sql exist. - tables := []string{ - "source_content", "output_content", - "request_cache", "negative_cache", "cache_stats", - } + tables := []string{"source_content", "negative_cache", "cache_stats"} for _, table := range tables { var count int diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index c046293..7f6f472 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -6,12 +6,7 @@ -- is the SHA-256 of the source's host, path and query CREATE TABLE IF NOT EXISTS source_content ( source_key TEXT PRIMARY KEY, - source_host TEXT NOT NULL, - source_path TEXT NOT NULL, - source_query TEXT NOT NULL DEFAULT '', - content_type TEXT NOT NULL, size_bytes INTEGER NOT NULL, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed @@ -27,8 +22,6 @@ CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed CREATE TABLE IF NOT EXISTS variant_content ( cache_key TEXT PRIMARY KEY, size_bytes INTEGER NOT NULL, - content_type TEXT NOT NULL, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed @@ -102,46 +95,14 @@ BEGIN WHERE id = 1; END; --- Output/transformed content blobs --- Not written: transformed images are stored in cache/variants and --- tracked in variant_content above. -CREATE TABLE IF NOT EXISTS output_content ( - content_hash TEXT PRIMARY KEY, - content_type TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP -); - --- Request cache - maps full request params to output content -CREATE TABLE IF NOT EXISTS request_cache ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - cache_key TEXT NOT NULL UNIQUE, - source_metadata_id INTEGER NOT NULL, - output_hash TEXT NOT NULL, - width INTEGER NOT NULL, - height INTEGER NOT NULL, - format TEXT NOT NULL, - quality INTEGER NOT NULL DEFAULT 85, - fit_mode TEXT NOT NULL DEFAULT 'cover', - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, - access_count INTEGER NOT NULL DEFAULT 1, - FOREIGN KEY (source_metadata_id) REFERENCES source_metadata(id), - FOREIGN KEY (output_hash) REFERENCES output_content(content_hash) -); -CREATE INDEX IF NOT EXISTS idx_request_cache_key ON request_cache(cache_key); -CREATE INDEX IF NOT EXISTS idx_request_cache_source ON request_cache(source_metadata_id); -CREATE INDEX IF NOT EXISTS idx_request_cache_output ON request_cache(output_hash); -CREATE INDEX IF NOT EXISTS idx_request_cache_fetched ON request_cache(fetched_at); - --- Negative cache for failed fetches (404s, timeouts, etc.) +-- Negative cache: source URLs whose fetch failed with an upstream error +-- status, or whose first bytes did not match their Content-Type. pixa does +-- not fetch such a URL again until its row expires. CREATE TABLE IF NOT EXISTS negative_cache ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_host TEXT NOT NULL, source_path TEXT NOT NULL, source_query TEXT NOT NULL DEFAULT '', - status_code INTEGER NOT NULL, - error_message TEXT, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, expires_at DATETIME NOT NULL, UNIQUE(source_host, source_path, source_query) ); diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 3cb1f2d..b03b586 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -12,7 +12,6 @@ import ( "sync/atomic" "time" - "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -212,7 +211,6 @@ func (c *Cache) StoreSource( ctx context.Context, req *imagerequest.Request, content io.Reader, - result *httpfetcher.FetchResult, ) (imagerequest.SourceKey, error) { if c.disabled { return "", nil @@ -231,17 +229,12 @@ func (c *Cache) StoreSource( } _, err = c.db.ExecContext(ctx, ` - INSERT INTO source_content - (source_key, source_host, source_path, source_query, - content_type, size_bytes) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO source_content (source_key, size_bytes) + VALUES (?, ?) ON CONFLICT(source_key) DO UPDATE SET - content_type = excluded.content_type, size_bytes = excluded.size_bytes, - fetched_at = CURRENT_TIMESTAMP, last_accessed_at = CURRENT_TIMESTAMP - `, string(key), req.Source.Host, req.Source.Path, req.Source.Query, - result.ContentType, size) + `, string(key), size) if err != nil { deleteErr := c.srcContent.Delete(key) if deleteErr != nil { @@ -262,8 +255,7 @@ func (c *Cache) StoreSource( // accounting insert is best-effort (the startup reconciliation pass // adopts any variant file that misses its accounting row). func (c *Cache) StoreVariant( - ctx context.Context, cacheKey imagerequest.VariantKey, - content io.Reader, contentType string, + ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader, ) error { if c.disabled { return nil @@ -275,13 +267,12 @@ func (c *Cache) StoreVariant( } _, err = c.db.ExecContext(ctx, ` - INSERT INTO variant_content (cache_key, size_bytes, content_type) - VALUES (?, ?, ?) + INSERT INTO variant_content (cache_key, size_bytes) + VALUES (?, ?) ON CONFLICT(cache_key) DO UPDATE SET size_bytes = excluded.size_bytes, - content_type = excluded.content_type, last_accessed_at = CURRENT_TIMESTAMP - `, string(cacheKey), size, contentType) + `, string(cacheKey), size) if err != nil { c.log.Warn("failed to record variant in size accounting", "cache_key", cacheKey, "error", err) @@ -293,60 +284,53 @@ func (c *Cache) StoreVariant( } // LookupSource checks if we have cached source content for a request. -// Returns the source key and content type if found, or empty values -// if not; a row whose file is missing is not found. A source found has -// its access time handed to the background writer without waiting (see +// Returns the source key if found, or an empty key if not; a row whose +// file is missing is not found. A source found has its access time +// handed to the background writer without waiting (see // touchSourceContent); a disabled cache always reports no cached source. func (c *Cache) LookupSource( ctx context.Context, req *imagerequest.Request, -) (imagerequest.SourceKey, string, error) { +) (imagerequest.SourceKey, error) { if c.disabled { - return "", "", nil + return "", nil } key := req.Source.Key() - var contentType string + var rowExists bool err := c.db.QueryRowContext(ctx, ` - SELECT content_type FROM source_content WHERE source_key = ? - `, string(key)).Scan(&contentType) - - if errors.Is(err, sql.ErrNoRows) { - return "", "", nil + SELECT EXISTS(SELECT 1 FROM source_content WHERE source_key = ?) + `, string(key)).Scan(&rowExists) + if err != nil { + return "", fmt.Errorf("failed to lookup source: %w", err) } - if err != nil { - return "", "", fmt.Errorf("failed to lookup source: %w", err) + if !rowExists { + return "", nil } // Verify the content file exists if !c.srcContent.Exists(key) { - return "", "", nil + return "", nil } c.touchSourceContent(key) - return key, contentType, nil + return key, nil } // StoreNegative stores a negative cache entry for a failed fetch. -func (c *Cache) StoreNegative( - ctx context.Context, req *imagerequest.Request, statusCode int, errMsg string, -) error { +func (c *Cache) StoreNegative(ctx context.Context, req *imagerequest.Request) error { expiresAt := time.Now().UTC().Add(c.config.NegativeTTL) _, err := c.db.ExecContext(ctx, ` INSERT INTO negative_cache - (source_host, source_path, source_query, status_code, - error_message, expires_at) - VALUES (?, ?, ?, ?, ?, ?) + (source_host, source_path, source_query, expires_at) + VALUES (?, ?, ?, ?) ON CONFLICT(source_host, source_path, source_query) DO UPDATE SET - status_code = excluded.status_code, - error_message = excluded.error_message, - fetched_at = CURRENT_TIMESTAMP, expires_at = excluded.expires_at - `, req.Source.Host, req.Source.Path, req.Source.Query, statusCode, errMsg, expiresAt) + `, req.Source.Host, req.Source.Path, req.Source.Query, expiresAt) if err != nil { return fmt.Errorf("failed to insert negative cache: %w", err) } diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index b24ea08..70135b8 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -16,9 +16,8 @@ import ( ) // TestGetVariant_ContentTypeFromFirstBytes stores an image in each output -// format as a variant, recorded with the content type text/plain, and checks -// that GetVariant serves it whole with the content type of its format, read -// from the file. +// format as a variant and checks that GetVariant serves it whole with the +// content type of its format, read from the file. func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { t.Parallel() @@ -48,7 +47,7 @@ func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { key := imagerequest.VariantKey("aabbccdd10" + string(format)) - err = cache.StoreVariant(t.Context(), key, bytes.NewReader(image), "text/plain") + err = cache.StoreVariant(t.Context(), key, bytes.NewReader(image)) if err != nil { t.Fatalf("StoreVariant(%s) error = %v", format, err) } diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index a05f049..6431300 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -22,41 +22,14 @@ func setupTestDB(t *testing.T) *sql.DB { schema := ` CREATE TABLE source_content ( source_key TEXT PRIMARY KEY, - source_host TEXT NOT NULL, - source_path TEXT NOT NULL, - source_query TEXT NOT NULL DEFAULT '', - content_type TEXT NOT NULL, size_bytes INTEGER NOT NULL, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); - CREATE TABLE output_content ( - content_hash TEXT PRIMARY KEY, - content_type TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE TABLE request_cache ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - cache_key TEXT NOT NULL UNIQUE, - source_metadata_id INTEGER NOT NULL, - output_hash TEXT NOT NULL, - width INTEGER NOT NULL, - height INTEGER NOT NULL, - format TEXT NOT NULL, - quality INTEGER NOT NULL DEFAULT 85, - fit_mode TEXT NOT NULL DEFAULT 'cover', - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, - access_count INTEGER NOT NULL DEFAULT 1 - ); CREATE TABLE negative_cache ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_host TEXT NOT NULL, source_path TEXT NOT NULL, source_query TEXT NOT NULL DEFAULT '', - status_code INTEGER NOT NULL, - error_message TEXT, - fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, expires_at DATETIME NOT NULL, UNIQUE(source_host, source_path, source_query) ); @@ -138,8 +111,8 @@ func TestCache_CleanExpired(t *testing.T) { // Insert expired negative cache entry directly _, err := db.ExecContext(ctx, ` INSERT INTO negative_cache - (source_host, source_path, source_query, status_code, expires_at) - VALUES ('example.com', '/old.jpg', '', 404, datetime('now', '-1 hour')) + (source_host, source_path, source_query, expires_at) + VALUES ('example.com', '/old.jpg', '', datetime('now', '-1 hour')) `) if err != nil { t.Fatalf("failed to insert test data: %v", err) diff --git a/internal/imgcache/cachesize_internal_test.go b/internal/imgcache/cachesize_internal_test.go index 2416366..7cca989 100644 --- a/internal/imgcache/cachesize_internal_test.go +++ b/internal/imgcache/cachesize_internal_test.go @@ -39,9 +39,8 @@ func TestComputeDefaultMaxBytesCountsWhatTheCacheHolds(t *testing.T) { // The cache now holds those 3 GiB, which leaves 1 GiB free. _, err = cache.db.ExecContext(t.Context(), - `INSERT INTO variant_content (cache_key, size_bytes, content_type) - VALUES (?, ?, ?)`, - string(testVariantKeyOne), 3221225472, testContentTypeWebP, + `INSERT INTO variant_content (cache_key, size_bytes) VALUES (?, ?)`, + string(testVariantKeyOne), 3221225472, ) if err != nil { t.Fatalf("failed to insert variant accounting row: %v", err) diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 509e1ac..6f3cc9f 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -51,11 +51,6 @@ const sqliteTimestampLayout = "2006-01-02 15:04:05" // writes (".tmp-*" patterns in the storage layer). const tempFilePrefix = ".tmp-" -// fallbackContentType is the content type reconciliation records for a -// variant file it adopts whose first bytes are not those of an output -// format. Such a file is never served. -const fallbackContentType = "application/octet-stream" - // UsageBytes returns the total number of bytes of cache content // tracked in the database (source content blobs plus processed // variants). It reads the total the database keeps up to date as rows @@ -590,25 +585,16 @@ func (c *Cache) reconcileVariantFile( } // adoptVariantFile inserts an accounting row for a variant file that -// has none, using the file's size and modification time, and the content -// type its first bytes show, as a hit reads it. +// has none, using the file's size and modification time. func (c *Cache) adoptVariantFile( ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey, ) error { modTime := info.ModTime().UTC().Format(sqliteTimestampLayout) - contentType := fallbackContentType - content, _, detectedType, err := c.variants.LoadWithContentType(cacheKey) - if err == nil { - _ = content.Close() - contentType = detectedType - } - - _, err = c.db.ExecContext(ctx, ` - INSERT INTO variant_content - (cache_key, size_bytes, content_type, created_at, last_accessed_at) - VALUES (?, ?, ?, ?, ?) - `, string(cacheKey), info.Size(), contentType, modTime, modTime) + _, err := c.db.ExecContext(ctx, ` + INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at) + VALUES (?, ?, ?) + `, string(cacheKey), info.Size(), modTime) if err != nil { return fmt.Errorf("failed to adopt variant file into accounting: %w", err) } diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 5cbc8a0..a02e6fc 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -16,7 +16,6 @@ import ( "modernc.org/sqlite" "sneak.berlin/go/pixa/internal/database" - "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -102,17 +101,8 @@ func storeEvictionTestSource( }, } - result := &httpfetcher.FetchResult{ - StatusCode: 200, - ContentType: testContentTypeJPEG, - ContentLength: int64(len(content)), - Headers: map[string][]string{ - testHeaderContentType: {testContentTypeJPEG}, - }, - } - key, err := cache.StoreSource( - context.Background(), req, bytes.NewReader(content), result, + context.Background(), req, bytes.NewReader(content), ) if err != nil { t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err) @@ -128,7 +118,7 @@ func storeEvictionTestVariant( ) { t.Helper() - err := cache.StoreVariant(t.Context(), key, bytes.NewReader(content), "image/webp") + err := cache.StoreVariant(t.Context(), key, bytes.NewReader(content)) if err != nil { t.Fatalf("StoreVariant(%s) failed: %v", key, err) } @@ -437,20 +427,13 @@ func assertDisabledCacheWritesAreNoOps( ctx := t.Context() err := cache.StoreVariant( - ctx, imagerequest.CacheKey(req), bytes.NewReader([]byte("data")), "image/webp", + ctx, imagerequest.CacheKey(req), bytes.NewReader([]byte("data")), ) if err != nil { t.Fatalf("StoreVariant on disabled cache must be a no-op, got error: %v", err) } - result := &httpfetcher.FetchResult{ - StatusCode: 200, - ContentType: testContentTypeJPEG, - ContentLength: 4, - Headers: map[string][]string{}, - } - - key, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data")), result) + key, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data"))) if err != nil { t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err) } @@ -469,14 +452,13 @@ func assertDisabledCacheReadsAlwaysMiss( ctx := t.Context() - key, srcType, err := cache.LookupSource(ctx, req) + key, err := cache.LookupSource(ctx, req) if err != nil { t.Fatalf("LookupSource on disabled cache failed: %v", err) } - if key != "" || srcType != "" { - t.Errorf("LookupSource on disabled cache = (%q, %q), want empty", - key, srcType) + if key != "" { + t.Errorf("LookupSource on disabled cache = %q, want empty", key) } } @@ -667,9 +649,8 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { // An accounting row whose file is missing must be dropped. _, err = cache.db.ExecContext(t.Context(), - `INSERT INTO variant_content (cache_key, size_bytes, content_type) - VALUES (?, ?, ?)`, - "deadbeef0001", 700, "image/webp", + `INSERT INTO variant_content (cache_key, size_bytes) VALUES (?, ?)`, + "deadbeef0001", 700, ) if err != nil { t.Fatalf("failed to insert stale variant accounting row: %v", err) diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imgcache/max_concurrent_processing_internal_test.go index dec7941..2ff8372 100644 --- a/internal/imgcache/max_concurrent_processing_internal_test.go +++ b/internal/imgcache/max_concurrent_processing_internal_test.go @@ -77,7 +77,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { _ = resp.Content.Close() - sourceKey, _, err := svc.cache.LookupSource(t.Context(), + sourceKey, err := svc.cache.LookupSource(t.Context(), widthOnlyRequest(fixtures, 50)) if err != nil || sourceKey == "" { t.Fatalf("LookupSource() = %q, %v; want the cached source", diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index c4c4269..dbfc69f 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -43,7 +43,7 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { } // Store a negative entry - err = cache.StoreNegative(ctx, req, 404, "not found") + err = cache.StoreNegative(ctx, req) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestNegativeCache_Expired(t *testing.T) { } // Store a negative entry with very short TTL - err = cache.StoreNegative(ctx, req, 500, "server error") + err = cache.StoreNegative(ctx, req) if err != nil { t.Fatal(err) } @@ -120,8 +120,7 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { _ = resp.Content.Close() err = svc.cache.StoreNegative(t.Context(), - photoVariant(fixtures, 85, imagerequest.FitCover), - httpStatusBadGateway, "upstream answered 503") + photoVariant(fixtures, 85, imagerequest.FitCover)) if err != nil { t.Fatalf("StoreNegative() error = %v", err) } @@ -147,7 +146,7 @@ func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { _ = resp.Content.Close() - sourceKey, _, err := svc.cache.LookupSource(t.Context(), + sourceKey, err := svc.cache.LookupSource(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) if err != nil || sourceKey == "" { t.Fatalf("LookupSource() = %q, %v, want the cached source", sourceKey, err) @@ -201,7 +200,7 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { Source: imagerequest.Source{Host: testHostExample, Path: "/gone.jpg"}, } - err = cache.StoreNegative(t.Context(), expired, 404, "not found") + err = cache.StoreNegative(t.Context(), expired) if err != nil { t.Fatalf("StoreNegative failed: %v", err) } @@ -210,8 +209,8 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { _, err = cache.db.ExecContext(t.Context(), ` INSERT INTO negative_cache - (source_host, source_path, status_code, expires_at) - VALUES (?, ?, 404, datetime('now', '+1 hour')) + (source_host, source_path, expires_at) + VALUES (?, ?, datetime('now', '+1 hour')) `, testHostExample, notExpiredPath) if err != nil { t.Fatalf("failed to insert a row that has not expired: %v", err) @@ -279,8 +278,8 @@ func TestCleanExpiredDeletesInBatches(t *testing.T) { _, err = cache.db.ExecContext(ctx, ` INSERT INTO negative_cache - (source_host, source_path, status_code, expires_at) - VALUES (?, ?, 404, datetime('now', '+1 hour')) + (source_host, source_path, expires_at) + VALUES (?, ?, datetime('now', '+1 hour')) `, testHostExample, notExpiredPath) if err != nil { t.Fatalf("failed to insert a row that has not expired: %v", err) @@ -378,8 +377,8 @@ func insertExpiredNegativeCacheRows(t *testing.T, cache *Cache, count int) { for i := range count { _, err := cache.db.ExecContext(t.Context(), ` INSERT INTO negative_cache - (source_host, source_path, status_code, expires_at) - VALUES (?, ?, 404, datetime('now', '-1 hour')) + (source_host, source_path, expires_at) + VALUES (?, ?, datetime('now', '-1 hour')) `, testHostExample, "/expired/"+strconv.Itoa(i)+".jpg") if err != nil { t.Fatalf("failed to insert an expired row: %v", err) diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index c8ff377..fa4b0ef 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -65,7 +65,7 @@ func cachedSourcePath( ) string { t.Helper() - sourceKey, _, err := svc.cache.LookupSource(t.Context(), req) + sourceKey, err := svc.cache.LookupSource(t.Context(), req) if err != nil || sourceKey == "" { t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err) } diff --git a/internal/imgcache/pendingcounts_internal_test.go b/internal/imgcache/pendingcounts_internal_test.go index aaa63cd..81a3b15 100644 --- a/internal/imgcache/pendingcounts_internal_test.go +++ b/internal/imgcache/pendingcounts_internal_test.go @@ -364,7 +364,7 @@ func TestLookupSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { ` anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat) - found, _, err := cache.LookupSource(t.Context(), &imagerequest.Request{ + found, err := cache.LookupSource(t.Context(), &imagerequest.Request{ Source: imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}, }) if err != nil || found != key { diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 37bf41a..46a3653 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -397,7 +397,7 @@ func (s *Service) processFromSourceOrFetch( cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { // Check if we have cached source content - sourceKey, _, err := s.cache.LookupSource(ctx, req) + sourceKey, err := s.cache.LookupSource(ctx, req) if err != nil { s.log.Warn("source lookup failed", "error", err) } @@ -458,7 +458,7 @@ func (s *Service) fetchAndProcess( if err != nil { // Store negative cache for certain errors if isNegativeCacheable(err) { - s.storeNegative(ctx, req, extractStatusCode(err), err) + s.storeNegative(ctx, req) } return nil, fmt.Errorf("upstream fetch failed: %w", err) @@ -512,13 +512,13 @@ func (s *Service) fetchAndProcess( s.log.Warn("source refused", "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, "path", req.Source.Path, "content_type", fetchResult.ContentType, "error", err) - s.storeNegative(ctx, req, httpStatusUnsupportedMediaType, err) + s.storeNegative(ctx, req) return nil, fmt.Errorf("content validation failed: %w", err) } // Store source content - _, err = s.cache.StoreSource(ctx, req, bytes.NewReader(sourceData), fetchResult) + _, err = s.cache.StoreSource(ctx, req, bytes.NewReader(sourceData)) if err != nil { s.log.Warn("failed to store source content", "error", err) // Continue even if caching fails @@ -530,15 +530,13 @@ func (s *Service) fetchAndProcess( } // storeNegative stores a negative cache entry for req's source, whose fetch -// failed with err, so the source is not fetched again until the entry -// expires. A failure to store the entry is only logged. -func (s *Service) storeNegative( - ctx context.Context, req *imagerequest.Request, statusCode int, err error, -) { - storeErr := s.cache.StoreNegative(ctx, req, statusCode, err.Error()) - if storeErr != nil { +// failed, so the source is not fetched again until the entry expires. A +// failure to store the entry is only logged. +func (s *Service) storeNegative(ctx context.Context, req *imagerequest.Request) { + err := s.cache.StoreNegative(ctx, req) + if err != nil { s.log.Warn("failed to store negative cache entry", - "host", req.Source.Host, "path", req.Source.Path, "error", storeErr) + "host", req.Source.Host, "path", req.Source.Path, "error", err) } } @@ -614,9 +612,7 @@ func (s *Service) processAndStore( ) // Store variant to cache - err = s.cache.StoreVariant( - ctx, cacheKey, bytes.NewReader(processedData), processResult.ContentType, - ) + err = s.cache.StoreVariant(ctx, cacheKey, bytes.NewReader(processedData)) if err != nil { s.log.Warn("failed to store variant", "error", err) // Continue even if caching fails @@ -629,28 +625,11 @@ func (s *Service) processAndStore( }, nil } -// HTTP status codes for error responses. -const ( - httpStatusBadGateway = 502 - httpStatusInternalError = 500 - httpStatusUnsupportedMediaType = 415 -) - // isNegativeCacheable returns true if the error should be cached. func isNegativeCacheable(err error) bool { return errors.Is(err, httpfetcher.ErrUpstreamError) } -// extractStatusCode extracts HTTP status code from error message. -func extractStatusCode(err error) int { - // Default to 502 Bad Gateway for upstream errors - if errors.Is(err, httpfetcher.ErrUpstreamError) { - return httpStatusBadGateway - } - - return httpStatusInternalError -} - // etagHashLength is the number of hash characters to use for ETags. const etagHashLength = 16 diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index 2183d30..cab3f2a 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -549,17 +549,15 @@ func TestService_Get_Caching(t *testing.T) { // TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and // a variant of it and sets back two days every time pixa stored for them. -// For the source: fetched_at and last_accessed_at in source_content, and the -// access and modification times of its file. For the variant: created_at and -// last_accessed_at in variant_content, and the access and modification times -// of its file. It then checks that the variant is still served from the -// cache and that a new variant is made from the cached source without -// fetching it again. +// For the source: last_accessed_at in source_content, and the access and +// modification times of its file. For the variant: last_accessed_at in +// variant_content, and the access and modification times of its file. It +// then checks that the variant is still served from the cache and that a new +// variant is made from the cached source without fetching it again. // // Left at the present: the files' status change and birth times, which a // program cannot set, and the times of the directories holding the files, -// which other entries share. The request_cache and output_content tables are -// never written. +// which other entries share. func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { t.Parallel() @@ -613,11 +611,6 @@ func backdateSource( t.Helper() key := req.Source.Key() - dbTime := when.UTC().Format(sqliteTimestampFormat) - - updateOneRow(t, cache, - `UPDATE source_content SET fetched_at = ? WHERE source_key = ?`, - dbTime, string(key)) setSourceLastAccessed(t, cache, key, when) path := cache.srcContent.keyToPath(key) @@ -639,9 +632,6 @@ func backdateVariant( key := imagerequest.CacheKey(req) path := cache.variants.keyToPath(key) - updateOneRow(t, cache, - `UPDATE variant_content SET created_at = ? WHERE cache_key = ?`, - when.UTC().Format(sqliteTimestampFormat), string(key)) setVariantLastAccessed(t, cache, key, when) err := os.Chtimes(path, when, when) @@ -650,22 +640,6 @@ func backdateVariant( } } -// updateOneRow runs the UPDATE statement query and fails the test unless it -// changed exactly one row. -func updateOneRow(t *testing.T, cache *Cache, query string, args ...any) { - t.Helper() - - result, err := cache.db.ExecContext(t.Context(), query, args...) - if err != nil { - t.Fatalf("%s: %v", query, err) - } - - rows, err := result.RowsAffected() - if err != nil || rows != 1 { - t.Fatalf("%s: rows = %d, error = %v, want 1 row", query, rows, err) - } -} - func TestService_Get_DifferentSizes(t *testing.T) { t.Parallel() diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go index 8d7be09..595b7c6 100644 --- a/internal/imgcache/source_content_internal_test.go +++ b/internal/imgcache/source_content_internal_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "testing" - "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -32,10 +31,8 @@ func TestStoreSource_StoresIdenticalBytesOfTwoURLsTwice(t *testing.T) { } for sourcePath, key := range keys { - rows := countRows(t, cache, ` - SELECT COUNT(*) FROM source_content - WHERE source_key = ? AND source_host = ? AND source_path = ? - `, string(key), testHostCDN, sourcePath) + rows := countRows(t, cache, + `SELECT COUNT(*) FROM source_content WHERE source_key = ?`, string(key)) if rows != 1 { t.Errorf("%s has %d source_content rows, want 1", sourcePath, rows) } @@ -87,10 +84,10 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { t.Errorf("the evicted source still has %d source_content rows", n) } - kept, contentType, err := cache.LookupSource(t.Context(), &imagerequest.Request{ + kept, err := cache.LookupSource(t.Context(), &imagerequest.Request{ Source: imagerequest.Source{Host: testHostCDN, Path: "/kept.jpg"}, }) - t.Logf("LookupSource() of the kept source = %q, %q, error %v", kept, contentType, err) + t.Logf("LookupSource() of the kept source = %q, error %v", kept, err) if err != nil || kept == "" { t.Fatalf("LookupSource() of the kept source = %q, %v; want it cached", kept, err) @@ -130,10 +127,8 @@ func TestStoreSource_FailedRowWriteLeavesNoFile(t *testing.T) { req := &imagerequest.Request{ Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, } - result := &httpfetcher.FetchResult{ContentType: testContentTypeJPEG} - key, err := cache.StoreSource(t.Context(), req, - bytes.NewReader([]byte("source bytes")), result) + bytes.NewReader([]byte("source bytes"))) t.Logf("StoreSource() = %q, error = %v", key, err) if err == nil { diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index 326537f..e813080 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -20,12 +20,10 @@ import ( // Shared test data literals, extracted as constants for goconst. const ( - testHostCDN = "cdn.example.com" - testHostExample = "example.com" - testPathCat = "/photos/cat.jpg" - testContentTypeJPEG = "image/jpeg" - testContentTypeWebP = "image/webp" - testHeaderContentType = "Content-Type" + testHostCDN = "cdn.example.com" + testHostExample = "example.com" + testPathCat = "/photos/cat.jpg" + testContentTypeJPEG = "image/jpeg" ) // TestFixtures contains paths to test files in the mock filesystem. diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go index cf0f4b5..2a4c25e 100644 --- a/internal/server/image_proxy_internal_test.go +++ b/internal/server/image_proxy_internal_test.go @@ -113,7 +113,7 @@ func TestImageProxyFlow(t *testing.T) { } checkSourceCached(t, db, stateDir, source) - checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType) + checkVariantCached(t, db, stateDir, first.Body.Bytes()) }) } } @@ -276,27 +276,19 @@ func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) } // checkVariantCached checks that the converted image served is recorded in -// variant_content with contentType, and stored under its cache key in -// cache/variants. -func checkVariantCached( - t *testing.T, db *sql.DB, stateDir string, served []byte, contentType string, -) { +// variant_content, and stored under its cache key in cache/variants. +func checkVariantCached(t *testing.T, db *sql.DB, stateDir string, served []byte) { t.Helper() - var cacheKey, storedType string + var cacheKey string err := db.QueryRowContext(t.Context(), - "SELECT cache_key, content_type FROM variant_content", - ).Scan(&cacheKey, &storedType) + "SELECT cache_key FROM variant_content", + ).Scan(&cacheKey) if err != nil { t.Fatalf("variant_content row: %v", err) } - if storedType != contentType { - t.Errorf("variant_content content_type = %q, want %q", - storedType, contentType) - } - checkFile(t, filepath.Join(stateDir, "cache", "variants", cacheKey[0:2], cacheKey[2:4], cacheKey), served) } From b49d0068df8396b8d5ce607892c64abed20c1e4f Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 10:20:42 +0100 Subject: [PATCH 101/131] Count hits, misses, fetches and transforms as Prometheus counters (closes #311) The five counts the cache_stats table held are now Prometheus counters in memory, as fields of the cache. NewCache makes them and registers them on the registry it is given. pixad gives it Prometheus' default registry, which /metrics serves; each test gives it a new one. The Increment methods add to their cache's counters, so counting makes no database call. cache_stats, Cache.Stats, Service.Stats, CacheStats and the pending counts are gone, with their tests. The background goroutine now writes only access times, and its names say so: StartPendingAccessTimeWrites, StopPendingAccessTimeWrites and pendingaccesstimes.go. The two stop tests whose write under way was a count now use access times. Model: opus-5-5 --- README.md | 65 ++- TODO.md | 12 + internal/app/app.go | 3 + .../concurrent_writes_internal_test.go | 25 +- internal/database/database_internal_test.go | 2 +- internal/db/migrations/001_schema.sql | 12 - internal/handlers/cache_hit_internal_test.go | 8 +- .../handlers/cache_max_bytes_internal_test.go | 6 +- internal/handlers/fetcher_internal_test.go | 4 +- .../handlers/first_bytes_internal_test.go | 3 +- internal/handlers/handlers.go | 26 +- internal/handlers/handlers_internal_test.go | 3 +- .../image_cache_control_internal_test.go | 3 +- .../handlers/image_errors_internal_test.go | 3 +- .../referer_blocklist_internal_test.go | 3 +- internal/imgcache/cache.go | 145 ++---- internal/imgcache/cache_hit_internal_test.go | 36 +- internal/imgcache/cache_internal_test.go | 57 +-- internal/imgcache/cachesize_internal_test.go | 6 +- .../concurrent_misses_internal_test.go | 6 +- internal/imgcache/counters.go | 45 ++ internal/imgcache/counters_internal_test.go | 84 ++++ internal/imgcache/eviction_internal_test.go | 3 +- internal/imgcache/imgcache.go | 19 - .../imgcache/negative_cache_internal_test.go | 16 +- .../partial_cache_files_internal_test.go | 8 +- internal/imgcache/pendingaccesstimes.go | 160 +++++++ .../pendingaccesstimes_internal_test.go | 319 +++++++++++++ internal/imgcache/pendingcounts.go | 221 --------- .../imgcache/pendingcounts_internal_test.go | 431 ------------------ internal/imgcache/service.go | 9 +- internal/imgcache/service_internal_test.go | 4 +- internal/imgcache/stats_internal_test.go | 273 +---------- internal/imgcache/testutil_internal_test.go | 3 +- internal/server/image_proxy_internal_test.go | 3 + .../server/login_rate_limit_internal_test.go | 2 + internal/server/logout_internal_test.go | 2 + 37 files changed, 835 insertions(+), 1195 deletions(-) create mode 100644 internal/imgcache/counters.go create mode 100644 internal/imgcache/counters_internal_test.go create mode 100644 internal/imgcache/pendingaccesstimes.go create mode 100644 internal/imgcache/pendingaccesstimes_internal_test.go delete mode 100644 internal/imgcache/pendingcounts.go delete mode 100644 internal/imgcache/pendingcounts_internal_test.go diff --git a/README.md b/README.md index 8c190de..87bf02b 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,17 @@ A load balancer's health check can request `/.well-known/healthcheck.json`, which answers 200 whenever pixa is running, in maintenance mode too (see `maintenance_mode`). -On SIGTERM or SIGINT pixa stops accepting connections, gives the requests in -progress and the images being processed 5 seconds to finish, then writes to the -database the counts of cache hits, misses, fetches and conversions and the times -cached images were last used that are not written yet (see Storage), and exits: -with 0, or with 1 when images were still being processed after those 5 seconds, -some of those counts or times could not be written, or another part of pixa -failed to stop. A request not finished by then is cut off. `docker stop` waits -10 seconds before it kills the container. +On SIGTERM or SIGINT pixa stops accepting connections. The requests in progress +and the images being processed get 5 seconds to finish. A request not finished +by then is cut off. pixa then writes the waiting last-used times of cached +images to the database (see Storage) and exits. It exits with 1 when: + +- images were still being processed after those 5 seconds, +- some of those times could not be written, or +- another part of pixa failed to stop. + +Otherwise it exits with 0. `docker stop` waits 10 seconds before it kills the +container. Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL @@ -209,14 +212,14 @@ Test). Toward it, a cache hit opens the transformed image's file and nothing else, and makes no database call. The content type comes from the file's first bytes, which are those of the format pixa wrote it in; a file whose first bytes are not those of a format pixa writes is not served, and the image is converted -again. The time the image was last served, which eviction orders by, and the hit -count are handed to a background writer without waiting for it, as are the -counts of misses, fetches and conversions, and the time a miss last used a -cached source image. Once a second that writer writes the waiting times in one -transaction and the counts in one statement. While 10,000 times of transformed -images, or 10,000 of source images, wait to be written, a further one is -dropped, and that image only looks less recently used to eviction. -`script/bench` measures how many hits pixa serves per second (see Load Test). +again. A hit hands the time it was served to a background writer and does not +wait for the writer. Eviction orders by that time. The hit is counted in memory. +A miss that uses a cached source image also hands that writer the time it used +the image. Once a second that writer writes the waiting times in one +transaction. While 10,000 times of transformed images, or 10,000 of source +images, wait to be written, a further one is dropped, and that image only looks +less recently used to eviction. `script/bench` measures how many hits pixa +serves per second (see Load Test). ### Routes @@ -814,15 +817,27 @@ See `configs/config.example.yml` for all options with defaults. - **Image processing**: govips (CGO wrapper for libvips) - **Database**: SQLite via modernc.org/sqlite - **Static assets**: embedded via `//go:embed` -- **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics - (duration, response size, requests in flight) and the Go runtime and process - metrics; requests are measured and `/metrics` is served only when - `metrics.username` and `metrics.password` are set. A request's `handler` label - is the pattern of the route it matches, such as `/v1/image/*`, never its path. - Requests that match no route share the label `unmatched`: those the router - answers 404 or 405, and CORS preflights. A request's `method` label is its - method when that is `GET`, `HEAD`, `POST`, `PUT`, `PATCH`, `DELETE`, - `OPTIONS`, `CONNECT` or `TRACE`, and `other` for any other method +- **Metrics**: Prometheus, at `/metrics`. pixa serves `/metrics` and measures + HTTP requests only when `metrics.username` and `metrics.password` are set. + `/metrics` holds: + - the duration of HTTP requests + - the size of HTTP responses + - the HTTP requests in flight + - the Go runtime and process metrics + - `pixa_cache_hits_total`: image requests served from the cache + - `pixa_cache_misses_total`: image requests not served from the cache + - `pixa_upstream_fetches_total`: fetches of a source image that read any + bytes + - `pixa_upstream_fetch_bytes_total`: the bytes those fetches read + - `pixa_transforms_total`: images converted + + A request's `handler` label is the pattern of the route it matches, such as + `/v1/image/*`, never its path. Requests that match no route share the label + `unmatched`: those the router answers 404 or 405, and CORS preflights. A + request's `method` label is its method when that is `GET`, `HEAD`, `POST`, + `PUT`, `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` or `TRACE`, and `other` for + any other method. + - **Logging**: stdlib slog ## Entrypoints diff --git a/TODO.md b/TODO.md index 97da3b8..a2785f7 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,18 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 cache hits, misses, upstream fetches, the bytes those fetches read + and transforms are counted as Prometheus counters in memory (closes #311). + They are `pixa_cache_hits_total`, `pixa_cache_misses_total`, + `pixa_upstream_fetches_total`, `pixa_upstream_fetch_bytes_total` and + `pixa_transforms_total`, fields of the cache. `NewCache` makes them and + registers them on the registry it is given (`internal/imgcache/counters.go`). + pixad gives it Prometheus' default registry, which `/metrics` serves; each + test gives it a new one. The `Increment` methods add to their cache's + counters. `cache_stats`, `Cache.Stats`, `Service.Stats`, `CacheStats` and the + pending counts are gone. The background goroutine writes only access times + now: `StartPendingAccessTimeWrites` starts it and + `StopPendingAccessTimeWrites` stops it, in `pendingaccesstimes.go`. - 2026-10-21 the schema file declares only the tables and columns pixa writes and reads (closes #310): `output_content` and `request_cache` are gone, and so are the columns nothing read: the host, path, query, `content_type` and diff --git a/internal/app/app.go b/internal/app/app.go index 3521236..c09f554 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -7,6 +7,7 @@ import ( "os/signal" "syscall" + "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" "go.uber.org/fx" "sneak.berlin/go/pixa/internal/config" @@ -61,6 +62,8 @@ func run(_ *cobra.Command, _ []string) { server.New, middleware.New, healthcheck.New, + // The registry /metrics serves + func() prometheus.Registerer { return prometheus.DefaultRegisterer }, ), fx.Invoke( func(log *logger.Logger) { log.Identify() }, diff --git a/internal/database/concurrent_writes_internal_test.go b/internal/database/concurrent_writes_internal_test.go index c067853..8dcfdbf 100644 --- a/internal/database/concurrent_writes_internal_test.go +++ b/internal/database/concurrent_writes_internal_test.go @@ -104,33 +104,24 @@ func writeConcurrently(t *testing.T, db *sql.DB) { wg.Wait() - var hits, sources int + var sources int - err := db.QueryRowContext(ctx, ` - SELECT hit_count, (SELECT COUNT(*) FROM source_content) - FROM cache_stats WHERE id = 1 - `).Scan(&hits, &sources) + err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM source_content`). + Scan(&sources) if err != nil { t.Fatalf("failed to count the writes: %v", err) } - if hits != totalRequests || sources != totalRequests { - t.Errorf("hit_count = %d and %d source_content rows, want %d of each", - hits, sources, totalRequests) + if sources != totalRequests { + t.Errorf("%d source_content rows, want %d", sources, totalRequests) } } // writeLikeOneRequest makes the writes one request and the eviction pass -// make: it counts a cache hit, stores a source, records a transformed image -// and deletes that record again. +// make: it stores a source, records a transformed image and deletes that +// record again. func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error { - _, err := db.ExecContext(ctx, - `UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`) - if err != nil { - return fmt.Errorf("counting a cache hit: %w", err) - } - - _, err = db.ExecContext(ctx, `INSERT INTO source_content + _, err := db.ExecContext(ctx, `INSERT INTO source_content (source_key, size_bytes) VALUES (?, 1)`, key) if err != nil { return fmt.Errorf("storing a source: %w", err) diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go index 5cc6186..1339ac7 100644 --- a/internal/database/database_internal_test.go +++ b/internal/database/database_internal_test.go @@ -159,7 +159,7 @@ func TestApplyMigrations_CreatesSchemaAndTables(t *testing.T) { } // Verify that the application tables created by 001.sql exist. - tables := []string{"source_content", "negative_cache", "cache_stats"} + tables := []string{"source_content", "negative_cache"} for _, table := range tables { var count int diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index 7f6f472..8aa1696 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -108,18 +108,6 @@ CREATE TABLE IF NOT EXISTS negative_cache ( ); CREATE INDEX IF NOT EXISTS idx_negative_cache_expires ON negative_cache(expires_at); --- Cache statistics for monitoring -CREATE TABLE IF NOT EXISTS cache_stats ( - id INTEGER PRIMARY KEY CHECK (id = 1), - hit_count INTEGER NOT NULL DEFAULT 0, - miss_count INTEGER NOT NULL DEFAULT 0, - upstream_fetch_count INTEGER NOT NULL DEFAULT 0, - upstream_fetch_bytes INTEGER NOT NULL DEFAULT 0, - transform_count INTEGER NOT NULL DEFAULT 0, - last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP -); -INSERT OR IGNORE INTO cache_stats (id) VALUES (1); - -- A random value that a login session carries from its login. pixa makes it -- when the table has no row, and each logout replaces it; a login session -- carrying another value is refused, so logging out ends every login session. diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go index 9592ce2..713395a 100644 --- a/internal/handlers/cache_hit_internal_test.go +++ b/internal/handlers/cache_hit_internal_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/encurl" @@ -101,9 +102,8 @@ func newCountingDatabase(tb testing.TB) (*sql.DB, *atomic.Int64) { // newCountingDatabaseServer returns a router for both image routes whose // image cache uses newCountingDatabase, the count of the statements run on // that database, and a URL on each route for a different image of -// photoPath on countingHost. Nothing writes the cache's pending counts and -// access times in the background, so every statement counted comes from a -// request. +// photoPath on countingHost. Nothing writes the cache's access times in the +// background, so every statement counted comes from a request. func newCountingDatabaseServer( tb testing.TB, ) (http.Handler, *atomic.Int64, []string) { @@ -121,7 +121,7 @@ func newCountingDatabaseServer( cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ StateDir: tb.TempDir(), NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { tb.Fatalf("imgcache.NewCache() error = %v", err) } diff --git a/internal/handlers/cache_max_bytes_internal_test.go b/internal/handlers/cache_max_bytes_internal_test.go index af0a8ab..1fec51d 100644 --- a/internal/handlers/cache_max_bytes_internal_test.go +++ b/internal/handlers/cache_max_bytes_internal_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx/fxtest" "sneak.berlin/go/pixa/internal/config" @@ -127,7 +128,10 @@ func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) { t.Fatalf("database.New() error = %v", err) } - _, err = New(lc, Params{Logger: log, Database: db, Config: cfg}) + _, err = New(lc, Params{ + Logger: log, Database: db, Config: cfg, + MetricsRegistry: prometheus.NewRegistry(), + }) if err != nil { t.Fatalf("New() error = %v", err) } diff --git a/internal/handlers/fetcher_internal_test.go b/internal/handlers/fetcher_internal_test.go index d78091a..e358f1b 100644 --- a/internal/handlers/fetcher_internal_test.go +++ b/internal/handlers/fetcher_internal_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "go.uber.org/fx/fxtest" @@ -47,7 +48,8 @@ func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) { app := fxtest.New(t, fx.Supply(cfg), - fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New), + fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New, + func() prometheus.Registerer { return prometheus.NewRegistry() }), fx.Populate(&h), ) app.RequireStart() diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go index 63e7aad..eeb80fe 100644 --- a/internal/handlers/first_bytes_internal_test.go +++ b/internal/handlers/first_bytes_internal_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" @@ -35,7 +36,7 @@ func newRoutesFetchingThrough( cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("imgcache.NewCache() error = %v", err) } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 3285c2b..ba42481 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -9,6 +9,7 @@ import ( "net/http" "time" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/config" @@ -30,6 +31,11 @@ type Params struct { Database *database.Database Config *config.Config + // MetricsRegistry is the registry the image cache registers its + // counters on. pixad provides Prometheus' default registry, the one + // /metrics serves. + MetricsRegistry prometheus.Registerer + // Fetcher, when provided, fetches upstream images in place of the // fetcher the handlers build from the config. Only tests provide one; // pixad does not. @@ -52,6 +58,10 @@ type Handlers struct { // refererBlocklist matches the hosts of referer_blocklist; its IsAllowed // reports whether a URL's host is on that list. refererBlocklist *allowlist.HostAllowList + + // metricsRegistry is the registry the image cache registers its + // counters on. + metricsRegistry prometheus.Registerer } // New creates a new Handlers instance. @@ -69,6 +79,7 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { fetcher: params.Fetcher, csrfProtect: csrfProtect, refererBlocklist: allowlist.New(params.Config.RefererBlocklist), + metricsRegistry: params.MetricsRegistry, } lc.Append(fx.Hook{ @@ -76,8 +87,8 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { OnStart: func(_ context.Context) error { return s.initImageService() }, - // The pending counts and access times are written here, before the - // database's stop hook closes it. + // The pending access times are written here, before the database's + // stop hook closes it. OnStop: func(ctx context.Context) error { if s.imgCache == nil { return nil @@ -85,7 +96,7 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) { return errors.Join( s.imgCache.StopEviction(ctx), - s.imgCache.StopPendingCountWrites(ctx), + s.imgCache.StopPendingAccessTimeWrites(ctx), ) }, }) @@ -116,7 +127,8 @@ func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig { // initImageService initializes the image cache and service. func (s *Handlers) initImageService() error { - cache, err := imgcache.NewCache(s.db.DB(), newCacheConfig(s.config, s.log)) + cache, err := imgcache.NewCache( + s.db.DB(), newCacheConfig(s.config, s.log), s.metricsRegistry) if err != nil { return err } @@ -128,9 +140,9 @@ func (s *Handlers) initImageService() error { // rows, which is all it does when the disk cache is disabled. cache.StartEviction(imgcache.DefaultEvictionInterval) - // Writes the counts and the access times of cache hits that requests - // hand over, so that no request waits for the database to record them - cache.StartPendingCountWrites() + // Writes the access times that requests hand over, so that no request + // waits for the database to record them + cache.StartPendingAccessTimeWrites() // Create the fetcher config fetcherCfg := httpfetcher.DefaultConfig() diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index f396a05..82957fb 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imgcache" @@ -48,7 +49,7 @@ func setupTestHandler(t *testing.T) *testFixtures { cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ StateDir: tmpDir, NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index e94c30f..0fc236f 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" @@ -33,7 +34,7 @@ func newSignedHostServer( cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("imgcache.NewCache() error = %v", err) } diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index 72acfd8..ef7130a 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" @@ -31,7 +32,7 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index 86f0c32..6cdfbe6 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" @@ -50,7 +51,7 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("imgcache.NewCache() error = %v", err) } diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index b03b586..5342080 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -8,10 +8,9 @@ import ( "io" "log/slog" "path/filepath" - "sync" - "sync/atomic" "time" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -81,33 +80,30 @@ type Cache struct { evictionDone chan struct{} evictionCancel context.CancelFunc - // The pending counts: hits, misses, upstream fetches and transforms - // that requests counted, kept here until they are written by the - // goroutine that StartPendingCountWrites starts, so that no request - // waits for the database to count it. pendingVariantAccessTimes holds - // the variants served from the cache, and pendingSourceAccessTimes the - // cached sources a miss used, whose access time is still to be written - // by that goroutine. As for eviction, the channels are created in + // pendingVariantAccessTimes holds the variants served from the cache, + // and pendingSourceAccessTimes the cached sources a miss used, whose + // access time is still to be written by the goroutine that + // StartPendingAccessTimeWrites starts, so that no request waits for the + // database to record it. As for eviction, the channels are created in // NewCache, so that requests can hand over access times before that - // goroutine starts; pendingCountsDone is closed when it returns. - // pendingCountsCancel, set by StartPendingCountWrites, cancels its - // context, which tells it to finish. - pendingHits atomic.Int64 - pendingMisses atomic.Int64 - pendingUpstreamFetches atomic.Int64 - pendingUpstreamFetchBytes atomic.Int64 - pendingTransforms atomic.Int64 + // goroutine starts; pendingAccessTimesDone is closed when it returns. + // pendingAccessTimesCancel, set by StartPendingAccessTimeWrites, + // cancels its context, which tells it to finish. pendingVariantAccessTimes chan imagerequest.VariantKey pendingSourceAccessTimes chan imagerequest.SourceKey - pendingCountsDone chan struct{} - pendingCountsCancel context.CancelFunc + pendingAccessTimesDone chan struct{} + pendingAccessTimesCancel context.CancelFunc - // Held through each write of the pending counts, and by - // readHitAndMissCounts to read them - pendingCountsWriteMutex sync.Mutex + // The counts of what the cache does, made and registered by + // registerCounters (see counters.go) + cacheHits prometheus.Counter + cacheMisses prometheus.Counter + upstreamFetches prometheus.Counter + upstreamFetchBytes prometheus.Counter + transforms prometheus.Counter // reconciliationPageSize is the most rows one read of a content table - // returns in the reconciliation pass and in Stats. newCache sets it to + // returns in the reconciliation pass. newCache sets it to // defaultReconciliationPageSize; tests set it smaller. reconciliationPageSize int @@ -118,15 +114,22 @@ type Cache struct { reconciliationReadTestHook func(rows int) } -// NewCache creates a new cache instance. -func NewCache(db *sql.DB, config CacheConfig) (*Cache, error) { - return newCache(db, config, defaultFreeSpaceProbe) +// NewCache creates a new cache instance and registers its counters on +// metricsRegistry. pixad passes Prometheus' default registry, the one +// /metrics serves. +func NewCache( + db *sql.DB, config CacheConfig, metricsRegistry prometheus.Registerer, +) (*Cache, error) { + return newCache(db, config, metricsRegistry, defaultFreeSpaceProbe) } // newCache is NewCache with the free-space probe passed in, so tests // can fake the free space the default limit is worked out from. func newCache( - db *sql.DB, config CacheConfig, probe FreeSpaceProbeFunc, + db *sql.DB, + config CacheConfig, + metricsRegistry prometheus.Registerer, + probe FreeSpaceProbeFunc, ) (*Cache, error) { log := config.Logger if log == nil { @@ -143,11 +146,16 @@ func newCache( pendingVariantAccessTimes: make(chan imagerequest.VariantKey, maxPendingAccessTimes), pendingSourceAccessTimes: make(chan imagerequest.SourceKey, maxPendingAccessTimes), - pendingCountsDone: make(chan struct{}), + pendingAccessTimesDone: make(chan struct{}), reconciliationPageSize: defaultReconciliationPageSize, } + err := c.registerCounters(metricsRegistry) + if err != nil { + return nil, err + } + if c.disabled { return c, nil } @@ -373,51 +381,15 @@ func (c *Cache) CleanExpired(ctx context.Context) error { } } -// Stats returns cache statistics. -func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) { - var stats CacheStats - - err := c.readHitAndMissCounts(ctx, &stats) - if err != nil && !errors.Is(err, sql.ErrNoRows) { - return nil, fmt.Errorf("failed to get cache stats: %w", err) - } - - // Count and size the cached source images and processed variants from - // their tables. A disabled cache holds none, whatever rows an earlier - // run left. - if !c.disabled { - err = c.db.QueryRowContext(ctx, ` - SELECT (SELECT COUNT(*) FROM source_content) - + (SELECT COUNT(*) FROM variant_content) - `).Scan(&stats.TotalItems) - if err != nil { - c.log.Warn("failed to count cache items for stats", "error", err) - } - - stats.TotalSizeBytes, err = c.sumContentSizeBytes(ctx) - if err != nil { - c.log.Warn("failed to sum cache size for stats", "error", err) - } - } - - // Compute hit rate as a ratio - if stats.HitCount+stats.MissCount > 0 { - stats.HitRate = float64(stats.HitCount) / float64(stats.HitCount+stats.MissCount) - } - - return &stats, nil -} - // IncrementStats counts a cache hit or miss, and an upstream fetch that read // fetchBytes bytes, as IncrementUpstreamFetch does. Like the other Increment -// methods, it only adds to the pending counts, which the goroutine -// StartPendingCountWrites starts writes to the database, so it never waits -// for the database or fails. +// methods, it only adds to the cache's Prometheus counters in memory (see +// counters.go), so it never waits for the database or fails. func (c *Cache) IncrementStats(hit bool, fetchBytes int64) { if hit { - c.pendingHits.Add(1) + c.cacheHits.Inc() } else { - c.pendingMisses.Add(1) + c.cacheMisses.Inc() } c.IncrementUpstreamFetch(fetchBytes) @@ -430,37 +402,18 @@ func (c *Cache) IncrementUpstreamFetch(fetchBytes int64) { return } - c.pendingUpstreamFetches.Add(1) - c.pendingUpstreamFetchBytes.Add(fetchBytes) + c.upstreamFetches.Inc() + c.upstreamFetchBytes.Add(float64(fetchBytes)) } // IncrementTransformCount counts one image transcoded by the image processor. func (c *Cache) IncrementTransformCount() { - c.pendingTransforms.Add(1) -} - -// readHitAndMissCounts sets the hit and miss counts of stats to those of the -// cache_stats row plus the pending hits and misses (see -// StartPendingCountWrites). It holds pendingCountsWriteMutex, so that no -// write of the pending counts falls between the two reads. -func (c *Cache) readHitAndMissCounts(ctx context.Context, stats *CacheStats) error { - c.pendingCountsWriteMutex.Lock() - defer c.pendingCountsWriteMutex.Unlock() - - err := c.db.QueryRowContext(ctx, ` - SELECT hit_count, miss_count - FROM cache_stats WHERE id = 1 - `).Scan(&stats.HitCount, &stats.MissCount) - - stats.HitCount += c.pendingHits.Load() - stats.MissCount += c.pendingMisses.Load() - - return err + c.transforms.Inc() } // touchVariant hands the access time of a variant served from the cache to -// the goroutine StartPendingCountWrites starts, which writes it in its next -// batch. It never waits: while maxPendingAccessTimes access times of +// the goroutine StartPendingAccessTimeWrites starts, which writes it in its +// next batch. It never waits: while maxPendingAccessTimes access times of // variants wait to be written, this one is dropped, and the variant only // looks colder to eviction. func (c *Cache) touchVariant(cacheKey imagerequest.VariantKey) { @@ -471,10 +424,10 @@ func (c *Cache) touchVariant(cacheKey imagerequest.VariantKey) { } // touchSourceContent hands the access time of a cached source a miss used to -// the goroutine StartPendingCountWrites starts, which writes it in its next -// batch. It never waits: while maxPendingAccessTimes access times of sources -// wait to be written, this one is dropped, and the source only looks colder -// to eviction. +// the goroutine StartPendingAccessTimeWrites starts, which writes it in its +// next batch. It never waits: while maxPendingAccessTimes access times of +// sources wait to be written, this one is dropped, and the source only looks +// colder to eviction. func (c *Cache) touchSourceContent(key imagerequest.SourceKey) { select { case c.pendingSourceAccessTimes <- key: diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index 70135b8..b2d2301 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -167,8 +167,8 @@ func TestService_Get_MakesAgainVariantOfNoOutputFormat(t *testing.T) { // TestService_Get_HitWaitsForNoDatabaseWrite holds the database's one // connection while a cached variant is requested. The hit must be answered -// while the connection is still held, and its access time and hit count -// must reach the database once it is free. +// while the connection is still held, and its access time must reach the +// database once it is free. func TestService_Get_HitWaitsForNoDatabaseWrite(t *testing.T) { t.Parallel() @@ -177,14 +177,18 @@ func TestService_Get_HitWaitsForNoDatabaseWrite(t *testing.T) { key := imagerequest.CacheKey(req) getImage(t, svc, req) - waitForPendingCountsWritten(t, svc.cache) + + // Each connection to an in-memory SQLite database opens a new, empty one, + // so the goroutine that writes the access times must share the + // connection that holds the schema. + svc.cache.db.SetMaxOpenConns(1) + svc.cache.StartPendingAccessTimeWrites() + + defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }() anHourAgo := time.Now().Add(-time.Hour) setVariantLastAccessed(t, svc.cache, key, anHourAgo) - want := readCacheStatsCounters(t, svc.cache) - want.hitCount++ - releaseDatabase := holdDatabase(t, svc.cache) defer releaseDatabase() @@ -195,15 +199,13 @@ func TestService_Get_HitWaitsForNoDatabaseWrite(t *testing.T) { if got.err != nil { t.Fatalf("Get() of the cached variant error = %v", got.err) } - case <-time.After(pendingCountsWait): + case <-time.After(pendingAccessTimesWait): t.Fatal("the cache hit waited for the database") } releaseDatabase() - waitForCounters(t, svc.cache, want) - - deadline := time.Now().Add(pendingCountsWait) + deadline := time.Now().Add(pendingAccessTimesWait) for countRows(t, svc.cache, ` SELECT COUNT(*) FROM variant_content @@ -239,7 +241,7 @@ func TestTouchVariantAndTouchSourceContent_DropAccessTimesPastTheLimit(t *testin select { case <-done: - case <-time.After(pendingCountsWait): + case <-time.After(pendingAccessTimesWait): t.Fatal("touchVariant or touchSourceContent waited once the access " + "times reached the limit") } @@ -255,11 +257,11 @@ func TestTouchVariantAndTouchSourceContent_DropAccessTimesPastTheLimit(t *testin } } -// TestStopPendingCountWrites_WritesPendingAccessTimes hands the access time -// of one of two variants, three times, then stops the goroutine that writes -// them, which never started. The stop must write that access time and +// TestStopPendingAccessTimeWrites_WritesPendingAccessTimes hands the access +// time of one of two variants, three times, then stops the goroutine that +// writes them, which never started. The stop must write that access time and // leave the other variant's. -func TestStopPendingCountWrites_WritesPendingAccessTimes(t *testing.T) { +func TestStopPendingAccessTimeWrites_WritesPendingAccessTimes(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) @@ -274,9 +276,9 @@ func TestStopPendingCountWrites_WritesPendingAccessTimes(t *testing.T) { cache.touchVariant(testVariantKeyOne) } - err := cache.StopPendingCountWrites(t.Context()) + err := cache.StopPendingAccessTimeWrites(t.Context()) if err != nil { - t.Fatalf("StopPendingCountWrites() error = %v", err) + t.Fatalf("StopPendingAccessTimeWrites() error = %v", err) } for key, wantTouched := range map[imagerequest.VariantKey]int{ diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index 6431300..56cc5a8 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" _ "modernc.org/sqlite" ) @@ -33,16 +34,6 @@ func setupTestDB(t *testing.T) *sql.DB { expires_at DATETIME NOT NULL, UNIQUE(source_host, source_path, source_query) ); - CREATE TABLE cache_stats ( - id INTEGER PRIMARY KEY CHECK (id = 1), - hit_count INTEGER NOT NULL DEFAULT 0, - miss_count INTEGER NOT NULL DEFAULT 0, - upstream_fetch_count INTEGER NOT NULL DEFAULT 0, - upstream_fetch_bytes INTEGER NOT NULL DEFAULT 0, - transform_count INTEGER NOT NULL DEFAULT 0, - last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - INSERT INTO cache_stats (id) VALUES (1); ` _, err = db.ExecContext(t.Context(), schema) @@ -53,48 +44,6 @@ func setupTestDB(t *testing.T) *sql.DB { return db } -func setupTestCache(t *testing.T) *Cache { - t.Helper() - - tmpDir := t.TempDir() - db := setupTestDB(t) - - cache, err := NewCache(db, CacheConfig{ - StateDir: tmpDir, - NegativeTTL: 5 * time.Minute, - }) - if err != nil { - t.Fatalf("failed to create cache: %v", err) - } - - return cache -} - -func TestCache_Stats(t *testing.T) { - t.Parallel() - - cache := setupTestCache(t) - ctx := context.Background() - - // Increment some stats - cache.IncrementStats(true, 0) - cache.IncrementStats(true, 0) - cache.IncrementStats(false, 1024) - - stats, err := cache.Stats(ctx) - if err != nil { - t.Fatalf("Stats() error = %v", err) - } - - if stats.HitCount != 2 { - t.Errorf("Stats() HitCount = %d, want 2", stats.HitCount) - } - - if stats.MissCount != 1 { - t.Errorf("Stats() MissCount = %d, want 1", stats.MissCount) - } -} - func TestCache_CleanExpired(t *testing.T) { t.Parallel() @@ -104,7 +53,7 @@ func TestCache_CleanExpired(t *testing.T) { cache, _ := NewCache(db, CacheConfig{ StateDir: tmpDir, NegativeTTL: 1 * time.Millisecond, - }) + }, prometheus.NewRegistry()) ctx := context.Background() @@ -155,7 +104,7 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) { _, err := NewCache(db, CacheConfig{ StateDir: tmpDir, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("NewCache() error = %v", err) } diff --git a/internal/imgcache/cachesize_internal_test.go b/internal/imgcache/cachesize_internal_test.go index 7cca989..e67f953 100644 --- a/internal/imgcache/cachesize_internal_test.go +++ b/internal/imgcache/cachesize_internal_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/prometheus/client_golang/prometheus" ) // Static errors returned by the stub free-space probes below. @@ -144,7 +146,7 @@ func TestNewCacheComputesDefaultMaxBytesWhenAsked(t *testing.T) { cache, err := newCache(evictionTestDB(t), CacheConfig{ StateDir: stateDir, UseDefaultMaxBytes: true, - }, probe) + }, prometheus.NewRegistry(), probe) if err != nil { t.Fatalf("newCache returned error: %v", err) } @@ -175,7 +177,7 @@ func TestNewCacheKeepsExplicitMaxBytes(t *testing.T) { cache, err := newCache(evictionTestDB(t), CacheConfig{ StateDir: t.TempDir(), MaxBytes: 1024, - }, probe) + }, prometheus.NewRegistry(), probe) if err != nil { t.Fatalf("newCache returned error: %v", err) } diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index f46b346..7fb3e0e 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -191,11 +191,9 @@ func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { t.Fatal(err) } - want := cacheStatsCounters{0, requests, 1, int64(len(photo)), 1} + want := counterValues{0, requests, 1, int64(len(photo)), 1} - waitForPendingCountsWritten(t, svc.cache) - - if got := readCacheStatsCounters(t, svc.cache); got != want { + if got := readCounters(svc.cache); got != want { t.Errorf("counters = %+v, want %+v", got, want) } } diff --git a/internal/imgcache/counters.go b/internal/imgcache/counters.go new file mode 100644 index 0000000..8e50780 --- /dev/null +++ b/internal/imgcache/counters.go @@ -0,0 +1,45 @@ +package imgcache + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" +) + +// registerCounters makes the cache's counters and registers them on +// metricsRegistry. The counters are kept in memory, so adding to one waits +// for nothing and makes no database call. It fails when metricsRegistry +// already holds counters of the same names. +func (c *Cache) registerCounters(metricsRegistry prometheus.Registerer) error { + c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_cache_hits_total", + Help: "Image requests served from the cache.", + }) + c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_cache_misses_total", + Help: "Image requests not served from the cache.", + }) + c.upstreamFetches = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_upstream_fetches_total", + Help: "Fetches of a source image from upstream that read any bytes.", + }) + c.upstreamFetchBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_upstream_fetch_bytes_total", + Help: "Bytes read from upstream by fetches of source images.", + }) + c.transforms = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_transforms_total", + Help: "Images the image processor transformed.", + }) + + for _, counter := range []prometheus.Counter{ + c.cacheHits, c.cacheMisses, c.upstreamFetches, c.upstreamFetchBytes, c.transforms, + } { + err := metricsRegistry.Register(counter) + if err != nil { + return fmt.Errorf("failed to register the cache's counters: %w", err) + } + } + + return nil +} diff --git a/internal/imgcache/counters_internal_test.go b/internal/imgcache/counters_internal_test.go new file mode 100644 index 0000000..7886cbf --- /dev/null +++ b/internal/imgcache/counters_internal_test.go @@ -0,0 +1,84 @@ +package imgcache + +import ( + "errors" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// counterValues holds the values of a cache's counters. +type counterValues struct { + hitCount int64 + missCount int64 + upstreamFetchCount int64 + upstreamFetchBytes int64 + transformCount int64 +} + +// readCounters reads cache's counters. +func readCounters(cache *Cache) counterValues { + return counterValues{ + hitCount: int64(testutil.ToFloat64(cache.cacheHits)), + missCount: int64(testutil.ToFloat64(cache.cacheMisses)), + upstreamFetchCount: int64(testutil.ToFloat64(cache.upstreamFetches)), + upstreamFetchBytes: int64(testutil.ToFloat64(cache.upstreamFetchBytes)), + transformCount: int64(testutil.ToFloat64(cache.transforms)), + } +} + +// TestNewCache_RegistersItsCountersOnItsRegistry checks that NewCache +// registers the cache's counters on the registry it is given, under the +// names README.md gives them. +func TestNewCache_RegistersItsCountersOnItsRegistry(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + + _, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry) + if err != nil { + t.Fatalf("NewCache() error = %v", err) + } + + names := []string{ + "pixa_cache_hits_total", + "pixa_cache_misses_total", + "pixa_upstream_fetches_total", + "pixa_upstream_fetch_bytes_total", + "pixa_transforms_total", + } + + count, err := testutil.GatherAndCount(registry, names...) + if err != nil { + t.Fatalf("testutil.GatherAndCount() error = %v", err) + } + + if count != len(names) { + t.Errorf("%d of the counters %v are on the registry, want %d", + count, names, len(names)) + } +} + +// TestNewCache_FailsWhenItsCountersCannotBeRegistered makes a second cache +// on the registry of a first. Its counters have the same names, so the +// registry refuses them, and NewCache must return that error. +func TestNewCache_FailsWhenItsCountersCannotBeRegistered(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + + _, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry) + if err != nil { + t.Fatalf("NewCache() of the first cache error = %v", err) + } + + _, err = NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry) + t.Logf("NewCache() of the second cache error = %v", err) + + var alreadyRegistered prometheus.AlreadyRegisteredError + if !errors.As(err, &alreadyRegistered) { + t.Errorf("NewCache() of the second cache error = %v, want %T", + err, alreadyRegistered) + } +} diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index a02e6fc..99ba09e 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "modernc.org/sqlite" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/imagerequest" @@ -74,7 +75,7 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) { NegativeTTL: 5 * time.Minute, MaxBytes: maxBytes, DisableDiskCache: maxBytes == 0, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index 8354741..6eb082f 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -44,23 +44,4 @@ type ImageCache interface { // Warm pre-fetches and caches an image without returning it Warm(ctx context.Context, req *imagerequest.Request) error - - // Stats returns cache statistics - Stats(ctx context.Context) (*CacheStats, error) -} - -// CacheStats contains cache statistics -type CacheStats struct { - // TotalItems is the number of cached source images plus processed - // variants - TotalItems int64 - // TotalSizeBytes is the total size of cached source images and - // processed variants - TotalSizeBytes int64 - // HitCount is the number of cache hits - HitCount int64 - // MissCount is the number of cache misses - MissCount int64 - // HitRate is HitCount / (HitCount + MissCount) - HitRate float64 } diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index dbfc69f..3aa7a10 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -19,7 +20,7 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatal(err) } @@ -68,7 +69,7 @@ func TestNegativeCache_Expired(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, NegativeTTL: 1 * time.Millisecond, // very short TTL - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatal(err) } @@ -191,7 +192,7 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { StateDir: t.TempDir(), NegativeTTL: time.Second, DisableDiskCache: tt.disableDiskCache, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } @@ -265,7 +266,8 @@ func negativeCacheRowCount(tb testing.TB, cache *Cache, sourcePath string) int { func TestCleanExpiredDeletesInBatches(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}) + cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, + prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } @@ -319,7 +321,8 @@ func TestCleanExpiredDeletesInBatches(t *testing.T) { func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}) + cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, + prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } @@ -349,7 +352,8 @@ func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) { func TestCleanExpiredStopsOnceContextEnds(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}) + cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, + prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index fa4b0ef..647cf2c 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -128,9 +128,7 @@ func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) { getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) - waitForPendingCountsWritten(t, svc.cache) - - fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount + fetches := readCounters(svc.cache).upstreamFetchCount t.Logf("upstream fetches = %d", fetches) if fetches != 2 { @@ -166,9 +164,7 @@ func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) { getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) - waitForPendingCountsWritten(t, svc.cache) - - fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount + fetches := readCounters(svc.cache).upstreamFetchCount t.Logf("upstream fetches = %d", fetches) if fetches != 2 { diff --git a/internal/imgcache/pendingaccesstimes.go b/internal/imgcache/pendingaccesstimes.go new file mode 100644 index 0000000..a89048e --- /dev/null +++ b/internal/imgcache/pendingaccesstimes.go @@ -0,0 +1,160 @@ +package imgcache + +import ( + "context" + "fmt" + "time" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// pendingWriteInterval is how often the goroutine +// StartPendingAccessTimeWrites starts writes the pending access times. +// SQLite's CURRENT_TIMESTAMP, which an access time is written as, counts +// whole seconds. +const pendingWriteInterval = time.Second + +// StartPendingAccessTimeWrites starts the goroutine that writes, every +// pendingWriteInterval, the pending access times in one transaction. +// Requests only hand these to it, so none waits for the database to record +// them, and only this goroutine's write ever waits for it. It is a no-op +// when already started. The goroutine outlives the caller, so it runs with +// its own context, which StopPendingAccessTimeWrites cancels to tell it to +// finish. +func (c *Cache) StartPendingAccessTimeWrites() { + if c.pendingAccessTimesCancel != nil { + return + } + + ctx, cancel := context.WithCancel(context.Background()) + c.pendingAccessTimesCancel = cancel + + go func() { + c.pendingAccessTimeWriteLoop(ctx) + }() +} + +// StopPendingAccessTimeWrites tells the goroutine +// StartPendingAccessTimeWrites started, if it did, to finish, and waits for +// its write under way to end and for it to return. It then writes the +// pending access times left, once, so that they are written before the +// database closes. It waits for all of this at most until ctx ends. When +// access times are not written, it logs that and returns an error. +func (c *Cache) StopPendingAccessTimeWrites(ctx context.Context) error { + var err error + + if c.pendingAccessTimesCancel != nil { + c.pendingAccessTimesCancel() + + select { + case <-c.pendingAccessTimesDone: + case <-ctx.Done(): + // The write given up on here may not reach the database: + // database/sql closes without waiting for a query, and a write + // still waiting for the connection fails when the database + // closes. Nothing is written here. + err = fmt.Errorf("access times still being written: %w", ctx.Err()) + } + } + + if err == nil { + err = c.writePendingAccessTimes(ctx) + } + + if err != nil { + c.log.Error("access times not written at shutdown", "error", err) + } + + return err +} + +// pendingAccessTimeWriteLoop is the body of the goroutine +// StartPendingAccessTimeWrites starts. Once ctx is cancelled it starts no +// write and returns, but does not cut off a write under way then. Access +// times whose write fails are dropped. +func (c *Cache) pendingAccessTimeWriteLoop(ctx context.Context) { + defer close(c.pendingAccessTimesDone) + + ticker := time.NewTicker(pendingWriteInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + // select picks at random when ctx is cancelled and a tick waits + // too. + if ctx.Err() != nil { + return + } + + err := c.writePendingAccessTimes(context.WithoutCancel(ctx)) + if err != nil { + c.log.Warn("failed to write access times", "error", err) + } + } +} + +// writePendingAccessTimes takes out the variants waiting in +// pendingVariantAccessTimes and the sources waiting in +// pendingSourceAccessTimes, and sets their last_accessed_at to now in one +// transaction, once for each however often it was used. Nothing else takes +// from those channels while it runs: only the goroutine +// StartPendingAccessTimeWrites starts calls it, or +// StopPendingAccessTimeWrites once that goroutine has returned. +func (c *Cache) writePendingAccessTimes(ctx context.Context) error { + variants := make(map[imagerequest.VariantKey]struct{}) + + for range len(c.pendingVariantAccessTimes) { + variants[<-c.pendingVariantAccessTimes] = struct{}{} + } + + sources := make(map[imagerequest.SourceKey]struct{}) + + for range len(c.pendingSourceAccessTimes) { + sources[<-c.pendingSourceAccessTimes] = struct{}{} + } + + accessTimes := len(variants) + len(sources) + if accessTimes == 0 { + return nil + } + + tx, err := c.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + + // Does nothing once the transaction is committed + defer func() { _ = tx.Rollback() }() + + for cacheKey := range variants { + _, err = tx.ExecContext(ctx, ` + UPDATE variant_content SET last_accessed_at = CURRENT_TIMESTAMP + WHERE cache_key = ? + `, string(cacheKey)) + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + } + + for sourceKey := range sources { + _, err = tx.ExecContext(ctx, ` + UPDATE source_content SET last_accessed_at = CURRENT_TIMESTAMP + WHERE source_key = ? + `, string(sourceKey)) + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) + } + + return nil +} diff --git a/internal/imgcache/pendingaccesstimes_internal_test.go b/internal/imgcache/pendingaccesstimes_internal_test.go new file mode 100644 index 0000000..aa8a100 --- /dev/null +++ b/internal/imgcache/pendingaccesstimes_internal_test.go @@ -0,0 +1,319 @@ +package imgcache + +import ( + "bytes" + "context" + "database/sql" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// pendingAccessTimesWait is how long a test waits for the database's +// connection, for an access time to reach the database, for a write to start +// waiting for the connection, for a request to return past its deadline, or +// for StopPendingAccessTimeWrites to return. +const pendingAccessTimesWait = 5 * time.Second + +// otherWritesWait is how long a test gives other writes, once one write waits +// for the database's connection, to start waiting for it too. +const otherWritesWait = 100 * time.Millisecond + +// holdDatabase takes the one connection of cache's database, so that every +// other query waits for it, and returns the func that frees it. +func holdDatabase(t *testing.T, cache *Cache) func() { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), pendingAccessTimesWait) + defer cancel() + + conn, err := cache.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to take the database connection: %v", err) + } + + return func() { _ = conn.Close() } +} + +// waitForConnectionWaits waits until db.Stats().WaitCount, the number of +// times a caller has waited for a connection, reaches waits. +func waitForConnectionWaits(t *testing.T, db *sql.DB, waits int64) { + t.Helper() + + deadline := time.Now().Add(pendingAccessTimesWait) + + for db.Stats().WaitCount < waits { + if time.Now().After(deadline) { + t.Fatalf("callers waited for the database connection %d times, want %d", + db.Stats().WaitCount, waits) + } + + time.Sleep(10 * time.Millisecond) + } +} + +// TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the +// database's one connection while a request whose fetch is held reaches its +// deadline. The request must return with the deadline's error while the +// connection is still held, as it waits for no database write past its +// deadline. +func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchService(t) + + svc.cache.StartPendingAccessTimeWrites() + defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }() + + // Room for the request to reach its fetch on a busy host + const timeout = 2 * time.Second + + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + + deadline, _ := ctx.Deadline() + + results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + + // The request has made its database reads by the time it fetches. + select { + case <-fetcher.started: + case <-time.After(timeout): + t.Fatal("request did not reach its fetch by its deadline") + } + + releaseDatabase := holdDatabase(t, svc.cache) + defer releaseDatabase() + + select { + case got := <-results: + t.Logf("Get() returned %v after its deadline, error = %v", + time.Since(deadline), got.err) + + if !errors.Is(got.err, context.DeadlineExceeded) { + t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded) + } + case <-time.After(time.Until(deadline) + pendingAccessTimesWait): + t.Fatal("request did not return while the database was busy") + } +} + +// TestStopPendingAccessTimeWrites_LetsTheWriteUnderWayFinish holds the +// database's one connection while the goroutine that writes the pending +// access times waits for it, hands over another access time, then stops that +// goroutine. The stop must leave the write under way to finish rather than +// cut it off, and once the connection is free it must write the access time +// that write left. +func TestStopPendingAccessTimeWrites_LetsTheWriteUnderWayFinish(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + anHourAgo := time.Now().Add(-time.Hour) + + for _, key := range []imagerequest.VariantKey{testVariantKeyOne, testVariantKeyTwo} { + storeEvictionTestVariant(t, cache, key, []byte("variant bytes")) + setVariantLastAccessed(t, cache, key, anHourAgo) + } + + cache.StartPendingAccessTimeWrites() + defer func() { _ = cache.StopPendingAccessTimeWrites(t.Context()) }() + + releaseDatabase := holdDatabase(t, cache) + defer releaseDatabase() + + waitsBefore := cache.db.Stats().WaitCount + + cache.touchVariant(testVariantKeyOne) + + // The goroutine's write waits for the connection. + waitForConnectionWaits(t, cache.db, waitsBefore+1) + + // Handed over after that write took the pending access times + cache.touchVariant(testVariantKeyTwo) + + // The goroutine is told to finish here as well as by the stop, so that it + // is told before its write under way ends, however late the stop runs. + // The access time above then reaches the database only through the + // stop's own write. + cache.pendingAccessTimesCancel() + + stopped := make(chan error, 1) + + go func() { + stopped <- cache.StopPendingAccessTimeWrites(t.Context()) + }() + + // Had the stop cut off the write under way, its own write would wait for + // the connection too. + time.Sleep(otherWritesWait) + + waits := cache.db.Stats().WaitCount - waitsBefore + t.Logf("%d writes waited for the database", waits) + + if waits > 1 { + t.Errorf("%d writes waited for the database, want 1: "+ + "the stop cut off the write under way", waits) + } + + releaseDatabase() + + select { + case err := <-stopped: + if err != nil { + t.Fatalf("StopPendingAccessTimeWrites() error = %v", err) + } + case <-time.After(pendingAccessTimesWait): + t.Fatal("StopPendingAccessTimeWrites() did not return once the database was free") + } + + for _, key := range []imagerequest.VariantKey{testVariantKeyOne, testVariantKeyTwo} { + if countRows(t, cache, ` + SELECT COUNT(*) FROM variant_content + WHERE cache_key = ? AND last_accessed_at > ? + `, string(key), anHourAgo.UTC().Format(sqliteTimestampFormat)) != 1 { + t.Errorf("the access time of variant %s was not written", key) + } + } +} + +// TestStopPendingAccessTimeWrites_ReturnsWhenItsContextEnds holds the +// database's one connection while the goroutine that writes the pending +// access times waits for it, then stops that goroutine with a context that +// ends first. The stop must return the context's error, and the write under +// way must still reach the database once the connection is free. +func TestStopPendingAccessTimeWrites_ReturnsWhenItsContextEnds(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + storeEvictionTestVariant(t, cache, testVariantKeyOne, []byte("variant bytes")) + + anHourAgo := time.Now().Add(-time.Hour) + setVariantLastAccessed(t, cache, testVariantKeyOne, anHourAgo) + + cache.StartPendingAccessTimeWrites() + defer func() { _ = cache.StopPendingAccessTimeWrites(t.Context()) }() + + releaseDatabase := holdDatabase(t, cache) + defer releaseDatabase() + + waitsBefore := cache.db.Stats().WaitCount + + cache.touchVariant(testVariantKeyOne) + + // The goroutine's write waits for the connection. + waitForConnectionWaits(t, cache.db, waitsBefore+1) + + stopCtx, cancelStop := context.WithCancel(t.Context()) + cancelStop() + + stopped := make(chan error, 1) + + go func() { + stopped <- cache.StopPendingAccessTimeWrites(stopCtx) + }() + + select { + case err := <-stopped: + if !errors.Is(err, context.Canceled) { + t.Errorf("StopPendingAccessTimeWrites() error = %v, want %v", + err, context.Canceled) + } + case <-time.After(pendingAccessTimesWait): + t.Fatal("StopPendingAccessTimeWrites() did not return once its context ended") + } + + releaseDatabase() + + deadline := time.Now().Add(pendingAccessTimesWait) + + for countRows(t, cache, ` + SELECT COUNT(*) FROM variant_content + WHERE cache_key = ? AND last_accessed_at > ? + `, string(testVariantKeyOne), anHourAgo.UTC().Format(sqliteTimestampFormat)) == 0 { + if time.Now().After(deadline) { + t.Fatal("the access time being written was not written") + } + + time.Sleep(10 * time.Millisecond) + } +} + +// TestLookupSource_HandsAccessTimeToTheBackgroundWriter finds a cached source +// whose access time is an hour old. LookupSource must not write the access +// time itself but hand it over to be written, and StopPendingAccessTimeWrites +// must then write it. +func TestLookupSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg", + []byte("source bytes")) + anHourAgo := time.Now().Add(-time.Hour) + setSourceLastAccessed(t, cache, key, anHourAgo) + + touchedQuery := ` + SELECT COUNT(*) FROM source_content + WHERE source_key = ? AND last_accessed_at > ? + ` + anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat) + + found, err := cache.LookupSource(t.Context(), &imagerequest.Request{ + Source: imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}, + }) + if err != nil || found != key { + t.Fatalf("LookupSource() = %q, %v; want %q", found, err, key) + } + + if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 0 { + t.Error("LookupSource wrote the source's access time itself") + } + + if waiting := len(cache.pendingSourceAccessTimes); waiting != 1 { + t.Errorf("%d access times of sources wait to be written, want 1", waiting) + } + + err = cache.StopPendingAccessTimeWrites(t.Context()) + if err != nil { + t.Fatalf("StopPendingAccessTimeWrites() error = %v", err) + } + + if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 1 { + t.Error("the source's access time was not written") + } +} + +// TestStopPendingAccessTimeWrites_LogsWhatWasNotWritten makes the write of +// the access times fail at shutdown, by dropping the table they are written +// to. The stop must return an error and log that access times were not +// written. +func TestStopPendingAccessTimeWrites_LogsWhatWasNotWritten(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + var logs bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logs, nil)) + + cache.touchVariant(testVariantKeyOne) + + _, err := cache.db.ExecContext(t.Context(), `DROP TABLE variant_content`) + if err != nil { + t.Fatalf("dropping variant_content: %v", err) + } + + err = cache.StopPendingAccessTimeWrites(t.Context()) + t.Logf("StopPendingAccessTimeWrites() error = %v; logged:\n%s", err, logs.String()) + + if err == nil { + t.Error("StopPendingAccessTimeWrites() error = nil, want the access times' error") + } + + if !strings.Contains(logs.String(), "access times not written at shutdown") { + t.Error("the access times not written were not logged") + } +} diff --git a/internal/imgcache/pendingcounts.go b/internal/imgcache/pendingcounts.go deleted file mode 100644 index 5a7eef0..0000000 --- a/internal/imgcache/pendingcounts.go +++ /dev/null @@ -1,221 +0,0 @@ -package imgcache - -import ( - "context" - "errors" - "fmt" - "time" - - "sneak.berlin/go/pixa/internal/imagerequest" -) - -// pendingWriteInterval is how often the goroutine StartPendingCountWrites -// starts writes the pending access times and counts. SQLite's -// CURRENT_TIMESTAMP, which an access time is written as, counts whole -// seconds. -const pendingWriteInterval = time.Second - -// StartPendingCountWrites starts the goroutine that writes, every -// pendingWriteInterval, the pending access times in one transaction and then -// the pending counts in one UPDATE. Requests only hand these to it, so none -// waits for the database to record what it did, and only this goroutine's -// write ever waits for it. It is a no-op when already started. The goroutine -// outlives the caller, so it runs with its own context, which -// StopPendingCountWrites cancels to tell it to finish. -func (c *Cache) StartPendingCountWrites() { - if c.pendingCountsCancel != nil { - return - } - - ctx, cancel := context.WithCancel(context.Background()) - c.pendingCountsCancel = cancel - - go func() { - c.pendingCountWriteLoop(ctx) - }() -} - -// StopPendingCountWrites tells the goroutine StartPendingCountWrites -// started, if it did, to finish, and waits for its write under way to end -// and for it to return. It then writes the pending access times and counts -// left, once, so that they are written before the database closes. It waits -// for all of this at most until ctx ends. It logs what was not written, -// access times or counts, and returns an error. -func (c *Cache) StopPendingCountWrites(ctx context.Context) error { - var accessTimesErr, countsErr error - - if c.pendingCountsCancel != nil { - c.pendingCountsCancel() - - select { - case <-c.pendingCountsDone: - case <-ctx.Done(): - // The write given up on here, of access times or of counts, - // may not reach the database: database/sql closes without - // waiting for a query, and a write still waiting for the - // connection fails when the database closes. Neither is - // written here. - accessTimesErr = fmt.Errorf("access times still being written: %w", ctx.Err()) - countsErr = fmt.Errorf("pending counts still being written: %w", ctx.Err()) - } - } - - if accessTimesErr == nil && countsErr == nil { - accessTimesErr = c.writePendingAccessTimes(ctx) - countsErr = c.writePendingCounts(ctx) - } - - if accessTimesErr != nil { - c.log.Error("access times not written at shutdown", "error", accessTimesErr) - } - - if countsErr != nil { - c.log.Error("counts not written at shutdown", - "hits", c.pendingHits.Load(), - "misses", c.pendingMisses.Load(), - "upstream_fetches", c.pendingUpstreamFetches.Load(), - "upstream_fetch_bytes", c.pendingUpstreamFetchBytes.Load(), - "transforms", c.pendingTransforms.Load(), - "error", countsErr, - ) - } - - return errors.Join(accessTimesErr, countsErr) -} - -// pendingCountWriteLoop is the body of the goroutine -// StartPendingCountWrites starts. Once ctx is cancelled it starts no write -// and returns, but does not cut off a write under way then. Counts whose -// write fails stay pending, for the next write; access times whose write -// fails are dropped. -func (c *Cache) pendingCountWriteLoop(ctx context.Context) { - defer close(c.pendingCountsDone) - - ticker := time.NewTicker(pendingWriteInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - - // select picks at random when ctx is cancelled and a tick waits - // too. - if ctx.Err() != nil { - return - } - - err := c.writePendingAccessTimes(context.WithoutCancel(ctx)) - if err != nil { - c.log.Warn("failed to write access times", "error", err) - } - - err = c.writePendingCounts(context.WithoutCancel(ctx)) - if err != nil { - c.log.Warn("failed to write pending counts", "error", err) - } - } -} - -// writePendingAccessTimes takes out the variants waiting in -// pendingVariantAccessTimes and the sources waiting in -// pendingSourceAccessTimes, and sets their last_accessed_at to now in one -// transaction, once for each however often it was used. Nothing else takes -// from those channels while it runs: only the goroutine -// StartPendingCountWrites starts calls it, or StopPendingCountWrites once -// that goroutine has returned. -func (c *Cache) writePendingAccessTimes(ctx context.Context) error { - variants := make(map[imagerequest.VariantKey]struct{}) - - for range len(c.pendingVariantAccessTimes) { - variants[<-c.pendingVariantAccessTimes] = struct{}{} - } - - sources := make(map[imagerequest.SourceKey]struct{}) - - for range len(c.pendingSourceAccessTimes) { - sources[<-c.pendingSourceAccessTimes] = struct{}{} - } - - accessTimes := len(variants) + len(sources) - if accessTimes == 0 { - return nil - } - - tx, err := c.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) - } - - // Does nothing once the transaction is committed - defer func() { _ = tx.Rollback() }() - - for cacheKey := range variants { - _, err = tx.ExecContext(ctx, ` - UPDATE variant_content SET last_accessed_at = CURRENT_TIMESTAMP - WHERE cache_key = ? - `, string(cacheKey)) - if err != nil { - return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) - } - } - - for sourceKey := range sources { - _, err = tx.ExecContext(ctx, ` - UPDATE source_content SET last_accessed_at = CURRENT_TIMESTAMP - WHERE source_key = ? - `, string(sourceKey)) - if err != nil { - return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) - } - } - - err = tx.Commit() - if err != nil { - return fmt.Errorf("failed to write %d access times: %w", accessTimes, err) - } - - return nil -} - -// writePendingCounts adds the pending counts to the cache_stats row in one -// UPDATE, then takes what it wrote out of them, leaving any added -// meanwhile. When the UPDATE fails, they all stay pending. -func (c *Cache) writePendingCounts(ctx context.Context) error { - c.pendingCountsWriteMutex.Lock() - defer c.pendingCountsWriteMutex.Unlock() - - hits := c.pendingHits.Load() - misses := c.pendingMisses.Load() - upstreamFetches := c.pendingUpstreamFetches.Load() - upstreamFetchBytes := c.pendingUpstreamFetchBytes.Load() - transforms := c.pendingTransforms.Load() - - if hits+misses+upstreamFetches+upstreamFetchBytes+transforms == 0 { - return nil - } - - _, err := c.db.ExecContext(ctx, ` - UPDATE cache_stats - SET hit_count = hit_count + ?, - miss_count = miss_count + ?, - upstream_fetch_count = upstream_fetch_count + ?, - upstream_fetch_bytes = upstream_fetch_bytes + ?, - transform_count = transform_count + ?, - last_updated_at = CURRENT_TIMESTAMP - WHERE id = 1 - `, hits, misses, upstreamFetches, upstreamFetchBytes, transforms) - if err != nil { - return fmt.Errorf("failed to write pending counts: %w", err) - } - - c.pendingHits.Add(-hits) - c.pendingMisses.Add(-misses) - c.pendingUpstreamFetches.Add(-upstreamFetches) - c.pendingUpstreamFetchBytes.Add(-upstreamFetchBytes) - c.pendingTransforms.Add(-transforms) - - return nil -} diff --git a/internal/imgcache/pendingcounts_internal_test.go b/internal/imgcache/pendingcounts_internal_test.go deleted file mode 100644 index 81a3b15..0000000 --- a/internal/imgcache/pendingcounts_internal_test.go +++ /dev/null @@ -1,431 +0,0 @@ -package imgcache - -import ( - "bytes" - "context" - "database/sql" - "errors" - "log/slog" - "strings" - "testing" - "time" - - "sneak.berlin/go/pixa/internal/imagerequest" -) - -// pendingCountsWait is how long a test waits for the database's connection, -// for a count to reach the database, for a write to start waiting for the -// connection, for a request to return past its deadline, or for -// StopPendingCountWrites to return. -const pendingCountsWait = 5 * time.Second - -// otherWritesWait is how long a test gives other writes, once one write waits -// for the database's connection, to start waiting for it too. -const otherWritesWait = 100 * time.Millisecond - -// holdDatabase takes the one connection of cache's database, so that every -// other query waits for it, and returns the func that frees it. -func holdDatabase(t *testing.T, cache *Cache) func() { - t.Helper() - - ctx, cancel := context.WithTimeout(t.Context(), pendingCountsWait) - defer cancel() - - conn, err := cache.db.Conn(ctx) - if err != nil { - t.Fatalf("failed to take the database connection: %v", err) - } - - return func() { _ = conn.Close() } -} - -// waitForConnectionWaits waits until db.Stats().WaitCount, the number of -// times a caller has waited for a connection, reaches waits. -func waitForConnectionWaits(t *testing.T, db *sql.DB, waits int64) { - t.Helper() - - deadline := time.Now().Add(pendingCountsWait) - - for db.Stats().WaitCount < waits { - if time.Now().After(deadline) { - t.Fatalf("callers waited for the database connection %d times, want %d", - db.Stats().WaitCount, waits) - } - - time.Sleep(10 * time.Millisecond) - } -} - -// waitForCounters waits until the cache_stats row holds want. -func waitForCounters(t *testing.T, cache *Cache, want cacheStatsCounters) { - t.Helper() - - deadline := time.Now().Add(pendingCountsWait) - - for { - got := readCacheStatsCounters(t, cache) - if got == want { - return - } - - if time.Now().After(deadline) { - t.Fatalf("counters = %+v, want %+v", got, want) - } - - time.Sleep(10 * time.Millisecond) - } -} - -// waitForPendingCountsWritten starts the goroutine that writes cache's -// pending counts, unless it is running, and waits until it has written them -// all, so that the database holds every count made so far. The goroutine is -// stopped when the test ends. The database is limited to one connection -// first: each connection to an in-memory SQLite database opens a new, empty -// one, so the goroutine must share the connection that holds the schema. -func waitForPendingCountsWritten(t *testing.T, cache *Cache) { - t.Helper() - - if cache.pendingCountsCancel == nil { - cache.db.SetMaxOpenConns(1) - cache.StartPendingCountWrites() - - t.Cleanup(func() { _ = cache.StopPendingCountWrites(context.Background()) }) - } - - deadline := time.Now().Add(pendingCountsWait) - - for cache.pendingHits.Load()+cache.pendingMisses.Load()+ - cache.pendingUpstreamFetches.Load()+cache.pendingUpstreamFetchBytes.Load()+ - cache.pendingTransforms.Load() > 0 { - if time.Now().After(deadline) { - t.Fatal("the pending counts were not written") - } - - time.Sleep(10 * time.Millisecond) - } -} - -// TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the -// database's one connection while a request whose fetch is held reaches its -// deadline. The request must return with the deadline's error while the -// connection is still held, as it waits for no database write past its -// deadline, and its miss must reach the database once the connection is free. -func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { - t.Parallel() - - svc, fixtures, fetcher := setupHeldFetchService(t) - - svc.cache.StartPendingCountWrites() - defer func() { _ = svc.cache.StopPendingCountWrites(t.Context()) }() - - // Room for the request to reach its fetch on a busy host - const timeout = 2 * time.Second - - ctx, cancel := context.WithTimeout(t.Context(), timeout) - defer cancel() - - deadline, _ := ctx.Deadline() - - results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - - // The request has made its database reads by the time it fetches. - select { - case <-fetcher.started: - case <-time.After(timeout): - t.Fatal("request did not reach its fetch by its deadline") - } - - releaseDatabase := holdDatabase(t, svc.cache) - defer releaseDatabase() - - select { - case got := <-results: - t.Logf("Get() returned %v after its deadline, error = %v", - time.Since(deadline), got.err) - - if !errors.Is(got.err, context.DeadlineExceeded) { - t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded) - } - case <-time.After(time.Until(deadline) + pendingCountsWait): - t.Fatal("request did not return while the database was busy") - } - - releaseDatabase() - - waitForCounters(t, svc.cache, cacheStatsCounters{missCount: 1}) -} - -// TestIncrementStats_ManyCountsLeaveOneWriteWaiting holds the database's one -// connection while many misses are added to the pending counts, which the -// goroutine StartPendingCountWrites starts writes to the database. At most one -// write may then wait for the connection, and every miss must reach the -// database once it is free. -func TestIncrementStats_ManyCountsLeaveOneWriteWaiting(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - cache.StartPendingCountWrites() - defer func() { _ = cache.StopPendingCountWrites(t.Context()) }() - - releaseDatabase := holdDatabase(t, cache) - defer releaseDatabase() - - waitsBefore := cache.db.Stats().WaitCount - - const misses = 20 - - for range misses { - cache.IncrementStats(false, 0) - } - - // Once one write waits for the connection, any others start waiting too. - waitForConnectionWaits(t, cache.db, waitsBefore+1) - time.Sleep(otherWritesWait) - - waits := cache.db.Stats().WaitCount - waitsBefore - t.Logf("%d writes waited for the database", waits) - - if waits > 1 { - t.Errorf("%d writes waited for the database, want at most 1", waits) - } - - releaseDatabase() - - waitForCounters(t, cache, cacheStatsCounters{missCount: misses}) -} - -// TestStats_IncludesPendingCounts adds hits and misses to the pending counts, -// with no goroutine started to write them, and checks that Stats adds them to -// the database's counts before they are written. -func TestStats_IncludesPendingCounts(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - _, err := cache.db.ExecContext(t.Context(), - `UPDATE cache_stats SET hit_count = 75, miss_count = 25 WHERE id = 1`) - if err != nil { - t.Fatal(err) - } - - cache.IncrementStats(true, 0) - cache.IncrementStats(true, 0) - cache.IncrementStats(false, 0) - - stats, err := cache.Stats(t.Context()) - if err != nil { - t.Fatalf("Stats() error = %v", err) - } - - if stats.HitCount != 77 || stats.MissCount != 26 { - t.Errorf("HitCount = %d, MissCount = %d, want 77 and 26", - stats.HitCount, stats.MissCount) - } - - want := cacheStatsCounters{hitCount: 75, missCount: 25} - if got := readCacheStatsCounters(t, cache); got != want { - t.Errorf("counters in the database = %+v, want %+v", got, want) - } -} - -// TestStopPendingCountWrites_WritesPendingCounts holds the database's one -// connection while the goroutine that writes the pending counts waits for it, -// counts more, then stops that goroutine. The stop must leave the write under -// way to finish rather than cut it off, and once the connection is free it -// must write the counts that write left. -func TestStopPendingCountWrites_WritesPendingCounts(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - cache.StartPendingCountWrites() - defer func() { _ = cache.StopPendingCountWrites(t.Context()) }() - - releaseDatabase := holdDatabase(t, cache) - defer releaseDatabase() - - waitsBefore := cache.db.Stats().WaitCount - - cache.IncrementStats(true, 0) - - // The goroutine's write waits for the connection. - waitForConnectionWaits(t, cache.db, waitsBefore+1) - - // Counted after that write read the pending counts - cache.IncrementStats(false, 0) - cache.IncrementUpstreamFetch(1024) - cache.IncrementTransformCount() - - // The goroutine is told to finish here as well as by the stop, so that it - // is told before its write under way ends, however late the stop runs. - // The counts above then reach the database only through the stop's own - // write. - cache.pendingCountsCancel() - - stopped := make(chan error, 1) - - go func() { - stopped <- cache.StopPendingCountWrites(t.Context()) - }() - - // Had the stop cut off the write under way, its own write would wait for - // the connection too. - time.Sleep(otherWritesWait) - - waits := cache.db.Stats().WaitCount - waitsBefore - t.Logf("%d writes waited for the database", waits) - - if waits > 1 { - t.Errorf("%d writes waited for the database, want 1: "+ - "the stop cut off the write under way", waits) - } - - releaseDatabase() - - select { - case err := <-stopped: - if err != nil { - t.Fatalf("StopPendingCountWrites() error = %v", err) - } - case <-time.After(pendingCountsWait): - t.Fatal("StopPendingCountWrites() did not return once the database was free") - } - - want := cacheStatsCounters{1, 1, 1, 1024, 1} - if got := readCacheStatsCounters(t, cache); got != want { - t.Errorf("counters = %+v, want %+v", got, want) - } -} - -// TestStopPendingCountWrites_ReturnsWhenItsContextEnds holds the database's -// one connection while the goroutine that writes the pending counts waits for -// it, then stops that goroutine with a context that ends first. The stop must -// return the context's error, and the write under way must still reach the -// database once the connection is free. -func TestStopPendingCountWrites_ReturnsWhenItsContextEnds(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - cache.StartPendingCountWrites() - defer func() { _ = cache.StopPendingCountWrites(t.Context()) }() - - releaseDatabase := holdDatabase(t, cache) - defer releaseDatabase() - - waitsBefore := cache.db.Stats().WaitCount - - cache.IncrementStats(true, 0) - - // The goroutine's write waits for the connection. - waitForConnectionWaits(t, cache.db, waitsBefore+1) - - stopCtx, cancelStop := context.WithCancel(t.Context()) - cancelStop() - - stopped := make(chan error, 1) - - go func() { - stopped <- cache.StopPendingCountWrites(stopCtx) - }() - - select { - case err := <-stopped: - if !errors.Is(err, context.Canceled) { - t.Errorf("StopPendingCountWrites() error = %v, want %v", - err, context.Canceled) - } - case <-time.After(pendingCountsWait): - t.Fatal("StopPendingCountWrites() did not return once its context ended") - } - - releaseDatabase() - - waitForCounters(t, cache, cacheStatsCounters{hitCount: 1}) -} - -// TestLookupSource_HandsAccessTimeToTheBackgroundWriter finds a cached source -// whose access time is an hour old. LookupSource must not write the access -// time itself but hand it over to be written, and StopPendingCountWrites must -// then write it. -func TestLookupSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg", - []byte("source bytes")) - anHourAgo := time.Now().Add(-time.Hour) - setSourceLastAccessed(t, cache, key, anHourAgo) - - touchedQuery := ` - SELECT COUNT(*) FROM source_content - WHERE source_key = ? AND last_accessed_at > ? - ` - anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat) - - found, err := cache.LookupSource(t.Context(), &imagerequest.Request{ - Source: imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}, - }) - if err != nil || found != key { - t.Fatalf("LookupSource() = %q, %v; want %q", found, err, key) - } - - if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 0 { - t.Error("LookupSource wrote the source's access time itself") - } - - if waiting := len(cache.pendingSourceAccessTimes); waiting != 1 { - t.Errorf("%d access times of sources wait to be written, want 1", waiting) - } - - err = cache.StopPendingCountWrites(t.Context()) - if err != nil { - t.Fatalf("StopPendingCountWrites() error = %v", err) - } - - if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 1 { - t.Error("the source's access time was not written") - } -} - -// TestStopPendingCountWrites_LogsWhatWasNotWritten makes the write of the -// access times fail at shutdown, by dropping the table they are written to, -// while a hit count is written. The stop must return an error and log that -// access times were not written, and must not log the counts as not written. -func TestStopPendingCountWrites_LogsWhatWasNotWritten(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - var logs bytes.Buffer - - cache.log = slog.New(slog.NewJSONHandler(&logs, nil)) - - cache.touchVariant(testVariantKeyOne) - cache.IncrementStats(true, 0) - - _, err := cache.db.ExecContext(t.Context(), `DROP TABLE variant_content`) - if err != nil { - t.Fatalf("dropping variant_content: %v", err) - } - - err = cache.StopPendingCountWrites(t.Context()) - t.Logf("StopPendingCountWrites() error = %v; logged:\n%s", err, logs.String()) - - if err == nil { - t.Error("StopPendingCountWrites() error = nil, want the access times' error") - } - - if !strings.Contains(logs.String(), "access times not written at shutdown") { - t.Error("the access times not written were not logged") - } - - if strings.Contains(logs.String(), "counts not written at shutdown") { - t.Error("the counts, which were written, were logged as not written") - } - - if got := readCacheStatsCounters(t, cache); got.hitCount != 1 { - t.Errorf("hit count in the database = %d, want 1", got.hitCount) - } -} diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 46a3653..fa564cd 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -142,8 +142,8 @@ var ErrProcessingTimedOut = errors.New( // only a request that needs the source fetched is refused. // // A cache hit opens the variant file and makes no database call: its access -// time and its hit count are handed to the goroutine that -// StartPendingCountWrites starts, without waiting for it. A variant file +// time is handed to the goroutine that StartPendingAccessTimeWrites starts, +// without waiting for it, and its hit is counted in memory. A variant file // that cannot be served, such as one whose first bytes are not those of an // output format, is a miss, and the variant is made again. func (s *Service) Get( @@ -196,11 +196,6 @@ func (s *Service) Warm(ctx context.Context, req *imagerequest.Request) error { return err } -// Stats returns cache statistics. -func (s *Service) Stats(ctx context.Context) (*CacheStats, error) { - return s.cache.Stats(ctx) -} - // WaitForProcessing waits until no image is being processed, or until ctx // ends, and returns how many images were still being processed then. func (s *Service) WaitForProcessing(ctx context.Context) int { diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index cab3f2a..31573e1 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -595,9 +595,7 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { _ = resp.Content.Close() - waitForPendingCountsWritten(t, svc.cache) - - fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount + fetches := readCounters(svc.cache).upstreamFetchCount if fetches != 1 { t.Errorf("upstream fetches = %d, want 1", fetches) } diff --git a/internal/imgcache/stats_internal_test.go b/internal/imgcache/stats_internal_test.go index d1da15b..6a90fbe 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imgcache/stats_internal_test.go @@ -3,245 +3,18 @@ package imgcache import ( "bytes" "context" - "database/sql" "image/color" "io" "io/fs" - "log/slog" - "math" - "strings" "testing" - "time" - "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imagerequest" ) -func setupStatsTestDB(t *testing.T) *sql.DB { - t.Helper() - - db, err := sql.Open("sqlite", ":memory:") - if err != nil { - t.Fatal(err) - } - - err = database.ApplyMigrations(context.Background(), db, nil) - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { _ = db.Close() }) - - return db -} - -func TestStats_HitRateIsRatio(t *testing.T) { - t.Parallel() - - db := setupStatsTestDB(t) - dir := t.TempDir() - - cache, err := NewCache(db, CacheConfig{ - StateDir: dir, - NegativeTTL: 5 * time.Minute, - }) - if err != nil { - t.Fatal(err) - } - - ctx := context.Background() - - // Set some hit/miss counts and a transform_count - _, err = db.ExecContext(ctx, ` - UPDATE cache_stats - SET hit_count = 75, miss_count = 25, transform_count = 9999 - WHERE id = 1 - `) - if err != nil { - t.Fatal(err) - } - - stats, err := cache.Stats(ctx) - if err != nil { - t.Fatal(err) - } - - if stats.HitCount != 75 { - t.Errorf("HitCount = %d, want 75", stats.HitCount) - } - - if stats.MissCount != 25 { - t.Errorf("MissCount = %d, want 25", stats.MissCount) - } - - // HitRate should be 0.75, NOT 9999 (transform_count) - expectedRate := 0.75 - if math.Abs(stats.HitRate-expectedRate) > 0.001 { - t.Errorf("HitRate = %f, want %f (was it scanning transform_count?)", - stats.HitRate, expectedRate) - } -} - -func TestStats_ZeroCounts(t *testing.T) { - t.Parallel() - - db := setupStatsTestDB(t) - dir := t.TempDir() - - cache, err := NewCache(db, CacheConfig{ - StateDir: dir, - NegativeTTL: 5 * time.Minute, - }) - if err != nil { - t.Fatal(err) - } - - stats, err := cache.Stats(context.Background()) - if err != nil { - t.Fatal(err) - } - - // With zero hits and misses, HitRate should be 0, not some garbage value - if stats.HitRate != 0.0 { - t.Errorf("HitRate = %f, want 0.0 for zero counts", stats.HitRate) - } -} - -// TestStats_LogsFailedCountQueries verifies that a failed item count query -// and a failed size query are each logged at warn and Stats still succeeds. -func TestStats_LogsFailedCountQueries(t *testing.T) { - t.Parallel() - - db := setupStatsTestDB(t) - - var logBuf bytes.Buffer - - cache, err := NewCache(db, CacheConfig{ - StateDir: t.TempDir(), - NegativeTTL: 5 * time.Minute, - Logger: slog.New(slog.NewJSONHandler(&logBuf, nil)), - }) - if err != nil { - t.Fatal(err) - } - - _, err = db.ExecContext(t.Context(), - `DROP TABLE source_content; DROP TABLE variant_content`) - if err != nil { - t.Fatal(err) - } - - _, err = cache.Stats(t.Context()) - if err != nil { - t.Fatalf("Stats() error = %v, want nil", err) - } - - for _, msg := range []string{ - "failed to count cache items for stats", - "failed to sum cache size for stats", - } { - want := `"level":"WARN","msg":"` + msg + `"` - if !strings.Contains(logBuf.String(), want) { - t.Errorf("log missing %s; got %q", want, logBuf.String()) - } - } -} - -// TestStats_TotalsCountSourcesAndVariants verifies that TotalItems and -// TotalSizeBytes cover the stored source images and processed variants. -func TestStats_TotalsCountSourcesAndVariants(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - storeEvictionTestSource(t, cache, testHostCDN, testPathCat, - bytes.Repeat([]byte{0xAA}, 1000)) - storeEvictionTestVariant(t, cache, testVariantKeyOne, - bytes.Repeat([]byte{0xAB}, 500)) - storeEvictionTestVariant(t, cache, testVariantKeyTwo, - bytes.Repeat([]byte{0xAC}, 250)) - - stats, err := cache.Stats(t.Context()) - if err != nil { - t.Fatalf("Stats() error = %v", err) - } - - if stats.TotalItems != 3 { - t.Errorf("TotalItems = %d, want 3 (1 source, 2 variants)", stats.TotalItems) - } - - if stats.TotalSizeBytes != 1750 { - t.Errorf("TotalSizeBytes = %d, want 1750 (1000+500+250)", - stats.TotalSizeBytes) - } -} - -// TestStats_DisabledCacheReportsNoItems verifies that a disabled disk cache -// reports no items and no size, even when its database still holds the -// rows of an earlier run with the disk cache enabled. -func TestStats_DisabledCacheReportsNoItems(t *testing.T) { - t.Parallel() - - enabled, _ := newEvictionTestCache(t, 1<<30) - - storeEvictionTestSource(t, enabled, testHostCDN, testPathCat, - bytes.Repeat([]byte{0xAA}, 1000)) - storeEvictionTestVariant(t, enabled, testVariantKeyOne, - bytes.Repeat([]byte{0xAB}, 500)) - - disabled, err := NewCache(enabled.db, CacheConfig{ - StateDir: t.TempDir(), - NegativeTTL: 5 * time.Minute, - DisableDiskCache: true, - }) - if err != nil { - t.Fatal(err) - } - - stats, err := disabled.Stats(t.Context()) - if err != nil { - t.Fatalf("Stats() error = %v", err) - } - - if stats.TotalItems != 0 || stats.TotalSizeBytes != 0 { - t.Errorf("TotalItems = %d, TotalSizeBytes = %d, want 0 and 0", - stats.TotalItems, stats.TotalSizeBytes) - } -} - -// cacheStatsCounters holds the counters of the cache_stats row, in column -// order. -type cacheStatsCounters struct { - hitCount int64 - missCount int64 - upstreamFetchCount int64 - upstreamFetchBytes int64 - transformCount int64 -} - -// readCacheStatsCounters reads the counters of the cache_stats row. -func readCacheStatsCounters(t *testing.T, cache *Cache) cacheStatsCounters { - t.Helper() - - var got cacheStatsCounters - - err := cache.db.QueryRowContext(t.Context(), ` - SELECT hit_count, miss_count, upstream_fetch_count, - upstream_fetch_bytes, transform_count - FROM cache_stats WHERE id = 1 - `).Scan(&got.hitCount, &got.missCount, &got.upstreamFetchCount, - &got.upstreamFetchBytes, &got.transformCount) - if err != nil { - t.Fatalf("failed to read cache_stats: %v", err) - } - - return got -} - // TestService_Get_CountsStats walks Get through a miss that fetches the // source, a hit, a miss that reuses the cached source, and two misses whose -// source cannot be used, checking every cache_stats counter after each. +// source cannot be used, checking every counter of the cache after each. func TestService_Get_CountsStats(t *testing.T) { t.Parallel() @@ -268,18 +41,18 @@ func TestService_Get_CountsStats(t *testing.T) { path string size int wantErr bool - want cacheStatsCounters + want counterValues }{ {"miss that fetches the source", testPathPhoto, 50, false, - cacheStatsCounters{0, 1, 1, photoBytes, 1}}, + counterValues{0, 1, 1, photoBytes, 1}}, {"hit", testPathPhoto, 50, false, - cacheStatsCounters{1, 1, 1, photoBytes, 1}}, + counterValues{1, 1, 1, photoBytes, 1}}, {"miss that reuses the cached source", testPathPhoto, 25, false, - cacheStatsCounters{1, 2, 1, photoBytes, 2}}, + counterValues{1, 2, 1, photoBytes, 2}}, {"miss whose source fails the magic byte check", "/images/fake.jpg", 50, true, - cacheStatsCounters{1, 3, 2, photoBytes + fakeBytes, 2}}, + counterValues{1, 3, 2, photoBytes + fakeBytes, 2}}, {"miss whose source is not found", "/images/nonexistent.jpg", 50, true, - cacheStatsCounters{1, 4, 2, photoBytes + fakeBytes, 2}}, + counterValues{1, 4, 2, photoBytes + fakeBytes, 2}}, } for _, step := range steps { @@ -303,9 +76,7 @@ func TestService_Get_CountsStats(t *testing.T) { _ = resp.Content.Close() } - waitForPendingCountsWritten(t, svc.cache) - - got := readCacheStatsCounters(t, svc.cache) + got := readCounters(svc.cache) if got != step.want { t.Fatalf("after the %s: counters = %+v, want %+v", step.name, got, step.want) } @@ -343,7 +114,7 @@ func (u *fakeUpstream) Read(p []byte) (int, error) { return 0, u.err } -// TestService_Get_CountsInterruptedMisses checks every cache_stats counter +// TestService_Get_CountsInterruptedMisses checks every counter of the cache // after a miss whose request context ends during or after the upstream // fetch, and after a miss whose upstream body is over the size limit. func TestService_Get_CountsInterruptedMisses(t *testing.T) { @@ -359,15 +130,15 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { cancel bool // whether the body then ends the request context readErr error // what the body then returns wantErr bool - want cacheStatsCounters + want counterValues }{ {"request context ends during the fetch", half, true, context.Canceled, true, - cacheStatsCounters{0, 1, 1, int64(half), 0}}, + counterValues{0, 1, 1, int64(half), 0}}, {"request context ends after the fetch", len(photo), true, io.EOF, false, - cacheStatsCounters{0, 1, 1, int64(len(photo)), 1}}, + counterValues{0, 1, 1, int64(len(photo)), 1}}, {"upstream body over the size limit", half, false, httpfetcher.ErrResponseTooLarge, true, - cacheStatsCounters{0, 1, 1, int64(half), 0}}, + counterValues{0, 1, 1, int64(half), 0}}, } for _, tc := range tests { @@ -411,9 +182,7 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { _ = resp.Content.Close() } - waitForPendingCountsWritten(t, svc.cache) - - got := readCacheStatsCounters(t, svc.cache) + got := readCounters(svc.cache) if got != tc.want { t.Errorf("counters = %+v, want %+v", got, tc.want) } @@ -421,9 +190,9 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { } } -// TestService_Get_CountsHitAfterRequestEnds checks every cache_stats counter -// after a hit served with a request context that has already ended: only -// the hit count moves. +// TestService_Get_CountsHitAfterRequestEnds checks every counter of the +// cache after a hit served with a request context that has already ended: +// only the hit count moves. func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { t.Parallel() @@ -450,9 +219,7 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { _ = resp.Content.Close() - waitForPendingCountsWritten(t, svc.cache) - - want := readCacheStatsCounters(t, svc.cache) + want := readCounters(svc.cache) want.hitCount++ ctx, cancel := context.WithCancel(t.Context()) @@ -469,9 +236,7 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { t.Fatalf("CacheStatus = %v, want %v", resp.CacheStatus, CacheHit) } - waitForPendingCountsWritten(t, svc.cache) - - got := readCacheStatsCounters(t, svc.cache) + got := readCounters(svc.cache) if got != want { t.Errorf("counters = %+v, want %+v", got, want) } diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index e813080..bd5ad8b 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -14,6 +14,7 @@ import ( "testing/fstest" "time" + "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" ) @@ -184,7 +185,7 @@ func SetupTestService( cache, err := NewCache(db, CacheConfig{ StateDir: tmpDir, NegativeTTL: 5 * time.Minute, - }) + }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go index 2a4c25e..ace59d5 100644 --- a/internal/server/image_proxy_internal_test.go +++ b/internal/server/image_proxy_internal_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "go.uber.org/fx/fxtest" @@ -170,6 +171,7 @@ func startImageProxy( handlers.New, middleware.New, func() httpfetcher.Fetcher { return fetcher }, + func() prometheus.Registerer { return prometheus.NewRegistry() }, ), fx.Populate(&h, &mw, &db), ) @@ -223,6 +225,7 @@ func startImageProxyWithBlockedNetworks( healthcheck.New, handlers.New, middleware.New, + func() prometheus.Registerer { return prometheus.NewRegistry() }, ), fx.Populate(&h, &mw), ) diff --git a/internal/server/login_rate_limit_internal_test.go b/internal/server/login_rate_limit_internal_test.go index f5d8c81..e55fb79 100644 --- a/internal/server/login_rate_limit_internal_test.go +++ b/internal/server/login_rate_limit_internal_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx/fxtest" "sneak.berlin/go/pixa/internal/config" @@ -81,6 +82,7 @@ func newTestServer(t *testing.T) *Server { h, err := handlers.New(lc, handlers.Params{ Logger: log, Healthcheck: hc, Database: db, Config: cfg, + MetricsRegistry: prometheus.NewRegistry(), }) if err != nil { t.Fatalf("handlers.New() error = %v", err) diff --git a/internal/server/logout_internal_test.go b/internal/server/logout_internal_test.go index 2d46d76..d518743 100644 --- a/internal/server/logout_internal_test.go +++ b/internal/server/logout_internal_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "go.uber.org/fx/fxtest" @@ -46,6 +47,7 @@ func startPixa(t *testing.T, cfg *config.Config) *Server { healthcheck.New, handlers.New, middleware.New, + func() prometheus.Registerer { return prometheus.NewRegistry() }, ), fx.Populate(&h, &mw), ) From ac2bae04898c5cddf2e16653b0e7871d0473af58 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 10:54:14 +0100 Subject: [PATCH 102/131] Remove the code nothing in production calls (closes #312) Service.Warm, the ImageCache interface, both GenerateSignedURL methods, signature.ParseParams, imagerequest.ParseImagePath, four symbols of internal/magic, the image processor's lists of supported formats, IsEmpty and Count of the allowlist, ssrfSafeDialer, CacheStale, ErrCacheMiss, ErrNegativeCache, ImageResponse.LastModified and the Load methods of ContentStorage and VariantStorage had no caller outside tests. They are gone, with the tests that tested only them. The handlers tests sign their image URLs with generateSignedURL, a test helper that builds the URL as the removed generator did. The SSRF dialer tests call dialSSRFSafe with no operator blocklist, which is all ssrfSafeDialer did. The tests of ParseImagePath and ContentStorage.Load call ParseImageURL and LoadWithSize instead. MIMEToImageFormat, ContentStorage.Store and CleanExpired stay, as pixa calls them. Model: opus-5-5 --- TODO.md | 16 ++ internal/allowlist/allowlist.go | 10 - internal/allowlist/allowlist_test.go | 84 ------- .../handlers/format_auto_internal_test.go | 16 +- internal/handlers/handlers_internal_test.go | 2 - .../image_cache_control_internal_test.go | 7 +- .../handlers/image_signature_internal_test.go | 146 +++++++----- internal/handlers/request_id_internal_test.go | 13 +- internal/httpfetcher/fetch_internal_test.go | 8 +- internal/httpfetcher/httpfetcher.go | 7 - internal/imageprocessor/imageprocessor.go | 24 -- .../imageprocessor_internal_test.go | 16 -- internal/imagerequest/urlparser.go | 17 -- .../imagerequest/urlparser_internal_test.go | 120 ++-------- internal/imgcache/cache.go | 6 - internal/imgcache/imgcache.go | 28 +-- .../partial_cache_files_internal_test.go | 4 +- internal/imgcache/service.go | 34 +-- internal/imgcache/storage.go | 32 --- internal/imgcache/storage_internal_test.go | 14 +- internal/magic/jpegxl_internal_test.go | 11 +- internal/magic/magic.go | 77 +------ internal/magic/magic_internal_test.go | 176 --------------- internal/server/source_url_internal_test.go | 33 +-- internal/signature/golden_test.go | 36 +-- internal/signature/signature.go | 54 ----- internal/signature/signature_test.go | 213 +----------------- 27 files changed, 187 insertions(+), 1017 deletions(-) diff --git a/TODO.md b/TODO.md index a2785f7..8c2ff37 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,22 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 the code nothing in production called is gone (closes #312). The + handlers tests sign their image URLs with `generateSignedURL`, a test helper + that writes them as the removed generator did. Removed: + - `Service.Warm` and the `ImageCache` interface + - `Service.GenerateSignedURL` and `Signer.GenerateSignedURL` + - `signature.ParseParams` + - `imagerequest.ParseImagePath` + - `PeekAndValidate`, `IsSupportedMIMEType`, `ImageFormatToMIME` and + `FormatOriginal` of `internal/magic` + - `SupportedInputFormats` and `SupportedOutputFormats` of the image + processor + - `IsEmpty` and `Count` of the allowlist + - `ssrfSafeDialer` + - `CacheStale`, `ErrCacheMiss`, `ErrNegativeCache` and + `ImageResponse.LastModified` + - `ContentStorage.Load` and `VariantStorage.Load` - 2026-10-26 cache hits, misses, upstream fetches, the bytes those fetches read and transforms are counted as Prometheus counters in memory (closes #311). They are `pixa_cache_hits_total`, `pixa_cache_misses_total`, diff --git a/internal/allowlist/allowlist.go b/internal/allowlist/allowlist.go index 6dc2c4b..256c764 100644 --- a/internal/allowlist/allowlist.go +++ b/internal/allowlist/allowlist.go @@ -72,13 +72,3 @@ func (w *HostAllowList) IsAllowed(u *url.URL) bool { return false } - -// IsEmpty returns true if the allow list has no entries. -func (w *HostAllowList) IsEmpty() bool { - return len(w.exactHosts) == 0 && len(w.suffixHosts) == 0 -} - -// Count returns the total number of allow list entries. -func (w *HostAllowList) Count() int { - return len(w.exactHosts) + len(w.suffixHosts) -} diff --git a/internal/allowlist/allowlist_test.go b/internal/allowlist/allowlist_test.go index 1d68a14..a1730b9 100644 --- a/internal/allowlist/allowlist_test.go +++ b/internal/allowlist/allowlist_test.go @@ -139,87 +139,3 @@ func TestHostAllowList_IsAllowed_SuffixMatch(t *testing.T) { }, }) } - -func TestHostAllowList_IsEmpty(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - patterns []string - want bool - }{ - { - name: "empty", - patterns: []string{}, - want: true, - }, - { - name: "nil", - patterns: nil, - want: true, - }, - { - name: "whitespace only", - patterns: []string{" ", ""}, - want: true, - }, - { - name: "has entries", - patterns: []string{"example.com"}, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - w := allowlist.New(tt.patterns) - if got := w.IsEmpty(); got != tt.want { - t.Errorf("IsEmpty() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestHostAllowList_Count(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - patterns []string - want int - }{ - { - name: "empty", - patterns: []string{}, - want: 0, - }, - { - name: "exact hosts only", - patterns: []string{"a.com", "b.com", "c.com"}, - want: 3, - }, - { - name: "suffix hosts only", - patterns: []string{".a.com", ".b.com"}, - want: 2, - }, - { - name: "mixed", - patterns: []string{"exact.com", ".suffix.com"}, - want: 2, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - w := allowlist.New(tt.patterns) - if got := w.Count(); got != tt.want { - t.Errorf("Count() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/internal/handlers/format_auto_internal_test.go b/internal/handlers/format_auto_internal_test.go index d899b36..c543902 100644 --- a/internal/handlers/format_auto_internal_test.go +++ b/internal/handlers/format_auto_internal_test.go @@ -75,11 +75,13 @@ func TestFormatForAccept(t *testing.T) { // autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL, // both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in -// the format auto, made with h's image service and generator. +// the format auto. The first is signed with testSigningKey, the key of the +// image service newSignedHostServer builds, and the second is made with h's +// generator. func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { t.Helper() - signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + signedURL := generateSignedURL(testSigningKey, &imagerequest.Request{ Source: imagerequest.Source{ Host: signedHost, Path: photoPath, @@ -89,9 +91,6 @@ func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { Format: imagerequest.FormatAuto, }, }, time.Minute) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } token, err := h.encGen.Generate(&imagerequest.Request{ Source: imagerequest.Source{ @@ -188,9 +187,9 @@ func TestFormatAuto_ChosenFromAccept(t *testing.T) { func TestFormatAuto_SignatureCoversAuto(t *testing.T) { t.Parallel() - h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + _, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + signedForAVIF := generateSignedURL(testSigningKey, &imagerequest.Request{ Source: imagerequest.Source{ Host: signedHost, Path: photoPath, @@ -200,9 +199,6 @@ func TestFormatAuto_SignatureCoversAuto(t *testing.T) { Format: imagerequest.FormatAVIF, }, }, time.Minute) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1) if target == signedForAVIF { diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index 82957fb..d175569 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -27,7 +27,6 @@ import ( // testFixtures contains test data. type testFixtures struct { handler *Handlers - service *imgcache.Service goodHost string } @@ -71,7 +70,6 @@ func setupTestHandler(t *testing.T) *testFixtures { return &testFixtures{ handler: h, - service: svc, goodHost: goodHost, } } diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index 0fc236f..79c4a38 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -107,9 +107,9 @@ func getMaxAge(t *testing.T, srv http.Handler, target string) int { func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) { t.Parallel() - h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + _, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) - signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + signedURL := generateSignedURL(testSigningKey, &imagerequest.Request{ Source: imagerequest.Source{ Host: signedHost, Path: photoPath, @@ -119,9 +119,6 @@ func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) { Format: imagerequest.FormatJPEG, }, }, time.Minute) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } maxAge := getMaxAge(t, srv, signedURL) if maxAge < 50 || maxAge > 60 { diff --git a/internal/handlers/image_signature_internal_test.go b/internal/handlers/image_signature_internal_test.go index 2849565..3d2b766 100644 --- a/internal/handlers/image_signature_internal_test.go +++ b/internal/handlers/image_signature_internal_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "net/url" "testing" "time" @@ -36,6 +37,43 @@ func getImage(t *testing.T, fix *testFixtures, target string) int { return rec.Code } +// generateSignedURL returns the /v1/image/ URL of req, signed with key and +// expiring after ttl, and sets req's expiry. An unset quality or fit of req is +// first set to 85 or cover, the values the route uses when a URL has no q or +// fit. The URL carries q and fit next to sig and exp, so the route verifies it +// for the quality and fit it was signed with. +func generateSignedURL( + key string, req *imagerequest.Request, ttl time.Duration, +) string { + if req.Transform.Quality == 0 { + req.Transform.Quality = 85 + } + + if req.Transform.FitMode == "" { + req.Transform.FitMode = imagerequest.FitCover + } + + req.Expires = time.Now().Add(ttl) + sig := signature.New(key).Sign(req) + + // A source query follows the source path after a "?". The image route + // reads the path decoded once, so the whole path is written in Go's own + // percent-encoding of a URL path, which writes "?" as %3F and "%" as %25, + // and leaves "/", ";", ",", "=" and "&" as they are. + sourcePathAndQuery := req.Source.Path + if req.Source.Query != "" { + sourcePathAndQuery += "?" + req.Source.Query + } + + unescapedPath := fmt.Sprintf("/v1/image/%s%s/%dx%d.%s", + req.Source.Host, sourcePathAndQuery, + req.Transform.Size.Width, req.Transform.Size.Height, req.Transform.Format) + + return fmt.Sprintf("%s?sig=%s&exp=%d&q=%d&fit=%s", + (&url.URL{Path: unescapedPath}).EscapedPath(), sig, req.Expires.Unix(), + req.Transform.Quality, req.Transform.FitMode) +} + // TestHandleImage_SignatureCoversQualityAndFit signs a URL for quality 85 // and fit cover, the values the route uses when a URL has no q or fit, and // sends that signature with each q and fit below. @@ -83,8 +121,8 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) { } } -// TestHandleImage_GeneratedSignedURLVerifies sends URLs built by the -// service's signed-URL generator to the route. +// TestHandleImage_GeneratedSignedURLVerifies sends URLs built by +// generateSignedURL to the route. func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) { t.Parallel() @@ -101,25 +139,21 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - fix := setupTestHandler(t) + signedURL := generateSignedURL("test-signing-key-must-be-32-chars", + &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: "/images/photo.jpg", + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + Quality: tt.quality, + FitMode: tt.fitMode, + }, + }, time.Hour) - signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{ - Source: imagerequest.Source{ - Host: signedHost, - Path: "/images/photo.jpg", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: tt.quality, - FitMode: tt.fitMode, - }, - }, time.Hour) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } - - status := getImage(t, fix, signedURL) + status := getImage(t, setupTestHandler(t), signedURL) if status != http.StatusBadGateway { t.Errorf("status = %d, want %d", status, http.StatusBadGateway) } @@ -178,10 +212,10 @@ func TestHandleImage_InvalidExp_Returns400(t *testing.T) { } // TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies sends URLs built -// by the service's signed-URL generator for source URLs with a query to the -// route, through a chi router: chi routes on the path as the client wrote it -// whenever that is not Go's own encoding of it. A 502 means the signature -// passed, so the route read the source path and query that were signed. +// by generateSignedURL for source URLs with a query to the route, through a +// chi router: chi routes on the path as the client wrote it whenever that is +// not Go's own encoding of it. A 502 means the signature passed, so the route +// read the source path and query that were signed. func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) { t.Parallel() @@ -201,24 +235,20 @@ func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - fix := setupTestHandler(t) + signedURL := generateSignedURL("test-signing-key-must-be-32-chars", + &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: photoPath, + Query: tt.query, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, + }, time.Hour) - signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{ - Source: imagerequest.Source{ - Host: signedHost, - Path: photoPath, - Query: tt.query, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - }, - }, time.Hour) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } - - status := getImage(t, fix, signedURL) + status := getImage(t, setupTestHandler(t), signedURL) if status != http.StatusBadGateway { t.Errorf("status = %d, want %d", status, http.StatusBadGateway) } @@ -280,7 +310,7 @@ func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) { // TestHandleImage_SourceURLHoldingPercent25Verifies signs a source path and a // source query that each hold %25, the percent-escape of a literal "%", and // sends each to the route written as the README says, with that "%" written as -// %25, and as GenerateSignedURL writes it. A 502 means the signature passed. +// %25, and as generateSignedURL writes it. A 502 means the signature passed. func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) { t.Parallel() @@ -326,27 +356,23 @@ func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) { } }) - t.Run(tt.name+" written by GenerateSignedURL", func(t *testing.T) { + t.Run(tt.name+" written by generateSignedURL", func(t *testing.T) { t.Parallel() - fix := setupTestHandler(t) + signedURL := generateSignedURL("test-signing-key-must-be-32-chars", + &imagerequest.Request{ + Source: imagerequest.Source{ + Host: signedHost, + Path: tt.sourcePath, + Query: tt.sourceQuery, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: 50, Height: 50}, + Format: imagerequest.FormatJPEG, + }, + }, time.Hour) - signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{ - Source: imagerequest.Source{ - Host: signedHost, - Path: tt.sourcePath, - Query: tt.sourceQuery, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - }, - }, time.Hour) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } - - status := getImage(t, fix, signedURL) + status := getImage(t, setupTestHandler(t), signedURL) if status != http.StatusBadGateway { t.Errorf("status = %d, want %d", status, http.StatusBadGateway) } diff --git a/internal/handlers/request_id_internal_test.go b/internal/handlers/request_id_internal_test.go index a785923..110d45c 100644 --- a/internal/handlers/request_id_internal_test.go +++ b/internal/handlers/request_id_internal_test.go @@ -17,11 +17,13 @@ import ( ) // signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the -// JPEG at photoPath on signedHost at 50x50, made with h's image service. -func signedPhotoURL(t *testing.T, h *Handlers) string { +// JPEG at photoPath on signedHost at 50x50. It is signed with testSigningKey, +// the key of the image service newSignedHostServer builds. It takes a Handlers +// it does not use, so that it has the type of encPhotoURL. +func signedPhotoURL(t *testing.T, _ *Handlers) string { t.Helper() - signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{ + return generateSignedURL(testSigningKey, &imagerequest.Request{ Source: imagerequest.Source{ Host: signedHost, Path: photoPath, @@ -31,11 +33,6 @@ func signedPhotoURL(t *testing.T, h *Handlers) string { Format: imagerequest.FormatJPEG, }, }, time.Minute) - if err != nil { - t.Fatalf("GenerateSignedURL() error = %v", err) - } - - return signedURL } // encPhotoURL returns an encrypted /v1/e/ URL, which never expires, for the diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index c443987..b6523a8 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -524,7 +524,7 @@ func TestFetchSemaphoreReleasedOnPartialReadClose(t *testing.T) { }) } -// The dial-time re-resolution in ssrfSafeDialer is what closes the DNS +// The dial-time re-resolution in dialSSRFSafe is what closes the DNS // rebinding window: even if validateURL saw a public answer earlier, the // dialer independently re-checks the address it is about to connect to. A full // rebinding simulation (a resolver returning public, then private) would mean @@ -544,9 +544,9 @@ func TestSSRFSafeDialerBlocksPrivateTarget(t *testing.T) { t.Run(addr, func(t *testing.T) { t.Parallel() - _, err := ssrfSafeDialer(context.Background(), "tcp", addr) + _, err := dialSSRFSafe(context.Background(), "tcp", addr, nil) if !errors.Is(err, ErrSSRFBlocked) { - t.Errorf("ssrfSafeDialer(%q) = %v, want ErrSSRFBlocked", addr, err) + t.Errorf("dialSSRFSafe(%q) = %v, want ErrSSRFBlocked", addr, err) } }) } @@ -560,7 +560,7 @@ func TestSSRFSafeDialerAllowsPublicTarget(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := ssrfSafeDialer(ctx, "tcp", testPublicHost+":80") + _, err := dialSSRFSafe(ctx, "tcp", testPublicHost+":80", nil) if err == nil { t.Fatal("expected a dial error for an unreachable public target") } diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 29373f4..8988d55 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -639,13 +639,6 @@ func ipv4CompatibleToIPv4(ip net.IP) net.IP { return ip } -// ssrfSafeDialer validates IP addresses against the built-in blocked ranges -// before connecting. New wraps dialSSRFSafe with the operator-supplied -// blocklist; this entry point enforces the built-in ranges alone. -func ssrfSafeDialer(ctx context.Context, network, addr string) (net.Conn, error) { - return dialSSRFSafe(ctx, network, addr, nil) -} - // dialSSRFSafe re-resolves addr and refuses to connect to any built-in // internal range or operator-supplied blocked prefix, closing the // DNS-rebinding window at connect time. diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index c344eea..c66148f 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -322,30 +322,6 @@ const ( mimeJXL = "image/jxl" ) -// SupportedInputFormats returns MIME types this processor can read. -func (p *ImageProcessor) SupportedInputFormats() []string { - return []string{ - mimeJPEG, - mimePNG, - mimeGIF, - mimeWebP, - mimeAVIF, - mimeJXL, - } -} - -// SupportedOutputFormats returns formats this processor can write. -func (p *ImageProcessor) SupportedOutputFormats() []imagerequest.Format { - return []imagerequest.Format{ - imagerequest.FormatJPEG, - imagerequest.FormatPNG, - imagerequest.FormatGIF, - imagerequest.FormatWebP, - imagerequest.FormatAVIF, - imagerequest.FormatJXL, - } -} - // FormatToMIME converts a Format to its MIME type string. func FormatToMIME(format imagerequest.Format) string { switch format { diff --git a/internal/imageprocessor/imageprocessor_internal_test.go b/internal/imageprocessor/imageprocessor_internal_test.go index f5a865e..90c45b9 100644 --- a/internal/imageprocessor/imageprocessor_internal_test.go +++ b/internal/imageprocessor/imageprocessor_internal_test.go @@ -312,22 +312,6 @@ func TestImageProcessor_ProcessPNG(t *testing.T) { } } -func TestImageProcessor_SupportedFormats(t *testing.T) { - t.Parallel() - - proc := New(Params{}) - - inputFormats := proc.SupportedInputFormats() - if len(inputFormats) == 0 { - t.Error("SupportedInputFormats() returned empty slice") - } - - outputFormats := proc.SupportedOutputFormats() - if len(outputFormats) == 0 { - t.Error("SupportedOutputFormats() returned empty slice") - } -} - func TestImageProcessor_RejectsOversizedInput(t *testing.T) { t.Parallel() diff --git a/internal/imagerequest/urlparser.go b/internal/imagerequest/urlparser.go index ce96231..9e8cc4c 100644 --- a/internal/imagerequest/urlparser.go +++ b/internal/imagerequest/urlparser.go @@ -66,23 +66,6 @@ var queryEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}|[^!-~]|["#'<>%]`) var hostRegex = regexp.MustCompile( `^(?:[A-Za-z0-9_.-]+|\[([0-9A-Fa-f:.]+)\])(?::[0-9]*)?$`) -// ParseImagePath parses an image URL's path after /v1/image/, decoded once: -// //., or // for JPEG XL. -// Examples: -// - cdn.example.com/photos/cat.jpg/800x600.webp -// - cdn.example.com/photos/cat.jpg/0x0.jpeg -// - cdn.example.com/photos/cat.jpg/orig.png -// - cdn.example.com/photos/cat.jpg/800x600 -func ParseImagePath(path string) (*Request, error) { - // Strip leading slash if present (chi may include it) - path = strings.TrimPrefix(path, "/") - if path == "" { - return nil, ErrMissingHost - } - - return parseImageComponents(path) -} - // ParseImageURL parses an image URL's path, decoded once as r.URL.Path holds // it: /v1/image///., or /v1/image/// // for JPEG XL. A source query follows the source path after a "?", which the diff --git a/internal/imagerequest/urlparser_internal_test.go b/internal/imagerequest/urlparser_internal_test.go index 83742ca..f80ebfd 100644 --- a/internal/imagerequest/urlparser_internal_test.go +++ b/internal/imagerequest/urlparser_internal_test.go @@ -192,76 +192,16 @@ func TestParseImageURL_Errors(t *testing.T) { } } -func TestParseImagePath(t *testing.T) { - t.Parallel() - - // ParseImagePath is for chi wildcard capture (no /v1/image/ prefix) - tests := []struct { - name string - input string - want *Request - wantErr bool - }{ - { - name: "chi wildcard capture", - input: "cdn.example.com/photos/cat.jpg/800x600.webp", - want: &Request{ - Source: Source{ - Host: testHostCDN, - Path: testPathCat, - }, - Transform: Transform{ - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - }, - }, - }, - { - name: "with leading slash from chi", - input: "/cdn.example.com/photos/cat.jpg/800x600.webp", - want: &Request{ - Source: Source{ - Host: testHostCDN, - Path: testPathCat, - }, - Transform: Transform{ - Size: Size{Width: 800, Height: 600}, - Format: FormatWebP, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := ParseImagePath(tt.input) - if (err != nil) != tt.wantErr { - t.Errorf("ParseImagePath() error = %v, wantErr %v", err, tt.wantErr) - - return - } - - if err != nil { - return - } - - assertRequest(t, got, tt.want) - }) - } -} - -// TestParseImagePath_LowerCasesHost verifies that the host is lower-cased, so +// TestParseImageURL_LowerCasesHost verifies that the host is lower-cased, so // that every spelling of its letter case is the same host to the allowlist, // the upstream connection limits and the cache, and that the path keeps its // case. -func TestParseImagePath_LowerCasesHost(t *testing.T) { +func TestParseImageURL_LowerCasesHost(t *testing.T) { t.Parallel() - got, err := ParseImagePath("CDN.Example.COM/Photos/Cat.jpg/800x600.webp") + got, err := ParseImageURL("/v1/image/CDN.Example.COM/Photos/Cat.jpg/800x600.webp") if err != nil { - t.Fatalf("ParseImagePath() error = %v", err) + t.Fatalf("ParseImageURL() error = %v", err) } t.Logf("parsed host %q, path %q", got.Source.Host, got.Source.Path) @@ -275,21 +215,21 @@ func TestParseImagePath_LowerCasesHost(t *testing.T) { } } -// TestParseImagePath_RefusesUserinfo verifies that a host holding "@" is +// TestParseImageURL_RefusesUserinfo verifies that a host holding "@" is // refused: a URL reads what comes before the "@" as a user name and password // and the rest as the host. -func TestParseImagePath_RefusesUserinfo(t *testing.T) { +func TestParseImageURL_RefusesUserinfo(t *testing.T) { t.Parallel() for _, input := range []string{ - "user@cdn.example.com/photos/cat.jpg/800x600.webp", - "user:password@cdn.example.com/photos/cat.jpg/800x600.webp", + "/v1/image/user@cdn.example.com/photos/cat.jpg/800x600.webp", + "/v1/image/user:password@cdn.example.com/photos/cat.jpg/800x600.webp", } { - got, err := ParseImagePath(input) - t.Logf("ParseImagePath(%q) = %+v, error = %v", input, got, err) + got, err := ParseImageURL(input) + t.Logf("ParseImageURL(%q) = %+v, error = %v", input, got, err) if !errors.Is(err, ErrInvalidHost) { - t.Errorf("ParseImagePath(%q) error = %v, want %v", + t.Errorf("ParseImageURL(%q) error = %v, want %v", input, err, ErrInvalidHost) } } @@ -315,6 +255,10 @@ func TestParseImageURL_PathTraversal(t *testing.T) { name: "parent in middle of path", input: "/v1/image/cdn.example.com/photos/../../../etc/passwd/800x600.jpeg", }, + { + name: "parent directory in path", + input: "/v1/image/cdn.example.com/photos/../secret/800x600.jpeg", + }, { name: "encoded parent directory", input: "/v1/image/cdn.example.com/photos/%2e%2e/secret/800x600.jpeg", @@ -357,40 +301,6 @@ func TestParseImageURL_PathTraversal(t *testing.T) { } } -func TestParseImagePath_PathTraversal(t *testing.T) { - t.Parallel() - - // Test path traversal via ParseImagePath (chi wildcard) - tests := []struct { - name string - input string - }{ - { - name: "parent directory in path", - input: "cdn.example.com/photos/../secret/800x600.jpeg", - }, - { - name: "encoded traversal", - input: "cdn.example.com/photos/%2e%2e/secret/800x600.jpeg", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - _, err := ParseImagePath(tt.input) - if err == nil { - t.Error("ParseImagePath() should reject path traversal attempts") - } - - if !errors.Is(err, ErrPathTraversal) { - t.Errorf("ParseImagePath() error = %v, want ErrPathTraversal", err) - } - }) - } -} - // TestParseImageURL_SourceURLHoldingPercent25 verifies that a source path and // a source query holding %25, the percent-escape of a literal "%", are // accepted as they are, although decoding them once more leaves a "%" that diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 5342080..08ccd9a 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -14,12 +14,6 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// Cache errors. -var ( - ErrCacheMiss = errors.New("cache miss") - ErrNegativeCache = errors.New("negative cache hit") -) - // maxPendingAccessTimes is how many access times of variants served from // the cache, and how many of cached sources a miss used, may each wait to be // written, at about 80 bytes each. An access time handed over while that diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go index 6eb082f..83dd0d8 100644 --- a/internal/imgcache/imgcache.go +++ b/internal/imgcache/imgcache.go @@ -1,13 +1,7 @@ -// Package imgcache provides interfaces and types for the image caching proxy. +// Package imgcache provides types for the image caching proxy. package imgcache -import ( - "context" - "io" - "time" - - "sneak.berlin/go/pixa/internal/imagerequest" -) +import "io" // ImageResponse represents a processed image ready to serve type ImageResponse struct { @@ -19,9 +13,7 @@ type ImageResponse struct { ContentType string // ETag is the entity tag for caching ETag string - // LastModified is when the content was last modified - LastModified time.Time - // CacheStatus indicates HIT, MISS, or STALE + // CacheStatus indicates HIT or MISS CacheStatus CacheStatus // FetchedBytes is the number of bytes fetched from upstream (0 if cache hit) FetchedBytes int64 @@ -32,16 +24,6 @@ type CacheStatus string // Cache status values for response headers. const ( - CacheHit CacheStatus = "HIT" - CacheMiss CacheStatus = "MISS" - CacheStale CacheStatus = "STALE" + CacheHit CacheStatus = "HIT" + CacheMiss CacheStatus = "MISS" ) - -// ImageCache is the main interface for the image caching proxy -type ImageCache interface { - // Get retrieves a processed image, fetching and processing if necessary - Get(ctx context.Context, req *imagerequest.Request) (*ImageResponse, error) - - // Warm pre-fetches and caches an image without returning it - Warm(ctx context.Context, req *imagerequest.Request) error -} diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index 647cf2c..2179396 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -290,9 +290,9 @@ func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) { t.Fatalf("Store() error = %v", err) } - stored, err := storage.Load(key) + stored, _, err := storage.LoadWithSize(key) if err != nil { - t.Fatalf("Load() error = %v", err) + t.Fatalf("LoadWithSize() error = %v", err) } defer func() { _ = stored.Close() }() diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index fa564cd..7d03c0a 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -24,8 +24,7 @@ import ( "sneak.berlin/go/pixa/internal/signature" ) -// Service implements the ImageCache interface, orchestrating cache, -// fetcher, and processor. +// Service orchestrates the cache, fetcher, and processor. type Service struct { cache *Cache fetcher httpfetcher.Fetcher @@ -189,13 +188,6 @@ func (s *Service) Get( return response, nil } -// Warm pre-fetches and caches an image without returning it. -func (s *Service) Warm(ctx context.Context, req *imagerequest.Request) error { - _, err := s.Get(ctx, req) - - return err -} - // WaitForProcessing waits until no image is being processed, or until ctx // ends, and returns how many images were still being processed then. func (s *Service) WaitForProcessing(ctx context.Context) int { @@ -229,30 +221,6 @@ func (s *Service) ValidateRequest(req *imagerequest.Request) error { return s.signer.Verify(req) } -// GenerateSignedURL generates a signed URL for the given request. The URL -// carries q and fit next to sig and exp, so the image route verifies it for -// the quality and fit it was signed with. An unset quality or fit is first -// set to 85 or cover, the values the route uses when a URL has no q or fit. -func (s *Service) GenerateSignedURL( - baseURL string, - req *imagerequest.Request, - ttl time.Duration, -) (string, error) { - if req.Transform.Quality == 0 { - req.Transform.Quality = 85 - } - - if req.Transform.FitMode == "" { - req.Transform.FitMode = imagerequest.FitCover - } - - // Sets the request's Expires and Signature - path, sig, exp := s.signer.GenerateSignedURL(req, ttl) - - return fmt.Sprintf("%s%s?sig=%s&exp=%d&q=%d&fit=%s", - baseURL, path, sig, exp, req.Transform.Quality, req.Transform.FitMode), nil -} - // errPanicked is returned when processing a variant panicked. var errPanicked = errors.New("panic while processing image") diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go index 4cfe4cd..d0da00f 100644 --- a/internal/imgcache/storage.go +++ b/internal/imgcache/storage.go @@ -62,22 +62,6 @@ func (s *ContentStorage) Store( return int64(len(data)), nil } -// Load returns a reader for the content with the given key. -func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) { - path := s.keyToPath(key) - - f, err := os.Open(path) //nolint:gosec // path derived from source key - if err != nil { - if os.IsNotExist(err) { - return nil, ErrNotFound - } - - return nil, fmt.Errorf("failed to open content: %w", err) - } - - return f, nil -} - // LoadWithSize returns a reader and file size for the content with the given key. func (s *ContentStorage) LoadWithSize( key imagerequest.SourceKey, @@ -251,22 +235,6 @@ func (s *VariantStorage) Store( return int64(len(data)), nil } -// Load returns a reader for the content at the given key. -func (s *VariantStorage) Load(key imagerequest.VariantKey) (io.ReadCloser, error) { - path := s.keyToPath(key) - - f, err := os.Open(path) //nolint:gosec // path derived from cache key - if err != nil { - if os.IsNotExist(err) { - return nil, ErrNotFound - } - - return nil, fmt.Errorf("failed to open content: %w", err) - } - - return f, nil -} - // LoadWithContentType opens the content at the given key and returns a // reader, the file size and the content type, which is read from the // file's first bytes. A missing file is ErrNotFound. A file whose first diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go index dc88cc9..a3e0e78 100644 --- a/internal/imgcache/storage_internal_test.go +++ b/internal/imgcache/storage_internal_test.go @@ -11,7 +11,7 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -func TestContentStorage_StoreAndLoad(t *testing.T) { +func TestContentStorage_StoreAndLoadWithSize(t *testing.T) { t.Parallel() tmpDir := t.TempDir() @@ -44,9 +44,9 @@ func TestContentStorage_StoreAndLoad(t *testing.T) { } // Load and verify content - r, err := storage.Load(key) + r, _, err := storage.LoadWithSize(key) if err != nil { - t.Fatalf("Load() error = %v", err) + t.Fatalf("LoadWithSize() error = %v", err) } defer func() { _ = r.Close() }() @@ -57,11 +57,11 @@ func TestContentStorage_StoreAndLoad(t *testing.T) { } if !bytes.Equal(loaded, content) { - t.Errorf("Load() content = %q, want %q", loaded, content) + t.Errorf("LoadWithSize() content = %q, want %q", loaded, content) } } -func TestContentStorage_LoadNotFound(t *testing.T) { +func TestContentStorage_LoadWithSizeNotFound(t *testing.T) { t.Parallel() tmpDir := t.TempDir() @@ -71,9 +71,9 @@ func TestContentStorage_LoadNotFound(t *testing.T) { t.Fatalf("NewContentStorage() error = %v", err) } - _, err = storage.Load(imagerequest.SourceKey("nonexistent")) + _, _, err = storage.LoadWithSize(imagerequest.SourceKey("nonexistent")) if !errors.Is(err, ErrNotFound) { - t.Errorf("Load() error = %v, want ErrNotFound", err) + t.Errorf("LoadWithSize() error = %v, want ErrNotFound", err) } } diff --git a/internal/magic/jpegxl_internal_test.go b/internal/magic/jpegxl_internal_test.go index 7592137..45d9197 100644 --- a/internal/magic/jpegxl_internal_test.go +++ b/internal/magic/jpegxl_internal_test.go @@ -37,22 +37,13 @@ func TestDetectFormatJPEGXL(t *testing.T) { } } -// TestJPEGXLMIMEType verifies that image/jxl is a supported input type and -// maps to and from the format jxl. +// TestJPEGXLMIMEType verifies that image/jxl maps to the format jxl. func TestJPEGXLMIMEType(t *testing.T) { t.Parallel() - if !IsSupportedMIMEType(testMIMEJXL) { - t.Errorf("IsSupportedMIMEType(%q) = false", testMIMEJXL) - } - format, ok := MIMEToImageFormat(testMIMEJXL) if format != FormatJXL || !ok { t.Errorf("MIMEToImageFormat(%q) = %q, %v, want %q, true", testMIMEJXL, format, ok, FormatJXL) } - - if got := ImageFormatToMIME(FormatJXL); got != testMIMEJXL { - t.Errorf("ImageFormatToMIME(%q) = %q, want %q", FormatJXL, got, testMIMEJXL) - } } diff --git a/internal/magic/magic.go b/internal/magic/magic.go index 922316a..db7781a 100644 --- a/internal/magic/magic.go +++ b/internal/magic/magic.go @@ -5,7 +5,6 @@ package magic import ( "bytes" "errors" - "io" "strings" ) @@ -36,22 +35,17 @@ type ImageFormat string // Supported image output formats. const ( - FormatOriginal ImageFormat = "orig" - FormatJPEG ImageFormat = "jpeg" - FormatPNG ImageFormat = "png" - FormatWebP ImageFormat = "webp" - FormatAVIF ImageFormat = "avif" - FormatJXL ImageFormat = "jxl" - FormatGIF ImageFormat = "gif" + FormatJPEG ImageFormat = "jpeg" + FormatPNG ImageFormat = "png" + FormatWebP ImageFormat = "webp" + FormatAVIF ImageFormat = "avif" + FormatJXL ImageFormat = "jxl" + FormatGIF ImageFormat = "gif" ) // MinMagicBytes is the minimum number of bytes needed to detect format. const MinMagicBytes = 12 -// mimeOctetStream is the fallback MIME type for formats without a -// specific MIME type. -const mimeOctetStream = "application/octet-stream" - // Magic byte signatures for supported formats. // These are effectively constants but Go doesn't support const slices. // @@ -190,42 +184,6 @@ func normalizeMIMEType(mimeType string) string { return strings.TrimSpace(strings.ToLower(mimeType)) } -// IsSupportedMIMEType checks if a MIME type is supported for input. -func IsSupportedMIMEType(mimeType string) bool { - normalized := normalizeMIMEType(mimeType) - switch MIMEType(normalized) { - case MIMETypeJPEG, MIMETypePNG, MIMETypeWebP, MIMETypeGIF, MIMETypeAVIF, - MIMETypeJXL, MIMETypeSVG: - return true - default: - return false - } -} - -// PeekAndValidate reads the minimum bytes needed for format detection, -// validates against the declared type, and returns a reader that includes -// those bytes for subsequent reading. -func PeekAndValidate(r io.Reader, declaredType string) (io.Reader, error) { - // Read minimum bytes for detection - buf := make([]byte, MinMagicBytes) - - n, err := io.ReadFull(r, buf) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { - return nil, err - } - - buf = buf[:n] - - // Validate magic bytes - err = ValidateMagicBytes(buf, declaredType) - if err != nil { - return nil, err - } - - // Return a reader that includes the peeked bytes - return io.MultiReader(bytes.NewReader(buf), r), nil -} - // MIMEToImageFormat converts a MIME type to an ImageFormat. func MIMEToImageFormat(mimeType string) (ImageFormat, bool) { normalized := normalizeMIMEType(mimeType) @@ -249,26 +207,3 @@ func MIMEToImageFormat(mimeType string) (ImageFormat, bool) { return "", false } } - -// ImageFormatToMIME converts an ImageFormat to a MIME type string. -func ImageFormatToMIME(format ImageFormat) string { - switch format { - case FormatJPEG: - return string(MIMETypeJPEG) - case FormatPNG: - return string(MIMETypePNG) - case FormatWebP: - return string(MIMETypeWebP) - case FormatGIF: - return string(MIMETypeGIF) - case FormatAVIF: - return string(MIMETypeAVIF) - case FormatJXL: - return string(MIMETypeJXL) - case FormatOriginal: - // Original format passes content through unchanged. - return mimeOctetStream - default: - return mimeOctetStream - } -} diff --git a/internal/magic/magic_internal_test.go b/internal/magic/magic_internal_test.go index e0295ee..fc081ea 100644 --- a/internal/magic/magic_internal_test.go +++ b/internal/magic/magic_internal_test.go @@ -3,9 +3,6 @@ package magic import ( "bytes" "errors" - "io" - "slices" - "strings" "testing" ) @@ -155,119 +152,6 @@ func TestValidateMagicBytes(t *testing.T) { } } -func TestIsSupportedMIMEType(t *testing.T) { - t.Parallel() - - tests := []struct { - mimeType string - want bool - }{ - {testMIMEJPEG, true}, - {testMIMEPNG, true}, - {testMIMEWebP, true}, - {testMIMEGIF, true}, - {testMIMEAVIF, true}, - {"image/svg+xml", true}, - {"IMAGE/JPEG", true}, - {testMIMEJPEGParams, true}, - {"image/tiff", false}, - {"image/bmp", false}, - {mimeOctetStream, false}, - {"text/plain", false}, - {"", false}, - } - - for _, tt := range tests { - t.Run(tt.mimeType, func(t *testing.T) { - t.Parallel() - - if got := IsSupportedMIMEType(tt.mimeType); got != tt.want { - t.Errorf("IsSupportedMIMEType(%q) = %v, want %v", tt.mimeType, got, tt.want) - } - }) - } -} - -func TestPeekAndValidate(t *testing.T) { - t.Parallel() - - jpegMagic := []byte{ - 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, - } - pngMagic := []byte{ - 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, - } - jpegData := slices.Concat(jpegMagic, []byte("rest of jpeg data")) - pngData := slices.Concat(pngMagic, []byte("rest of png data")) - - tests := []struct { - name string - data []byte - declaredType string - wantErr bool - wantData []byte - }{ - { - name: "valid JPEG", - data: jpegData, - declaredType: testMIMEJPEG, - wantErr: false, - wantData: jpegData, - }, - { - name: "valid PNG", - data: pngData, - declaredType: testMIMEPNG, - wantErr: false, - wantData: pngData, - }, - { - name: "mismatched type", - data: jpegData, - declaredType: testMIMEPNG, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - r := bytes.NewReader(tt.data) - - result, err := PeekAndValidate(r, tt.declaredType) - if tt.wantErr { - if err == nil { - t.Error("PeekAndValidate() expected error, got nil") - } - - return - } - - if err != nil { - t.Errorf("PeekAndValidate() unexpected error = %v", err) - - return - } - - // Read all data from result reader - got, err := io.ReadAll(result) - if err != nil { - t.Errorf("Failed to read result: %v", err) - - return - } - - if !bytes.Equal(got, tt.wantData) { - t.Errorf( - "PeekAndValidate() data mismatch: got %d bytes, want %d bytes", - len(got), len(tt.wantData), - ) - } - }) - } -} - func TestMIMEToImageFormat(t *testing.T) { t.Parallel() @@ -303,35 +187,6 @@ func TestMIMEToImageFormat(t *testing.T) { } } -func TestImageFormatToMIME(t *testing.T) { - t.Parallel() - - tests := []struct { - format ImageFormat - wantMIME string - }{ - {FormatJPEG, testMIMEJPEG}, - {FormatPNG, testMIMEPNG}, - {FormatWebP, testMIMEWebP}, - {FormatGIF, testMIMEGIF}, - {FormatAVIF, testMIMEAVIF}, - {FormatOriginal, mimeOctetStream}, - {"unknown", mimeOctetStream}, - } - - for _, tt := range tests { - t.Run(string(tt.format), func(t *testing.T) { - t.Parallel() - - got := ImageFormatToMIME(tt.format) - - if got != tt.wantMIME { - t.Errorf("ImageFormatToMIME(%q) = %v, want %v", tt.format, got, tt.wantMIME) - } - }) - } -} - func TestNormalizeMIMEType(t *testing.T) { t.Parallel() @@ -493,34 +348,3 @@ func TestDetectFormatFtypNotAVIF(t *testing.T) { t.Errorf("DetectFormat(MP4) error = %v, want %v", err, ErrUnknownFormat) } } - -func TestPeekAndValidatePreservesReader(t *testing.T) { - t.Parallel() - - // Ensure that after PeekAndValidate, we can read the complete - // original content - originalContent := append( - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D}, - []byte(strings.Repeat("PNG IDAT chunk data here ", 100))..., - ) - - r := bytes.NewReader(originalContent) - - validated, err := PeekAndValidate(r, testMIMEPNG) - if err != nil { - t.Fatalf("PeekAndValidate() error = %v", err) - } - - // Read everything from the validated reader - got, err := io.ReadAll(validated) - if err != nil { - t.Fatalf("io.ReadAll() error = %v", err) - } - - if !bytes.Equal(got, originalContent) { - t.Errorf( - "Content mismatch: got %d bytes, want %d bytes", - len(got), len(originalContent), - ) - } -} diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index 05e8ef4..550b8e5 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -513,20 +513,27 @@ func TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort(t *testing.T) { func signedImageURL(t *testing.T, sourceHost string) string { t.Helper() - path, sig, exp := signature.New(testSigningKey).GenerateSignedURL( - &imagerequest.Request{ - Source: imagerequest.Source{ - Host: sourceHost, - Path: ipv6SourcePath, - }, - Transform: imagerequest.Transform{ - Format: imagerequest.FormatPNG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - }, time.Hour) + expires := time.Now().Add(time.Hour) + sig := signature.New(testSigningKey).Sign(&imagerequest.Request{ + Source: imagerequest.Source{ + Host: sourceHost, + Path: ipv6SourcePath, + }, + Transform: imagerequest.Transform{ + Format: imagerequest.FormatPNG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + Expires: expires, + }) - return path + "?sig=" + sig + "&exp=" + strconv.FormatInt(exp, 10) + // Go's percent-encoding of a URL path writes the brackets of the IPv6 + // host as %5B and %5D. + path := (&url.URL{ + Path: "/v1/image/" + sourceHost + ipv6SourcePath + "/orig.png", + }).EscapedPath() + + return path + "?sig=" + sig + "&exp=" + strconv.FormatInt(expires.Unix(), 10) } // encryptedImageURL returns the encrypted URL of ipv6SourcePath on sourceHost diff --git a/internal/signature/golden_test.go b/internal/signature/golden_test.go index bb8eda2..f3d6e16 100644 --- a/internal/signature/golden_test.go +++ b/internal/signature/golden_test.go @@ -22,11 +22,6 @@ type goldenVector struct { // padded) HMAC-SHA256 signature for the request with Expires // set to goldenExpiresUnix. wantSignature string - // wantSignedPath is the exact path returned by - // GenerateSignedURL for the request. The signature and - // expiration are returned separately by GenerateSignedURL and - // are not embedded in the path. - wantSignedPath string } // goldenVectors returns the known-answer vectors. The expected values @@ -50,8 +45,7 @@ func goldenVectors() []goldenVector { }, // Signed data: // "cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover" - wantSignature: "kdqeGoW2SX7qnaYtoB970wEnLydn0UnIgQYQLfAnjXQ=", - wantSignedPath: testSignedPath, + wantSignature: "kdqeGoW2SX7qnaYtoB970wEnLydn0UnIgQYQLfAnjXQ=", }, { name: "resized with query string", @@ -71,8 +65,6 @@ func goldenVectors() []goldenVector { // Signed data: // "cdn.example.com:/photos/cat.jpg:token=abc&v=2:800:600:webp:1704067200:85:cover" wantSignature: "pKgVBOTd_Q_EikI7MNQLC9Q8Hurdxzyv3EIYvVhqc2I=", - wantSignedPath: "/v1/image/cdn.example.com/photos/cat.jpg" + - "%3Ftoken=abc&v=2/800x600.webp", }, { name: "original size without query", @@ -91,8 +83,7 @@ func goldenVectors() []goldenVector { }, // Signed data: // "cdn.example.com:/photos/cat.jpg::0:0:png:1704067200:85:cover" - wantSignature: "6_rZ0yyVbGZRs8kG7n7HLgLi5Jt8vjiWQljIEL1jbIs=", - wantSignedPath: "/v1/image/cdn.example.com/photos/cat.jpg/orig.png", + wantSignature: "6_rZ0yyVbGZRs8kG7n7HLgLi5Jt8vjiWQljIEL1jbIs=", }, { name: "non-default quality and fit", @@ -112,22 +103,17 @@ func goldenVectors() []goldenVector { // Signed data: // "cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:40:contain" wantSignature: "pGaXpPUbI3A7nMx-4T9bfq9bYWBNL0kY4bxlcv3g1F8=", - // The path is the same as for the default quality and fit: - // q=40&fit=contain go in the query string next to sig and - // exp (imgcache.Service.GenerateSignedURL adds all four). - wantSignedPath: testSignedPath, }, } } -// TestSigner_GoldenVectors pins the exact HMAC-SHA256 signature output and -// the exact generated signed URL path for fully-specified requests with a -// hardcoded signing key. +// TestSigner_GoldenVectors pins the exact HMAC-SHA256 signature output for +// fully-specified requests with a hardcoded signing key. // // If any of these assertions fail, the signed byte format -// ("host:path:query:width:height:format:expiration:quality:fit"), the -// base64url encoding, or the signed URL layout has changed. Update these -// constants only when that change is intended. +// ("host:path:query:width:height:format:expiration:quality:fit") or the +// base64url encoding has changed. Update these constants only when that +// change is intended. func TestSigner_GoldenVectors(t *testing.T) { t.Parallel() @@ -145,14 +131,6 @@ func TestSigner_GoldenVectors(t *testing.T) { t.Errorf("Sign() = %q, want %q (signed byte format changed?)", gotSignature, tt.wantSignature) } - - urlReq := tt.req - - gotPath, _, _ := signer.GenerateSignedURL(&urlReq, time.Hour) - if gotPath != tt.wantSignedPath { - t.Errorf("GenerateSignedURL() path = %q, want %q (signed URL layout changed?)", - gotPath, tt.wantSignedPath) - } }) } } diff --git a/internal/signature/signature.go b/internal/signature/signature.go index beff9b4..6982756 100644 --- a/internal/signature/signature.go +++ b/internal/signature/signature.go @@ -8,8 +8,6 @@ import ( "encoding/base64" "errors" "fmt" - "net/url" - "strconv" "strings" "time" @@ -79,44 +77,6 @@ func (s *Signer) Verify(req *imagerequest.Request) error { return nil } -// GenerateSignedURL creates a complete URL with signature and expiration. -// Returns the path portion that should be appended to the base URL. -func (s *Signer) GenerateSignedURL( - req *imagerequest.Request, ttl time.Duration, -) (string, string, int64) { - // Set expiration - req.Expires = time.Now().Add(ttl) - exp := req.Expires.Unix() - - // Generate signature - sig := s.Sign(req) - req.Signature = sig - - // Build the size component - var sizeStr string - if req.Transform.Size.Width == 0 && req.Transform.Size.Height == 0 { - sizeStr = "orig" - } else { - sizeStr = fmt.Sprintf("%dx%d", - req.Transform.Size.Width, req.Transform.Size.Height) - } - - // Build the path. A source query follows the source path after a "?". - // The image route reads the path decoded once, so the whole path is - // written in Go's own percent-encoding of a URL path, which writes "?" - // as %3F and "%" as %25, and leaves "/", ";", ",", "=" and "&" as they - // are. - sourcePathAndQuery := req.Source.Path - if req.Source.Query != "" { - sourcePathAndQuery += "?" + req.Source.Query - } - - unescapedPath := fmt.Sprintf("/v1/image/%s%s/%s.%s", - req.Source.Host, sourcePathAndQuery, sizeStr, req.Transform.Format) - - return (&url.URL{Path: unescapedPath}).EscapedPath(), sig, exp -} - // buildSignatureData creates the string to be signed. // Format: "host:path:query:width:height:format:expiration:quality:fit" // The path and query go through escapePercentAndColon, so that a ":" in @@ -146,17 +106,3 @@ func escapePercentAndColon(s string) string { return strings.ReplaceAll(s, ":", "%3A") } - -// ParseParams extracts signature and expiration from query parameters. -func ParseParams(sig, expStr string) (string, time.Time, error) { - if expStr == "" { - return sig, time.Time{}, nil - } - - expUnix, err := strconv.ParseInt(expStr, 10, 64) - if err != nil { - return "", time.Time{}, fmt.Errorf("invalid expiration: %w", err) - } - - return sig, time.Unix(expUnix, 0), nil -} diff --git a/internal/signature/signature_test.go b/internal/signature/signature_test.go index 119fb14..dfe08a2 100644 --- a/internal/signature/signature_test.go +++ b/internal/signature/signature_test.go @@ -2,7 +2,6 @@ package signature_test import ( "errors" - "strings" "testing" "time" @@ -18,8 +17,6 @@ const ( testFormatPNG = "png" testFitCover = "cover" testFitContain = "contain" - testSignedPath = "/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp" - testSig = "abc123" ) func TestSigner_Sign(t *testing.T) { @@ -378,140 +375,10 @@ func TestSigner_DifferentKeys(t *testing.T) { } } -func TestGenerateSignedURL(t *testing.T) { - t.Parallel() - - signer := signature.New("test-secret-key") - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHost, - Path: testPath, - Query: "", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: testFormatWebP, - }, - } - - ttl := 1 * time.Hour - path, sig, exp := signer.GenerateSignedURL(req, ttl) - - // Path should be correct format - if path != testSignedPath { - t.Errorf("GenerateSignedURL() path = %q, want %q", path, testSignedPath) - } - - // Signature should be non-empty - if sig == "" { - t.Error("GenerateSignedURL() produced empty signature") - } - - // Expiration should be approximately now + TTL - expTime := time.Unix(exp, 0) - - expectedExp := time.Now().Add(ttl) - if expTime.Sub(expectedExp) > time.Second { - t.Errorf("GenerateSignedURL() exp time off by too much") - } - - // Request should have been updated with signature and expiration - if req.Signature != sig { - t.Errorf("GenerateSignedURL() didn't update request signature") - } -} - -func TestGenerateSignedURL_OrigSize(t *testing.T) { - t.Parallel() - - signer := signature.New("test-secret-key") - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHost, - Path: testPath, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 0, Height: 0}, // Original size - Format: testFormatPNG, - }, - } - - path, _, _ := signer.GenerateSignedURL(req, time.Hour) - - expectedPath := "/v1/image/cdn.example.com/photos/cat.jpg/orig.png" - if path != expectedPath { - t.Errorf("GenerateSignedURL() path = %q, want %q", path, expectedPath) - } -} - -func TestGenerateSignedURL_WithQueryString(t *testing.T) { - t.Parallel() - - signer := signature.New("test-secret-key-for-testing!") - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHost, - Path: testPath, - Query: "token=abc&v=2", - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: testFormatWebP, - }, - } - - path, _, _ := signer.GenerateSignedURL(req, time.Hour) - - // The path must NOT contain a bare "?" that would be interpreted as - // a query string delimiter. The size segment must appear as the last - // path component. - if strings.Contains(path, "?token=abc") { - t.Errorf("GenerateSignedURL() produced bare query string in path: %q", path) - } - - // The size segment must be present in the path - if !strings.Contains(path, "/800x600.webp") { - t.Errorf("GenerateSignedURL() missing size segment in path: %q", path) - } - - // Path should end with the size.format, not with query params - if !strings.HasSuffix(path, "/800x600.webp") { - t.Errorf("GenerateSignedURL() path should end with size.format: %q", path) - } -} - -func TestGenerateSignedURL_WithoutQueryString(t *testing.T) { - t.Parallel() - - signer := signature.New("test-secret-key-for-testing!") - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHost, - Path: testPath, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 800, Height: 600}, - Format: testFormatWebP, - }, - } - - path, _, _ := signer.GenerateSignedURL(req, time.Hour) - - if path != testSignedPath { - t.Errorf("GenerateSignedURL() path = %q, want %q", path, testSignedPath) - } -} - -// TestGenerateSignedURL_ReadmeQueryExample checks the README's example of a -// source URL with a query, the image at -// https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg: its -// signature with the README's key, and the path GenerateSignedURL writes for -// it, with the "?" written as %3F and the "%" as %25. -func TestGenerateSignedURL_ReadmeQueryExample(t *testing.T) { +// TestSign_ReadmeQueryExample checks the signature, with the README's key, of +// the README's example of a source URL with a query, the image at +// https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg. +func TestSign_ReadmeQueryExample(t *testing.T) { t.Parallel() signer := signature.New("example-signing-key-for-documentation") @@ -537,76 +404,4 @@ func TestGenerateSignedURL_ReadmeQueryExample(t *testing.T) { if gotSig != wantSig { t.Errorf("Sign() = %q, want %q", gotSig, wantSig) } - - const wantPath = "/v1/image/cdn.example.com" + - "/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp" - - gotPath, _, _ := signer.GenerateSignedURL(&req, time.Hour) - if gotPath != wantPath { - t.Errorf("GenerateSignedURL() path = %q, want %q", gotPath, wantPath) - } -} - -func TestParseParams(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - sig string - expStr string - wantSig string - wantErr bool - checkTime bool - }{ - { - name: "valid params", - sig: testSig, - expStr: "1704067200", - wantSig: testSig, - wantErr: false, - }, - { - name: "empty expiration", - sig: testSig, - expStr: "", - wantSig: testSig, - wantErr: false, - }, - { - name: "invalid expiration", - sig: testSig, - expStr: "not-a-number", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - sig, exp, err := signature.ParseParams(tt.sig, tt.expStr) - - if tt.wantErr { - if err == nil { - t.Error("ParseParams() expected error, got nil") - } - - return - } - - if err != nil { - t.Errorf("ParseParams() unexpected error = %v", err) - - return - } - - if sig != tt.wantSig { - t.Errorf("sig = %q, want %q", sig, tt.wantSig) - } - - if tt.expStr != "" && exp.IsZero() { - t.Error("exp should not be zero when expStr is provided") - } - }) - } } From 63690a9ec934e64c3e7179db9e82dd012c84f686 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 12:28:23 +0100 Subject: [PATCH 103/131] Build the image pipeline in pixad's fx graph (closes #313) The new package internal/providers builds the cache, the fetcher, the processor, the image service, the session manager and the encrypted URL generator from the config. pixad's fx graph provides them with providers.Constructors(), and handlers.New receives them built. The server tests build the same graph, with only the fetcher's dial function replaced. database.New opens the database, so these parts can read it while fx builds them; the cache's stop hook still runs before the database's. Each setting's default is declared in internal/config, and internal/providers works out the default cache_max_bytes. The cache's MaxBytes is the one value for the disk cache limit, and 0 turns the disk cache off. The fetcher and the processor take explicit values. Model: opus-5-5 --- README.md | 4 +- TODO.md | 24 ++ internal/app/app.go | 2 + .../config/cache_max_bytes_internal_test.go | 34 +- internal/config/config.go | 51 ++- internal/database/database.go | 22 +- internal/handlers/cache_hit_internal_test.go | 13 +- .../handlers/cache_max_bytes_internal_test.go | 151 --------- internal/handlers/fetcher_internal_test.go | 63 ---- .../handlers/first_bytes_internal_test.go | 17 +- internal/handlers/handlers.go | 147 +------- internal/handlers/handlers_internal_test.go | 15 +- .../image_cache_control_internal_test.go | 11 +- .../handlers/image_errors_internal_test.go | 22 +- .../referer_blocklist_internal_test.go | 15 +- .../blocked_networks_internal_test.go | 16 +- .../httpfetcher/dial_context_internal_test.go | 20 +- internal/httpfetcher/fetch_internal_test.go | 39 ++- internal/httpfetcher/httpfetcher.go | 65 ++-- .../httpfetcher/httpfetcher_internal_test.go | 53 +-- internal/httpfetcher/jpegxl_internal_test.go | 18 +- .../max_connections_internal_test.go | 41 ++- .../avif_too_large_internal_test.go | 5 +- .../computed_side_internal_test.go | 10 +- .../imageprocessor/deadline_internal_test.go | 2 +- .../empty_format_internal_test.go | 5 +- .../encoder_settings_internal_test.go | 5 +- .../imageprocessor/fit_cover_internal_test.go | 15 +- .../fit_outside_internal_test.go | 10 +- internal/imageprocessor/imageprocessor.go | 26 +- .../imageprocessor_internal_test.go | 43 +-- .../jpegxl_output_internal_test.go | 10 +- ...max_concurrent_processing_internal_test.go | 43 +-- .../orig_format_internal_test.go | 5 +- .../source_formats_internal_test.go | 2 +- internal/imgcache/cache.go | 65 +--- internal/imgcache/cache_hit_internal_test.go | 5 +- internal/imgcache/cache_internal_test.go | 2 + internal/imgcache/cachesize.go | 85 ----- internal/imgcache/cachesize_internal_test.go | 189 ----------- .../concurrent_misses_internal_test.go | 2 +- internal/imgcache/counters_internal_test.go | 9 +- internal/imgcache/eviction.go | 27 +- internal/imgcache/eviction_internal_test.go | 12 +- ...max_concurrent_processing_internal_test.go | 2 +- .../imgcache/negative_cache_internal_test.go | 25 +- internal/imgcache/service.go | 53 +-- internal/imgcache/testutil_internal_test.go | 15 +- .../cache_max_bytes_internal_test.go | 315 ++++++++++++++++++ internal/providers/fetcher_internal_test.go | 103 ++++++ internal/providers/providers.go | 256 ++++++++++++++ internal/server/image_proxy_internal_test.go | 41 ++- .../server/login_rate_limit_internal_test.go | 49 +-- internal/server/logout_internal_test.go | 4 +- 54 files changed, 1181 insertions(+), 1102 deletions(-) delete mode 100644 internal/handlers/cache_max_bytes_internal_test.go delete mode 100644 internal/handlers/fetcher_internal_test.go delete mode 100644 internal/imgcache/cachesize.go delete mode 100644 internal/imgcache/cachesize_internal_test.go create mode 100644 internal/providers/cache_max_bytes_internal_test.go create mode 100644 internal/providers/fetcher_internal_test.go create mode 100644 internal/providers/providers.go diff --git a/README.md b/README.md index 87bf02b..d2170b9 100644 --- a/README.md +++ b/README.md @@ -812,7 +812,9 @@ See `configs/config.example.yml` for all options with defaults. ### Architecture -- **Dependency injection**: Uber fx +- **Dependency injection**: Uber fx. `internal/app` builds every part of pixad + from the config in one fx graph. Each setting's default is declared once, in + `internal/config`. - **HTTP router**: go-chi - **Image processing**: govips (CGO wrapper for libvips) - **Database**: SQLite via modernc.org/sqlite diff --git a/TODO.md b/TODO.md index 8c2ff37..2f56525 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,30 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 pixad's fx graph builds the image pipeline (closes #313). + `database.New` now opens the database, so the parts below can read it while + they are built; its stop hook still runs after the cache's. The cache's + `MaxBytes` is the one value for the disk cache limit, and 0 turns the disk + cache off. For an omitted `cache_max_bytes`, `internal/providers` works out + `config.DefaultCacheMaxBytes` before it builds the cache. `internal/app` and + the server tests build the parts below with `providers.Constructors()`; the + image proxy test replaces only the fetcher's dial function. + - Built by the providers in `internal/providers`, and given to + `handlers.New`: + - the cache + - the fetcher + - the processor + - the image service + - the session manager + - the encrypted URL generator + - Removed: + - `initImageService` and `handlers.Params.Fetcher` + - `CacheConfig.UseDefaultMaxBytes`, `CacheConfig.DisableDiskCache` and + `Cache.disabled` + - `httpfetcher.DefaultConfig()` and the fetcher's copies of setting + defaults + - `imageprocessor.DefaultMaxInputBytes`, and the processor's defaults + for a zero `MaxInputBytes` or `MaxConcurrentProcessing` - 2026-10-26 the code nothing in production called is gone (closes #312). The handlers tests sign their image URLs with `generateSignedURL`, a test helper that writes them as the removed generator did. Removed: diff --git a/internal/app/app.go b/internal/app/app.go index c09f554..0d9daf9 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -17,6 +17,7 @@ import ( "sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/middleware" + "sneak.berlin/go/pixa/internal/providers" "sneak.berlin/go/pixa/internal/server" ) @@ -65,6 +66,7 @@ func run(_ *cobra.Command, _ []string) { // The registry /metrics serves func() prometheus.Registerer { return prometheus.DefaultRegisterer }, ), + fx.Provide(providers.Constructors()...), fx.Invoke( func(log *logger.Logger) { log.Identify() }, func(*server.Server) {}, diff --git a/internal/config/cache_max_bytes_internal_test.go b/internal/config/cache_max_bytes_internal_test.go index 6af1554..3538530 100644 --- a/internal/config/cache_max_bytes_internal_test.go +++ b/internal/config/cache_max_bytes_internal_test.go @@ -135,10 +135,38 @@ func TestCacheMaxBytesInvalidValuesAbortStartup(t *testing.T) { } } +// TestDefaultCacheMaxBytesAppliesFloor verifies that when 75% of the +// free space plus what the cache holds is below 500 MiB, the default is +// floored at DefaultCacheMaxBytesFloor. +func TestDefaultCacheMaxBytesAppliesFloor(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + freeBytes uint64 + }{ + {name: "100 MiB free", freeBytes: 104857600}, + {name: "zero free", freeBytes: 0}, + {name: "just below floor threshold", freeBytes: 699050665}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := DefaultCacheMaxBytes(tc.freeBytes, 0) + if got != DefaultCacheMaxBytesFloor { + t.Errorf("DefaultCacheMaxBytes = %d, want floor %d", + got, DefaultCacheMaxBytesFloor) + } + }) + } +} + // TestCacheMaxBytesExplicitIsRecorded verifies that an omitted -// cache_max_bytes is recorded as not explicit, so the cache works out -// the default when it opens, and that an explicit zero is recorded as -// explicit, so it disables the disk cache instead. +// cache_max_bytes is recorded as not explicit, so pixad works out the +// default when it builds the cache, and that an explicit zero is recorded +// as explicit, so it disables the disk cache instead. func TestCacheMaxBytesExplicitIsRecorded(t *testing.T) { t.Parallel() diff --git a/internal/config/config.go b/internal/config/config.go index eff51ce..3f484b7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -45,6 +45,37 @@ func DefaultMaxConcurrentProcessing() int { return max(runtime.GOMAXPROCS(0)/2, 1) //nolint:mnd // half the CPUs } +// DefaultCacheMaxBytesFloor is the least DefaultCacheMaxBytes returns: +// 500 MiB. It applies only to the default, never to an explicit +// cache_max_bytes. +const DefaultCacheMaxBytesFloor int64 = 500 << 20 + +// freeSpaceFractionNumerator and freeSpaceFractionDenominator express +// the 75% share of DefaultCacheMaxBytes as integer arithmetic (dividing +// before multiplying avoids overflow). +const ( + freeSpaceFractionNumerator uint64 = 3 + freeSpaceFractionDenominator uint64 = 4 +) + +// DefaultCacheMaxBytes returns the disk cache limit when cache_max_bytes +// is omitted: 75% of the sum of freeBytes, the space free on the +// filesystem holding the cache, and heldBytes, the bytes the cache +// already holds, and at least DefaultCacheMaxBytesFloor. Counting what +// the cache holds keeps the limit from shrinking as the cache fills. +// pixad works it out when it builds the cache, as both numbers are known +// only then. +func DefaultCacheMaxBytes(freeBytes uint64, heldBytes int64) int64 { + // Both terms are at most math.MaxInt64, so the sum cannot overflow. + //nolint:gosec // G115: heldBytes is the cache's total, never negative + spaceBytes := min(freeBytes, math.MaxInt64) + uint64(heldBytes) + + computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator + computed = min(computed, math.MaxInt64) + + return max(int64(computed), DefaultCacheMaxBytesFloor) +} + // Configuration key names. const ( keyDebug = "debug" @@ -190,18 +221,14 @@ type Config struct { // address, and an explicit list replaces the default. TrustedProxies []netip.Prefix - // CacheMaxBytes is the disk cache size limit in bytes. Only an - // explicit zero (CacheMaxBytesExplicit true) disables the disk - // cache. Zero with CacheMaxBytesExplicit false means - // cache_max_bytes was omitted, and the cache works out the default - // limit when it opens. + // CacheMaxBytes is the disk cache size limit in bytes, as set; 0 + // turns the disk cache off. CacheMaxBytes int64 // CacheMaxBytesExplicit records whether cache_max_bytes was // explicitly set, in the environment or the configuration file. - // Explicit values are used exactly as given; for an omitted key the - // cache works out the default limit when it opens (see - // imgcache.CacheConfig.UseDefaultMaxBytes). + // Explicit values are used exactly as given. For an omitted key + // pixad works out DefaultCacheMaxBytes when it builds the cache. CacheMaxBytesExplicit bool } @@ -239,8 +266,8 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) { return nil, err } - // An omitted cache_max_bytes is worked out and logged when the - // cache opens. + // An omitted cache_max_bytes is worked out and logged when pixad + // builds the cache. if c.CacheMaxBytesExplicit { log.Info("effective cache size limit", "cache_max_bytes", c.CacheMaxBytes, @@ -325,8 +352,8 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { } // The default for an omitted cache_max_bytes is worked out when - // the cache opens; here we only record whether the operator set - // the key explicitly. + // pixad builds the cache; here we only record whether the operator + // set the key explicitly. _, c.CacheMaxBytesExplicit = lookupValue(sc, keyCacheMaxBytes) // Build DBURL from StateDir if not explicitly set. The derived URL diff --git a/internal/database/database.go b/internal/database/database.go index 4eea04e..31b74e2 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -82,7 +82,11 @@ func ParseMigrationVersion(filename string) (int, error) { return version, nil } -// New creates a new Database instance. +// New opens the database and applies the migrations, so that the parts +// fx builds after it can read the database while they are built. Its +// stop hook closes the database. fx runs stop hooks in the reverse of the +// order it built their parts in, so the stop hooks of those parts run +// before this one. func New(lc fx.Lifecycle, params Params) (*Database, error) { s := &Database{ log: params.Logger.Get(), @@ -91,20 +95,16 @@ func New(lc fx.Lifecycle, params Params) (*Database, error) { s.log.Info("Database instantiated") - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - s.log.Info("Database OnStart Hook") + err := s.connect(context.Background()) + if err != nil { + return nil, err + } - return s.connect(ctx) - }, + lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { s.log.Info("Database OnStop Hook") - if s.db != nil { - return s.db.Close() - } - - return nil + return s.db.Close() }, }) diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go index 713395a..56d1dda 100644 --- a/internal/handlers/cache_hit_internal_test.go +++ b/internal/handlers/cache_hit_internal_test.go @@ -21,6 +21,7 @@ import ( "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -121,6 +122,7 @@ func newCountingDatabaseServer( cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ StateDir: tb.TempDir(), NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { tb.Fatalf("imgcache.NewCache() error = %v", err) @@ -131,9 +133,14 @@ func newCountingDatabaseServer( Fetcher: newMockFetcher(fstest.MapFS{ countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()}, }), - SigningKey: testSigningKey, - Allowlist: []string{countingHost}, - Logger: slog.New(slog.DiscardHandler), + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: testSigningKey, + Allowlist: []string{countingHost}, + MaxResponseSize: 50 << 20, + Logger: slog.New(slog.DiscardHandler), }) if err != nil { tb.Fatalf("imgcache.NewService() error = %v", err) diff --git a/internal/handlers/cache_max_bytes_internal_test.go b/internal/handlers/cache_max_bytes_internal_test.go deleted file mode 100644 index 1fec51d..0000000 --- a/internal/handlers/cache_max_bytes_internal_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package handlers - -import ( - "os" - "path/filepath" - "testing" - - "github.com/prometheus/client_golang/prometheus" - "go.uber.org/fx/fxtest" - - "sneak.berlin/go/pixa/internal/config" - "sneak.berlin/go/pixa/internal/database" - "sneak.berlin/go/pixa/internal/globals" - "sneak.berlin/go/pixa/internal/logger" -) - -// TestNewCacheConfigFromCacheMaxBytes checks the cache configuration -// built from cache_max_bytes: omitted, the cache works out the default -// limit; 0 turns the disk cache off; a positive value is the limit, -// unchanged. -func TestNewCacheConfigFromCacheMaxBytes(t *testing.T) { - t.Parallel() - - const oneGiB = 1 << 30 - - cases := []struct { - name string - cacheMaxBytes int64 - cacheMaxBytesExplicit bool - wantMaxBytes int64 - wantUseDefaultMaxBytes bool - wantDisableDiskCache bool - }{ - { - name: "cache_max_bytes omitted", - cacheMaxBytes: 0, - cacheMaxBytesExplicit: false, - wantMaxBytes: 0, - wantUseDefaultMaxBytes: true, - wantDisableDiskCache: false, - }, - { - name: "cache_max_bytes: 0", - cacheMaxBytes: 0, - cacheMaxBytesExplicit: true, - wantMaxBytes: 0, - wantUseDefaultMaxBytes: false, - wantDisableDiskCache: true, - }, - { - name: "cache_max_bytes: 1 GiB", - cacheMaxBytes: oneGiB, - cacheMaxBytesExplicit: true, - wantMaxBytes: oneGiB, - wantUseDefaultMaxBytes: false, - wantDisableDiskCache: false, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - cfg := &config.Config{ - CacheMaxBytes: tc.cacheMaxBytes, - CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit, - } - - got := newCacheConfig(cfg, nil) - t.Logf("MaxBytes = %d, UseDefaultMaxBytes = %v, DisableDiskCache = %v", - got.MaxBytes, got.UseDefaultMaxBytes, got.DisableDiskCache) - - if got.MaxBytes != tc.wantMaxBytes { - t.Errorf("MaxBytes = %d, want %d", got.MaxBytes, tc.wantMaxBytes) - } - - if got.UseDefaultMaxBytes != tc.wantUseDefaultMaxBytes { - t.Errorf("UseDefaultMaxBytes = %v, want %v", - got.UseDefaultMaxBytes, tc.wantUseDefaultMaxBytes) - } - - if got.DisableDiskCache != tc.wantDisableDiskCache { - t.Errorf("DisableDiskCache = %v, want %v", - got.DisableDiskCache, tc.wantDisableDiskCache) - } - }) - } -} - -// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes starts the handlers -// once with cache_max_bytes omitted and once with cache_max_bytes: 0, -// and checks by whether the cache directories were created that the -// disk cache is on in the first case and off in the second. -func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - cacheMaxBytesExplicit bool - wantDiskCache bool - }{ - {name: "cache_max_bytes omitted", cacheMaxBytesExplicit: false, wantDiskCache: true}, - {name: "cache_max_bytes: 0", cacheMaxBytesExplicit: true, wantDiskCache: false}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - stateDir := t.TempDir() - cfg := &config.Config{ - SigningKey: testSigningKey, - StateDir: stateDir, - DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), - CacheMaxBytes: 0, - CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit, - } - - lc := fxtest.NewLifecycle(t) - - log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}}) - if err != nil { - t.Fatalf("logger.New() error = %v", err) - } - - db, err := database.New(lc, database.Params{Logger: log, Config: cfg}) - if err != nil { - t.Fatalf("database.New() error = %v", err) - } - - _, err = New(lc, Params{ - Logger: log, Database: db, Config: cfg, - MetricsRegistry: prometheus.NewRegistry(), - }) - if err != nil { - t.Fatalf("New() error = %v", err) - } - - lc.RequireStart() - t.Cleanup(lc.RequireStop) - - _, err = os.Stat(filepath.Join(stateDir, "cache", "variants")) - gotDiskCache := err == nil - - if gotDiskCache != tc.wantDiskCache { - t.Errorf("cache directories created = %v, want %v", - gotDiskCache, tc.wantDiskCache) - } - }) - } -} diff --git a/internal/handlers/fetcher_internal_test.go b/internal/handlers/fetcher_internal_test.go deleted file mode 100644 index e358f1b..0000000 --- a/internal/handlers/fetcher_internal_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package handlers - -import ( - "net/http" - "net/netip" - "path/filepath" - "testing" - "time" - - "github.com/go-chi/chi/v5" - "github.com/prometheus/client_golang/prometheus" - "go.uber.org/fx" - "go.uber.org/fx/fxtest" - - "sneak.berlin/go/pixa/internal/config" - "sneak.berlin/go/pixa/internal/database" - "sneak.berlin/go/pixa/internal/globals" - "sneak.berlin/go/pixa/internal/healthcheck" - "sneak.berlin/go/pixa/internal/logger" -) - -// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as -// pixad does, in an fx app that provides no fetcher, and requests an image -// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL -// check accepts that address; only the dialer that refuses internal -// addresses checks blocked_networks, so the answer is 403 only if the -// fetcher the handlers build from the config connects with that dialer. Any -// other dialer would try to connect until the upstream fetch timeout, which -// is short so that the test then fails quickly. -func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) { - t.Parallel() - - const host = "192.0.2.10" - - stateDir := t.TempDir() - cfg := &config.Config{ - SigningKey: testSigningKey, - StateDir: stateDir, - DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), - AllowlistHosts: []string{host}, - BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}, - UpstreamFetchTimeout: 2 * time.Second, - // With no connection slots, the fetch would fail before dialing. - UpstreamConnections: config.DefaultUpstreamConnections, - } - - var h *Handlers - - app := fxtest.New(t, - fx.Supply(cfg), - fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New, - func() prometheus.Registerer { return prometheus.NewRegistry() }), - fx.Populate(&h), - ) - app.RequireStart() - t.Cleanup(app.RequireStop) - - r := chi.NewRouter() - r.Get("/v1/image/*", h.HandleImage()) - - rec := sendGet(t, r, photoURL(host)) - checkErrorBody(t, rec, http.StatusForbidden, "forbidden") -} diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go index eeb80fe..4a31a7b 100644 --- a/internal/handlers/first_bytes_internal_test.go +++ b/internal/handlers/first_bytes_internal_test.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -36,17 +37,23 @@ func newRoutesFetchingThrough( cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("imgcache.NewCache() error = %v", err) } svc, err := imgcache.NewService(&imgcache.ServiceConfig{ - Cache: cache, - Fetcher: fetcher, - SigningKey: testSigningKey, - Allowlist: []string{allowlistedHost}, - Logger: log, + Cache: cache, + Fetcher: fetcher, + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: testSigningKey, + Allowlist: []string{allowlistedHost}, + MaxResponseSize: 50 << 20, + Logger: log, }) if err != nil { t.Fatalf("imgcache.NewService() error = %v", err) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index ba42481..555e9c6 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -4,19 +4,15 @@ package handlers import ( "context" "encoding/json" - "errors" "log/slog" "net/http" "time" - "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/config" - "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/healthcheck" - "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/session" @@ -28,29 +24,19 @@ type Params struct { Logger *logger.Logger Healthcheck *healthcheck.Healthcheck - Database *database.Database Config *config.Config - // MetricsRegistry is the registry the image cache registers its - // counters on. pixad provides Prometheus' default registry, the one - // /metrics serves. - MetricsRegistry prometheus.Registerer - - // Fetcher, when provided, fetches upstream images in place of the - // fetcher the handlers build from the config. Only tests provide one; - // pixad does not. - Fetcher httpfetcher.Fetcher `optional:"true"` + ImageService *imgcache.Service + SessionManager *session.Manager + EncryptedURLGenerator *encurl.Generator } // Handlers provides HTTP request handlers. type Handlers struct { log *slog.Logger hc *healthcheck.Healthcheck - db *database.Database config *config.Config - fetcher httpfetcher.Fetcher imgSvc *imgcache.Service - imgCache *imgcache.Cache sessMgr *session.Manager encGen *encurl.Generator csrfProtect func(http.Handler) http.Handler @@ -58,50 +44,25 @@ type Handlers struct { // refererBlocklist matches the hosts of referer_blocklist; its IsAllowed // reports whether a URL's host is on that list. refererBlocklist *allowlist.HostAllowList - - // metricsRegistry is the registry the image cache registers its - // counters on. - metricsRegistry prometheus.Registerer } // New creates a new Handlers instance. -func New(lc fx.Lifecycle, params Params) (*Handlers, error) { +func New(_ fx.Lifecycle, params Params) (*Handlers, error) { csrfProtect, err := newCSRFProtect(params.Config.SigningKey, params.Config.Debug) if err != nil { return nil, err } - s := &Handlers{ + return &Handlers{ log: params.Logger.Get(), hc: params.Healthcheck, - db: params.Database, config: params.Config, - fetcher: params.Fetcher, + imgSvc: params.ImageService, + sessMgr: params.SessionManager, + encGen: params.EncryptedURLGenerator, csrfProtect: csrfProtect, refererBlocklist: allowlist.New(params.Config.RefererBlocklist), - metricsRegistry: params.MetricsRegistry, - } - - lc.Append(fx.Hook{ - //nolint:contextcheck // the cache's goroutines outlive OnStart; OnStop stops them - OnStart: func(_ context.Context) error { - return s.initImageService() - }, - // The pending access times are written here, before the database's - // stop hook closes it. - OnStop: func(ctx context.Context) error { - if s.imgCache == nil { - return nil - } - - return errors.Join( - s.imgCache.StopEviction(ctx), - s.imgCache.StopPendingAccessTimeWrites(ctx), - ) - }, - }) - - return s, nil + }, nil } // WaitForProcessing waits until no image is being processed, or until ctx @@ -110,96 +71,6 @@ func (s *Handlers) WaitForProcessing(ctx context.Context) int { return s.imgSvc.WaitForProcessing(ctx) } -// newCacheConfig builds the image cache's configuration from cfg. -// cache_max_bytes: 0 disables the disk cache entirely; any other value -// is the eviction limit in bytes; when it is omitted, the cache works -// out the default limit itself. -func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig { - return imgcache.CacheConfig{ - StateDir: cfg.StateDir, - NegativeTTL: imgcache.DefaultNegativeTTL, - MaxBytes: cfg.CacheMaxBytes, - UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit, - DisableDiskCache: cfg.CacheMaxBytesExplicit && cfg.CacheMaxBytes == 0, - Logger: log, - } -} - -// initImageService initializes the image cache and service. -func (s *Handlers) initImageService() error { - cache, err := imgcache.NewCache( - s.db.DB(), newCacheConfig(s.config, s.log), s.metricsRegistry) - if err != nil { - return err - } - - s.imgCache = cache - - // Background eviction: startup reconciliation, then periodic and - // write-pressure passes, and the removal of expired negative cache - // rows, which is all it does when the disk cache is disabled. - cache.StartEviction(imgcache.DefaultEvictionInterval) - - // Writes the access times that requests hand over, so that no request - // waits for the database to record them - cache.StartPendingAccessTimeWrites() - - // Create the fetcher config - fetcherCfg := httpfetcher.DefaultConfig() - fetcherCfg.AllowHTTP = s.config.AllowHTTP - fetcherCfg.Timeout = s.config.UpstreamFetchTimeout - fetcherCfg.MaxResponseSize = s.config.UpstreamMaxResponseSize - - if s.config.UpstreamConnectionsPerHost > 0 { - fetcherCfg.MaxConnectionsPerHost = s.config.UpstreamConnectionsPerHost - } - - fetcherCfg.MaxConnections = s.config.UpstreamConnections - fetcherCfg.BlockedNetworks = s.config.BlockedNetworks - - // Create the service. With no fetcher provided, it builds its own from - // fetcherCfg. - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ - Cache: cache, - FetcherConfig: fetcherCfg, - Fetcher: s.fetcher, - SigningKey: s.config.SigningKey, - Allowlist: s.config.AllowlistHosts, - MaxConcurrentProcessing: s.config.MaxConcurrentProcessing, - Logger: s.log, - }) - if err != nil { - return err - } - - s.imgSvc = svc - s.log.Info("image service initialized") - - // Initialize session manager (signing key is validated at config load - // time). Session cookies are always Secure/HttpOnly/SameSite=Strict. - // The random value that each logout replaces is saved in the database, - // so it outlasts a restart. - sessMgr, err := session.NewManagerWithDatabase( - context.Background(), s.config.SigningKey, s.db.DB()) - if err != nil { - return err - } - - s.sessMgr = sessMgr - - // Initialize encrypted URL generator - encGen, err := encurl.NewGenerator(s.config.SigningKey) - if err != nil { - return err - } - - s.encGen = encGen - - s.log.Info("session manager and URL generator initialized") - - return nil -} - func (s *Handlers) respondJSON(w http.ResponseWriter, data any, status int) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index d175569..e463dda 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -48,16 +49,22 @@ func setupTestHandler(t *testing.T) *testFixtures { cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ StateDir: tmpDir, NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } svc, err := imgcache.NewService(&imgcache.ServiceConfig{ - Cache: cache, - Fetcher: newMockFetcher(mockFS), - SigningKey: "test-signing-key-must-be-32-chars", - Allowlist: []string{goodHost}, + Cache: cache, + Fetcher: newMockFetcher(mockFS), + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: "test-signing-key-must-be-32-chars", + Allowlist: []string{goodHost}, + MaxResponseSize: 50 << 20, }) if err != nil { t.Fatalf("failed to create service: %v", err) diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index 79c4a38..b27465d 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -34,6 +35,7 @@ func newSignedHostServer( cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("imgcache.NewCache() error = %v", err) @@ -46,8 +48,13 @@ func newSignedHostServer( Fetcher: newMockFetcher(fstest.MapFS{ signedHost + photoPath: &fstest.MapFile{Data: jpegData}, }), - SigningKey: testSigningKey, - Logger: log, + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: testSigningKey, + MaxResponseSize: 50 << 20, + Logger: log, }) if err != nil { t.Fatalf("imgcache.NewService() error = %v", err) diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index ef7130a..e4143ef 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -15,6 +15,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/signature" @@ -32,16 +33,22 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } svc, err := imgcache.NewService(&imgcache.ServiceConfig{ - Cache: cache, - Fetcher: fetcher, - SigningKey: testSigningKey, - Allowlist: []string{allowlistedHost}, + Cache: cache, + Fetcher: fetcher, + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: testSigningKey, + Allowlist: []string{allowlistedHost}, + MaxResponseSize: 50 << 20, }) if err != nil { t.Fatalf("failed to create service: %v", err) @@ -169,7 +176,12 @@ func TestHandleImage_ErrorAnswers(t *testing.T) { photos := newPhotoFetcher(t, signedHost, parentHost, siblingHost, subdomainHost, appendedHost) // The real fetcher refuses localhost before any lookup or connection. - realFetcher := httpfetcher.New(httpfetcher.DefaultConfig()) + realFetcher := httpfetcher.New(&httpfetcher.Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + }) exp := time.Now().Add(time.Hour) expired := time.Now().Add(-time.Hour) diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index 6cdfbe6..f3160a5 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" ) @@ -51,16 +52,22 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("imgcache.NewCache() error = %v", err) } svc, err := imgcache.NewService(&imgcache.ServiceConfig{ - Cache: cache, - Fetcher: fetcher, - SigningKey: testSigningKey, - Allowlist: []string{allowlistedHost}, + Cache: cache, + Fetcher: fetcher, + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: testSigningKey, + Allowlist: []string{allowlistedHost}, + MaxResponseSize: 50 << 20, }) if err != nil { t.Fatalf("imgcache.NewService() error = %v", err) diff --git a/internal/httpfetcher/blocked_networks_internal_test.go b/internal/httpfetcher/blocked_networks_internal_test.go index dfbf8f5..d28d599 100644 --- a/internal/httpfetcher/blocked_networks_internal_test.go +++ b/internal/httpfetcher/blocked_networks_internal_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/netip" "testing" + "time" ) // TestIsPrivateIPBlocksSpecialRanges covers the internal and special-use @@ -70,12 +71,15 @@ func transportOf(t *testing.T, f *HTTPFetcher) *http.Transport { func TestDialerEnforcesBlockedNetworks(t *testing.T) { t.Parallel() - cfg := DefaultConfig() - // TEST-NET-2 (198.51.100.0/24) is public to the built-in check, so - // blocking it can only come from the operator-supplied list. - cfg.BlockedNetworks = []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")} - - transport := transportOf(t, New(cfg)) + transport := transportOf(t, New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + // TEST-NET-2 (198.51.100.0/24) is public to the built-in check, so + // blocking it can only come from the operator-supplied list. + BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}, + })) blocked := []string{ "198.51.100.5:80", // operator-supplied range diff --git a/internal/httpfetcher/dial_context_internal_test.go b/internal/httpfetcher/dial_context_internal_test.go index 4553799..036bd78 100644 --- a/internal/httpfetcher/dial_context_internal_test.go +++ b/internal/httpfetcher/dial_context_internal_test.go @@ -4,6 +4,7 @@ import ( "errors" "net" "testing" + "time" ) // TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built @@ -13,7 +14,12 @@ func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) { t.Parallel() srv := startUpstream(t) - transport := transportOf(t, New(DefaultConfig())) + transport := transportOf(t, New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + })) addr := srv.Listener.Addr().String() @@ -32,10 +38,14 @@ func TestDialContextReplacesOnlyTheDialer(t *testing.T) { srv := startUpstream(t) dialer := &recordingDialer{target: srv.Listener.Addr().String()} - cfg := DefaultConfig() - cfg.AllowHTTP = true - cfg.DialContext = dialer.dialContext - f := New(cfg) + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + AllowHTTP: true, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + DialContext: dialer.dialContext, + }) if body := fetchBody(t, f, "/image"); body != imagePayload { t.Errorf("body = %q, want %q", body, imagePayload) diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index b6523a8..f2f3e83 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -120,7 +120,12 @@ func newServerFetcher( t.Helper() if cfg == nil { - cfg = DefaultConfig() + cfg = &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + } } cfg.AllowHTTP = true @@ -280,8 +285,12 @@ func TestFetchMaxResponseSizeEnforced(t *testing.T) { srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxResponseSize = 8 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 8, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + } f, _ := newServerFetcher(t, srv, cfg) @@ -310,8 +319,12 @@ func fetchWithMaxResponseSize( srv := httptest.NewServer(handler) t.Cleanup(srv.Close) - cfg := DefaultConfig() - cfg.MaxResponseSize = maxResponseSize + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: maxResponseSize, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + } f, _ := newServerFetcher(t, srv, cfg) @@ -436,8 +449,12 @@ func TestFetchSemaphoreReleasedOnError(t *testing.T) { srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxConnectionsPerHost = 1 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 1, + MaxConnections: 64, + } f, _ := newServerFetcher(t, srv, cfg) @@ -466,8 +483,12 @@ func assertSlotReleasedByClose( srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxConnectionsPerHost = 1 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 1, + MaxConnections: 64, + } f, _ := newServerFetcher(t, srv, cfg) diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 8988d55..c9d2eae 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -24,16 +24,15 @@ import ( // Fetcher configuration constants. const ( - DefaultFetchTimeout = 30 * time.Second - DefaultMaxResponseSize = 50 << 20 // 50MB - DefaultTLSTimeout = 10 * time.Second - DefaultMaxIdleConns = 100 - DefaultIdleConnTimeout = 90 * time.Second - DefaultMaxRedirects = 10 - DefaultMaxConnectionsPerHost = 20 - DefaultMaxConnections = 64 + DefaultTLSTimeout = 10 * time.Second + DefaultMaxIdleConns = 100 + DefaultIdleConnTimeout = 90 * time.Second + DefaultMaxRedirects = 10 ) +// userAgent is the User-Agent header sent to upstream servers. +const userAgent = "pixa/1.0" + // ConnectionWaitTimeout is how long Fetch waits for a free connection when // MaxConnections fetches are already in progress. const ConnectionWaitTimeout = 10 * time.Second @@ -120,16 +119,26 @@ type FetchResult struct { TLSCipherSuite string } -// Config holds configuration for the upstream fetcher. +// allowedContentTypes returns the content types an upstream response may +// have, which Fetch also sends as its Accept header. +func allowedContentTypes() []string { + return []string{ + contentTypeJPEG, + contentTypePNG, + contentTypeGIF, + contentTypeWebP, + contentTypeAVIF, + contentTypeJXL, + } +} + +// Config holds configuration for the upstream fetcher. Every field but +// AllowHTTP, BlockedNetworks and DialContext must be set. type Config struct { // Timeout for upstream requests. Timeout time.Duration // MaxResponseSize is the maximum allowed response body size. MaxResponseSize int64 - // UserAgent to send to upstream servers. - UserAgent string - // AllowedContentTypes is an allow list of MIME types to accept. - AllowedContentTypes []string // AllowHTTP allows non-TLS connections (for testing only). AllowHTTP bool // MaxConnectionsPerHost limits concurrent connections to each upstream host. @@ -147,26 +156,6 @@ type Config struct { DialContext func(ctx context.Context, network, addr string) (net.Conn, error) } -// DefaultConfig returns a Config with sensible defaults. -func DefaultConfig() *Config { - return &Config{ - Timeout: DefaultFetchTimeout, - MaxResponseSize: DefaultMaxResponseSize, - UserAgent: "pixa/1.0", - AllowedContentTypes: []string{ - contentTypeJPEG, - contentTypePNG, - contentTypeGIF, - contentTypeWebP, - contentTypeAVIF, - contentTypeJXL, - }, - AllowHTTP: false, - MaxConnectionsPerHost: DefaultMaxConnectionsPerHost, - MaxConnections: DefaultMaxConnections, - } -} - // HTTPFetcher implements Fetcher with SSRF protection and connection limits // per host and for all hosts together. type HTTPFetcher struct { @@ -194,10 +183,6 @@ type hostSemaphore struct { // New creates a new HTTPFetcher with SSRF protection. func New(config *Config) *HTTPFetcher { - if config == nil { - config = DefaultConfig() - } - // Unless config.DialContext replaces it, the transport connects with // the SSRF-safe dialer, which re-resolves and re-checks at connect time // (closing the DNS-rebinding window) against both the built-in ranges @@ -280,8 +265,8 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro Header: make(http.Header), } - req.Header.Set("User-Agent", f.config.UserAgent) - req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", ")) + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", strings.Join(allowedContentTypes(), ", ")) // The ID of the request this fetch serves, so the fetch can be found in // the upstream host's logs @@ -460,7 +445,7 @@ func (f *HTTPFetcher) isAllowedContentType(contentType string) bool { // Extract the MIME type without parameters mediaType := strings.TrimSpace(strings.Split(contentType, ";")[0]) - for _, allowed := range f.config.AllowedContentTypes { + for _, allowed := range allowedContentTypes() { if strings.EqualFold(mediaType, allowed) { return true } diff --git a/internal/httpfetcher/httpfetcher_internal_test.go b/internal/httpfetcher/httpfetcher_internal_test.go index c7a8970..7341d28 100644 --- a/internal/httpfetcher/httpfetcher_internal_test.go +++ b/internal/httpfetcher/httpfetcher_internal_test.go @@ -7,60 +7,21 @@ import ( "net" "testing" "testing/fstest" + "time" ) // testHost is the hostname used by mock fetch tests. const testHost = "example.com" -func TestDefaultConfig(t *testing.T) { - t.Parallel() - - cfg := DefaultConfig() - - if cfg.Timeout != DefaultFetchTimeout { - t.Errorf("Timeout = %v, want %v", cfg.Timeout, DefaultFetchTimeout) - } - - if cfg.MaxResponseSize != DefaultMaxResponseSize { - t.Errorf("MaxResponseSize = %d, want %d", cfg.MaxResponseSize, DefaultMaxResponseSize) - } - - if cfg.MaxConnectionsPerHost != DefaultMaxConnectionsPerHost { - t.Errorf("MaxConnectionsPerHost = %d, want %d", - cfg.MaxConnectionsPerHost, DefaultMaxConnectionsPerHost) - } - - if cfg.AllowHTTP { - t.Error("AllowHTTP should default to false") - } - - if len(cfg.AllowedContentTypes) == 0 { - t.Error("AllowedContentTypes should not be empty") - } -} - -func TestNewWithNilConfigUsesDefaults(t *testing.T) { - t.Parallel() - - f := New(nil) - - if f == nil { - t.Fatal("New(nil) returned nil") - } - - if f.config == nil { - t.Fatal("config should be populated from DefaultConfig") - } - - if f.config.Timeout != DefaultFetchTimeout { - t.Errorf("Timeout = %v, want %v", f.config.Timeout, DefaultFetchTimeout) - } -} - func TestIsAllowedContentType(t *testing.T) { t.Parallel() - f := New(DefaultConfig()) + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + }) tests := []struct { contentType string diff --git a/internal/httpfetcher/jpegxl_internal_test.go b/internal/httpfetcher/jpegxl_internal_test.go index 8be65ec..4df240b 100644 --- a/internal/httpfetcher/jpegxl_internal_test.go +++ b/internal/httpfetcher/jpegxl_internal_test.go @@ -1,16 +1,26 @@ package httpfetcher -import "testing" +import ( + "testing" + "time" +) // TestJPEGXLContentType verifies that the fetcher accepts an upstream answer -// of type image/jxl by default, and that the mock fetcher serves a .jxl file -// as image/jxl. +// of type image/jxl, and that the mock fetcher serves a .jxl file as +// image/jxl. func TestJPEGXLContentType(t *testing.T) { t.Parallel() const jxlType = "image/jxl" - if !New(DefaultConfig()).isAllowedContentType(jxlType) { + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + }) + + if !f.isAllowedContentType(jxlType) { t.Errorf("isAllowedContentType(%q) = false, want true", jxlType) } diff --git a/internal/httpfetcher/max_connections_internal_test.go b/internal/httpfetcher/max_connections_internal_test.go index a776460..d414178 100644 --- a/internal/httpfetcher/max_connections_internal_test.go +++ b/internal/httpfetcher/max_connections_internal_test.go @@ -18,14 +18,6 @@ func imageURLOnPort(port int) string { "/image" } -func TestDefaultConfigMaxConnections(t *testing.T) { - t.Parallel() - - if got := DefaultConfig().MaxConnections; got != DefaultMaxConnections { - t.Errorf("MaxConnections = %d, want %d", got, DefaultMaxConnections) - } -} - // TestFetchLimitsConnectionsToAllHostsTogether checks that MaxConnections // counts the fetches to every host together, apart from the per-host // limit: with MaxConnections at 2 and two responses open from two hosts, a @@ -37,8 +29,12 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxConnections = 2 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 2, + } f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond @@ -94,8 +90,12 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxConnections = 1 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 1, + } f, _ := newServerFetcher(t, srv, cfg) @@ -163,9 +163,12 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxConnections = 1 - cfg.MaxConnectionsPerHost = 1 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 1, + MaxConnections: 1, + } f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond @@ -236,8 +239,12 @@ func TestFetchReleasesConnectionOnError(t *testing.T) { srv := startUpstream(t) - cfg := DefaultConfig() - cfg.MaxConnections = 1 + cfg := &Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 1, + } f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond diff --git a/internal/imageprocessor/avif_too_large_internal_test.go b/internal/imageprocessor/avif_too_large_internal_test.go index 1c1a576..15b184c 100644 --- a/internal/imageprocessor/avif_too_large_internal_test.go +++ b/internal/imageprocessor/avif_too_large_internal_test.go @@ -69,7 +69,10 @@ func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - _, err := New(Params{}).Process(t.Context(), bytes.NewReader(tt.input), + _, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(tt.input), &imagerequest.Transform{ Size: imagerequest.Size{Width: tt.width, Height: tt.height}, Format: tt.format, diff --git a/internal/imageprocessor/computed_side_internal_test.go b/internal/imageprocessor/computed_side_internal_test.go index fbd49e6..6bfb23a 100644 --- a/internal/imageprocessor/computed_side_internal_test.go +++ b/internal/imageprocessor/computed_side_internal_test.go @@ -38,7 +38,10 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) { input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: tt.size, @@ -104,7 +107,10 @@ func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) { input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: tt.size, diff --git a/internal/imageprocessor/deadline_internal_test.go b/internal/imageprocessor/deadline_internal_test.go index fb9b2ea..9f6d394 100644 --- a/internal/imageprocessor/deadline_internal_test.go +++ b/internal/imageprocessor/deadline_internal_test.go @@ -84,7 +84,7 @@ func checkProcessStopsAtDeadline( ) { t.Helper() - proc := New(Params{MaxConcurrentProcessing: 1}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) result, err := proc.Process(ctx, input, smallJPEGTransform()) if result != nil { diff --git a/internal/imageprocessor/empty_format_internal_test.go b/internal/imageprocessor/empty_format_internal_test.go index 4feda08..48bba8e 100644 --- a/internal/imageprocessor/empty_format_internal_test.go +++ b/internal/imageprocessor/empty_format_internal_test.go @@ -14,7 +14,10 @@ import ( func TestImageProcessor_EmptyFormatRefused(t *testing.T) { t.Parallel() - _, err := New(Params{}).Process( + _, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process( t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), &imagerequest.Transform{}, ) diff --git a/internal/imageprocessor/encoder_settings_internal_test.go b/internal/imageprocessor/encoder_settings_internal_test.go index 8cf3388..dc93d76 100644 --- a/internal/imageprocessor/encoder_settings_internal_test.go +++ b/internal/imageprocessor/encoder_settings_internal_test.go @@ -18,7 +18,10 @@ func processedSize( ) int64 { t.Helper() - result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), transform) + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), transform) if err != nil { t.Fatalf("Process() error = %v", err) } diff --git a/internal/imageprocessor/fit_cover_internal_test.go b/internal/imageprocessor/fit_cover_internal_test.go index 030074e..a12d155 100644 --- a/internal/imageprocessor/fit_cover_internal_test.go +++ b/internal/imageprocessor/fit_cover_internal_test.go @@ -36,7 +36,10 @@ func TestImageProcessor_FitCoverServesThinSource(t *testing.T) { input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: tt.size, @@ -106,7 +109,10 @@ func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) { input := createBandsPNG(t, tt.inputWidth, tt.inputHeight) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: imagerequest.Size{Width: 50, Height: 50}, @@ -230,7 +236,10 @@ func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) { input := createSquarePNG(t, tt.side, tt.squareSide) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: tt.size, diff --git a/internal/imageprocessor/fit_outside_internal_test.go b/internal/imageprocessor/fit_outside_internal_test.go index 3d331a5..3c72d59 100644 --- a/internal/imageprocessor/fit_outside_internal_test.go +++ b/internal/imageprocessor/fit_outside_internal_test.go @@ -41,7 +41,10 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) { input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: tt.size, @@ -95,7 +98,10 @@ func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) { input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{ Size: tt.size, diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index c66148f..afe216e 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -11,7 +11,6 @@ import ( "time" "github.com/davidbyttow/govips/v2/vips" - "sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/imagerequest" ) @@ -79,10 +78,6 @@ type Result struct { // Images larger than this are rejected to prevent DoS via decompression bombs. const MaxInputDimension = 8192 -// DefaultMaxInputBytes is the default maximum input size in bytes (50 MiB). -// This matches the default upstream fetcher limit. -const DefaultMaxInputBytes = 50 << 20 - // ErrInputTooLarge is returned when input image dimensions exceed MaxInputDimension. var ErrInputTooLarge = errors.New("input image dimensions exceed maximum") @@ -142,35 +137,22 @@ type ImageProcessor struct { processingWaitTimeout time.Duration } -// Params holds configuration for creating an ImageProcessor. -// Zero values use sensible defaults (MaxInputBytes defaults to DefaultMaxInputBytes). +// Params holds configuration for creating an ImageProcessor. Both fields +// must be at least 1. type Params struct { // MaxInputBytes is the maximum allowed input size in bytes. - // If <= 0, DefaultMaxInputBytes is used. MaxInputBytes int64 // MaxConcurrentProcessing is the most images processed at once. - // If <= 0, config.DefaultMaxConcurrentProcessing() is used. MaxConcurrentProcessing int } // New creates a new image processor with the given parameters. -// A zero-value Params{} uses sensible defaults. func New(params Params) *ImageProcessor { initVips() - maxInputBytes := params.MaxInputBytes - if maxInputBytes <= 0 { - maxInputBytes = DefaultMaxInputBytes - } - - maxConcurrentProcessing := params.MaxConcurrentProcessing - if maxConcurrentProcessing <= 0 { - maxConcurrentProcessing = config.DefaultMaxConcurrentProcessing() - } - return &ImageProcessor{ - maxInputBytes: maxInputBytes, - processingSemaphore: make(chan struct{}, maxConcurrentProcessing), + maxInputBytes: params.MaxInputBytes, + processingSemaphore: make(chan struct{}, params.MaxConcurrentProcessing), processingWaitTimeout: ProcessingWaitTimeout, } } diff --git a/internal/imageprocessor/imageprocessor_internal_test.go b/internal/imageprocessor/imageprocessor_internal_test.go index 90c45b9..e9b3656 100644 --- a/internal/imageprocessor/imageprocessor_internal_test.go +++ b/internal/imageprocessor/imageprocessor_internal_test.go @@ -119,7 +119,7 @@ func detectMIME(data []byte) string { func TestImageProcessor_ResizeJPEG(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() input := createTestJPEG(t, 800, 600) @@ -165,7 +165,7 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) { func TestImageProcessor_ConvertToPNG(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() input := createTestJPEG(t, 200, 150) @@ -201,7 +201,7 @@ func processAndCheckSize( ) { t.Helper() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() input := createTestJPEG(t, inputW, inputH) @@ -239,7 +239,7 @@ func TestImageProcessor_OriginalSize(t *testing.T) { func TestImageProcessor_FitContain(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() // 800x400 image (2:1 aspect) into 400x400 box with contain @@ -285,7 +285,7 @@ func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) { func TestImageProcessor_ProcessPNG(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() input := createTestPNG(t, 400, 300) @@ -330,7 +330,7 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() input := createTestJPEG(t, tt.width, tt.height) @@ -356,7 +356,7 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() // Create an image at exactly MaxInputDimension - should be accepted @@ -387,7 +387,7 @@ func encodeAndCheck( ) { t.Helper() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() input := createTestJPEG(t, 200, 150) @@ -436,7 +436,7 @@ func TestImageProcessor_EncodeWebP(t *testing.T) { func TestImageProcessor_DecodeAVIF(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) ctx := context.Background() // Load test AVIF file @@ -478,7 +478,7 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) { // Create a processor with a very small byte limit const limit = 1024 - proc := New(Params{MaxInputBytes: limit}) + proc := New(Params{MaxInputBytes: limit, MaxConcurrentProcessing: 1}) ctx := context.Background() // Create a valid JPEG that exceeds the byte limit @@ -511,7 +511,7 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) { input := createTestJPEG(t, 10, 10) limit := int64(len(input)) * 10 // 10× headroom - proc := New(Params{MaxInputBytes: limit}) + proc := New(Params{MaxInputBytes: limit, MaxConcurrentProcessing: 1}) ctx := context.Background() transform := &imagerequest.Transform{ @@ -529,22 +529,6 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) { defer func() { _ = result.Content.Close() }() } -func TestImageProcessor_DefaultMaxInputBytes(t *testing.T) { - t.Parallel() - - // Passing 0 should use the default - proc := New(Params{}) - if proc.maxInputBytes != DefaultMaxInputBytes { - t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes) - } - - // Passing negative should also use the default - proc = New(Params{MaxInputBytes: -1}) - if proc.maxInputBytes != DefaultMaxInputBytes { - t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes) - } -} - func TestImageProcessor_EncodeAVIF(t *testing.T) { t.Parallel() @@ -558,7 +542,10 @@ func processAndDecode( ) *vips.ImageRef { t.Helper() - result, err := New(Params{}).Process( + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process( context.Background(), bytes.NewReader(input), transform, ) if err != nil { diff --git a/internal/imageprocessor/jpegxl_output_internal_test.go b/internal/imageprocessor/jpegxl_output_internal_test.go index 44b3f6a..29afb32 100644 --- a/internal/imageprocessor/jpegxl_output_internal_test.go +++ b/internal/imageprocessor/jpegxl_output_internal_test.go @@ -21,7 +21,10 @@ func TestImageProcessor_EncodeJPEGXL(t *testing.T) { Format: imagerequest.FormatJXL, } - result, err := New(Params{}).Process( + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process( t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), transform, ) if err != nil { @@ -64,7 +67,10 @@ func TestImageProcessor_JPEGXLQuality(t *testing.T) { outputBytes := make(map[int]int64) for _, quality := range []int{30, 90} { - result, err := New(Params{}).Process(t.Context(), + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process(t.Context(), bytes.NewReader(input), &imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: quality}) if err != nil { diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go index c3d428e..fff69d0 100644 --- a/internal/imageprocessor/max_concurrent_processing_internal_test.go +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "runtime" "strings" "sync" "testing" @@ -117,47 +116,15 @@ func waitForEntries(t *testing.T, entered <-chan struct{}, count int) { } } -func TestNewDefaultsMaxConcurrentProcessingToHalfTheCPUs(t *testing.T) { +func TestNewMakesMaxConcurrentProcessingSlots(t *testing.T) { t.Parallel() - for _, limit := range []int{0, -1} { - proc := New(Params{MaxConcurrentProcessing: limit}) - if got := cap(proc.processingSemaphore); got != max(runtime.GOMAXPROCS(0)/2, 1) { - t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, half the CPUs", - limit, got, max(runtime.GOMAXPROCS(0)/2, 1)) - } - } - - proc := New(Params{MaxConcurrentProcessing: 3}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 3}) if got := cap(proc.processingSemaphore); got != 3 { t.Errorf("MaxConcurrentProcessing 3: %d slots, want 3", got) } } -// TestNewDefaultIsHalfTheCPUsAndAtLeastOne checks that New, given no -// MaxConcurrentProcessing, makes one slot for every two CPUs Go uses, -// rounded down, and one slot when Go uses one CPU. -// -//nolint:paralleltest // it sets GOMAXPROCS, which other tests read -func TestNewDefaultIsHalfTheCPUsAndAtLeastOne(t *testing.T) { - cpusBefore := runtime.GOMAXPROCS(0) - - t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) }) - - for _, tc := range []struct{ cpus, want int }{ - {1, 1}, {2, 1}, {3, 1}, {8, 4}, - } { - runtime.GOMAXPROCS(tc.cpus) - - got := cap(New(Params{}).processingSemaphore) - t.Logf("GOMAXPROCS %d: %d slots", tc.cpus, got) - - if got != tc.want { - t.Errorf("GOMAXPROCS %d: %d slots, want %d", tc.cpus, got, tc.want) - } - } -} - // TestProcessNeverExceedsMaxConcurrentProcessing starts more Process calls // than MaxConcurrentProcessing allows and holds the first ones inside // Process until the test lets them go. No more than the limit may be @@ -171,7 +138,7 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) { calls = 6 ) - proc := New(Params{MaxConcurrentProcessing: limit}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: limit}) input := createTestJPEG(t, 50, 50) counter := &readingCounter{} @@ -218,7 +185,7 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) { func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { t.Parallel() - proc := New(Params{MaxConcurrentProcessing: 1}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc.processingWaitTimeout = 100 * time.Millisecond input := createTestJPEG(t, 10, 10) @@ -334,7 +301,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) { func TestWaitForProcessing(t *testing.T) { t.Parallel() - proc := New(Params{MaxConcurrentProcessing: 2}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2}) gate := make(chan struct{}) entered := make(chan struct{}, 1) diff --git a/internal/imageprocessor/orig_format_internal_test.go b/internal/imageprocessor/orig_format_internal_test.go index a0ad8dc..70b97ef 100644 --- a/internal/imageprocessor/orig_format_internal_test.go +++ b/internal/imageprocessor/orig_format_internal_test.go @@ -27,7 +27,10 @@ func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) { t.Fatalf("failed to save test TIFF: %v", err) } - result, err := New(Params{}).Process( + result, err := New(Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }).Process( t.Context(), bytes.NewReader(tiff), &imagerequest.Transform{Format: imagerequest.FormatOriginal}, ) diff --git a/internal/imageprocessor/source_formats_internal_test.go b/internal/imageprocessor/source_formats_internal_test.go index d724976..aaf3aec 100644 --- a/internal/imageprocessor/source_formats_internal_test.go +++ b/internal/imageprocessor/source_formats_internal_test.go @@ -14,7 +14,7 @@ import ( func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { t.Parallel() - proc := New(Params{}) + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) for _, format := range []imagerequest.Format{ imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 08ccd9a..e1b22a7 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -32,23 +32,12 @@ type CacheConfig struct { NegativeTTL time.Duration // MaxBytes is the disk cache size limit in bytes that eviction - // enforces. Zero means no limit is enforced (no eviction). + // enforces. Zero turns the disk cache off entirely: no cache + // directories are created, lookups always miss, stores are no-ops, + // and the eviction goroutine only removes the expired negative cache + // rows. MaxBytes int64 - // UseDefaultMaxBytes makes NewCache replace MaxBytes with the - // default limit: 75% of the sum of the space free on the filesystem - // holding the cache and the bytes the cache already holds, at least - // DefaultCacheMaxBytesFloor. The config layer sets this when the - // operator omits cache_max_bytes. - UseDefaultMaxBytes bool - - // DisableDiskCache turns the disk cache off entirely: no cache - // directories are created, lookups always miss, stores are - // no-ops, and the eviction goroutine only removes the expired - // negative cache rows. The config layer sets this when the - // operator configures cache_max_bytes: 0. - DisableDiskCache bool - // Logger receives accounting and eviction log output. A nil // Logger means slog.Default(). Logger *slog.Logger @@ -62,10 +51,6 @@ type Cache struct { config CacheConfig log *slog.Logger - // disabled means the disk cache is turned off entirely: lookups - // always miss, stores are no-ops, and no eviction runs. - disabled bool - // Eviction machinery. The channels are created in NewCache so // stores can signal write pressure without racing StartEviction. // evictionCancel, set by StartEviction, cancels the eviction @@ -97,7 +82,7 @@ type Cache struct { transforms prometheus.Counter // reconciliationPageSize is the most rows one read of a content table - // returns in the reconciliation pass. newCache sets it to + // returns in the reconciliation pass. NewCache sets it to // defaultReconciliationPageSize; tests set it smaller. reconciliationPageSize int @@ -110,20 +95,9 @@ type Cache struct { // NewCache creates a new cache instance and registers its counters on // metricsRegistry. pixad passes Prometheus' default registry, the one -// /metrics serves. +// /metrics serves. It makes no database call. func NewCache( db *sql.DB, config CacheConfig, metricsRegistry prometheus.Registerer, -) (*Cache, error) { - return newCache(db, config, metricsRegistry, defaultFreeSpaceProbe) -} - -// newCache is NewCache with the free-space probe passed in, so tests -// can fake the free space the default limit is worked out from. -func newCache( - db *sql.DB, - config CacheConfig, - metricsRegistry prometheus.Registerer, - probe FreeSpaceProbeFunc, ) (*Cache, error) { log := config.Logger if log == nil { @@ -134,7 +108,6 @@ func newCache( db: db, config: config, log: log, - disabled: config.DisableDiskCache, evictionPressure: make(chan struct{}, 1), evictionDone: make(chan struct{}), @@ -150,7 +123,7 @@ func newCache( return nil, err } - if c.disabled { + if config.MaxBytes == 0 { return c, nil } @@ -171,20 +144,6 @@ func newCache( c.srcContent = srcContent c.variants = variants - if config.UseDefaultMaxBytes { - limit, err := c.computeDefaultMaxBytes(context.Background(), probe) - if err != nil { - return nil, err - } - - c.config.MaxBytes = limit - - log.Info("computed default cache size limit from free space and cache contents", - "cache_max_bytes", limit, - "cache_dir", filepath.Join(config.StateDir, "cache"), - ) - } - return c, nil } @@ -196,7 +155,7 @@ func newCache( func (c *Cache) GetVariant( cacheKey imagerequest.VariantKey, ) (io.ReadCloser, int64, string, error) { - if c.disabled { + if c.config.MaxBytes == 0 { return nil, 0, "", ErrNotFound } @@ -214,7 +173,7 @@ func (c *Cache) StoreSource( req *imagerequest.Request, content io.Reader, ) (imagerequest.SourceKey, error) { - if c.disabled { + if c.config.MaxBytes == 0 { return "", nil } @@ -259,7 +218,7 @@ func (c *Cache) StoreSource( func (c *Cache) StoreVariant( ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader, ) error { - if c.disabled { + if c.config.MaxBytes == 0 { return nil } @@ -293,7 +252,7 @@ func (c *Cache) StoreVariant( func (c *Cache) LookupSource( ctx context.Context, req *imagerequest.Request, ) (imagerequest.SourceKey, error) { - if c.disabled { + if c.config.MaxBytes == 0 { return "", nil } @@ -345,7 +304,7 @@ func (c *Cache) StoreNegative(ctx context.Context, req *imagerequest.Request) er func (c *Cache) GetSourceContent( key imagerequest.SourceKey, ) (io.ReadCloser, int64, error) { - if c.disabled { + if c.config.MaxBytes == 0 { return nil, 0, ErrNotFound } diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index b2d2301..d4a3c76 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -22,7 +22,10 @@ func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) - processor := imageprocessor.New(imageprocessor.Params{}) + processor := imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }) source := generateTestJPEG(t, 40, 30, color.RGBA{200, 40, 40, 255}) for _, format := range []imagerequest.Format{ diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index 56cc5a8..b37af29 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -53,6 +53,7 @@ func TestCache_CleanExpired(t *testing.T) { cache, _ := NewCache(db, CacheConfig{ StateDir: tmpDir, NegativeTTL: 1 * time.Millisecond, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) ctx := context.Background() @@ -104,6 +105,7 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) { _, err := NewCache(db, CacheConfig{ StateDir: tmpDir, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("NewCache() error = %v", err) diff --git a/internal/imgcache/cachesize.go b/internal/imgcache/cachesize.go deleted file mode 100644 index 231a907..0000000 --- a/internal/imgcache/cachesize.go +++ /dev/null @@ -1,85 +0,0 @@ -package imgcache - -import ( - "context" - "errors" - "fmt" - "math" - "path/filepath" - "syscall" -) - -// DefaultCacheMaxBytesFloor is the minimum computed default for the -// cache_max_bytes setting: 500 MiB. The floor applies only to the -// computed default (when the key is omitted from the configuration), -// never to explicitly configured values. -const DefaultCacheMaxBytesFloor int64 = 524288000 - -// freeSpaceFractionNumerator and freeSpaceFractionDenominator express -// the 75% share used for the computed default limit as integer -// arithmetic (dividing before multiplying avoids overflow). -const ( - freeSpaceFractionNumerator uint64 = 3 - freeSpaceFractionDenominator uint64 = 4 -) - -var errNegativeBlockSize = errors.New("statfs reported negative block size") - -// FreeSpaceProbeFunc reports the number of free bytes available on the -// filesystem containing path. It is a function type so tests can -// inject a fake probe instead of depending on the host disk. -type FreeSpaceProbeFunc func(path string) (uint64, error) - -// defaultFreeSpaceProbe reports free filesystem bytes via statfs on -// the given path, as available to unprivileged processes. -func defaultFreeSpaceProbe(path string) (uint64, error) { - var stat syscall.Statfs_t - - err := syscall.Statfs(path, &stat) - if err != nil { - return 0, err - } - - if stat.Bsize < 0 { - return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path) - } - - blockSize := uint64(stat.Bsize) - - return stat.Bavail * blockSize, nil -} - -// computeDefaultMaxBytes returns the default cache size limit: 75% of -// the sum of the free bytes probe reports for /cache/ and -// the bytes the cache already holds, with a floor of -// DefaultCacheMaxBytesFloor. Counting what the cache holds keeps the -// limit from shrinking as the cache fills. -func (c *Cache) computeDefaultMaxBytes( - ctx context.Context, probe FreeSpaceProbeFunc, -) (int64, error) { - cacheDir := filepath.Join(c.config.StateDir, "cache") - - freeBytes, err := probe(cacheDir) - if err != nil { - return 0, fmt.Errorf( - "default cache_max_bytes: cannot determine free space for %q: %w", - cacheDir, err) - } - - usedBytes, err := c.UsageBytes(ctx) - if err != nil { - return 0, err - } - - // Both terms are at most math.MaxInt64, so the sum cannot overflow. - //nolint:gosec // G115: UsageBytes returns the total cache usage, never negative - spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes) - - computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator - computed = min(computed, math.MaxInt64) - - limit := int64(computed) - limit = max(limit, DefaultCacheMaxBytesFloor) - - return limit, nil -} diff --git a/internal/imgcache/cachesize_internal_test.go b/internal/imgcache/cachesize_internal_test.go deleted file mode 100644 index e67f953..0000000 --- a/internal/imgcache/cachesize_internal_test.go +++ /dev/null @@ -1,189 +0,0 @@ -package imgcache - -import ( - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/prometheus/client_golang/prometheus" -) - -// Static errors returned by the stub free-space probes below. -var ( - errTestStatfsFailed = errors.New("statfs failed") - errTestProbeNotExpected = errors.New("probe must not be called") -) - -// TestComputeDefaultMaxBytesCountsWhatTheCacheHolds verifies that the -// default limit is 75% of the free space plus what the cache already -// holds, so a cache filled to its limit keeps that limit across a -// restart instead of shrinking to 75% of the space left free. -func TestComputeDefaultMaxBytesCountsWhatTheCacheHolds(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - // Empty cache, 4 GiB free -> 3 GiB default. - got, err := cache.computeDefaultMaxBytes(t.Context(), - func(string) (uint64, error) { return 4294967296, nil }) - if err != nil { - t.Fatalf("computeDefaultMaxBytes returned error: %v", err) - } - - t.Logf("default for an empty cache with 4 GiB free: %d", got) - - if got != 3221225472 { - t.Errorf("default for an empty cache = %d, want 3221225472 (75%% of 4 GiB)", - got) - } - - // The cache now holds those 3 GiB, which leaves 1 GiB free. - _, err = cache.db.ExecContext(t.Context(), - `INSERT INTO variant_content (cache_key, size_bytes) VALUES (?, ?)`, - string(testVariantKeyOne), 3221225472, - ) - if err != nil { - t.Fatalf("failed to insert variant accounting row: %v", err) - } - - got, err = cache.computeDefaultMaxBytes(t.Context(), - func(string) (uint64, error) { return 1073741824, nil }) - if err != nil { - t.Fatalf("computeDefaultMaxBytes returned error: %v", err) - } - - t.Logf("default for a cache holding 3 GiB with 1 GiB free: %d", got) - - if got != 3221225472 { - t.Errorf("default for a cache holding 3 GiB with 1 GiB free = %d, "+ - "want 3221225472 (75%% of 1 GiB + 3 GiB)", got) - } -} - -// TestComputeDefaultMaxBytesAppliesFloor verifies that when 75% of the -// free space plus what the cache holds is below 500 MiB, the default -// is floored at DefaultCacheMaxBytesFloor. -func TestComputeDefaultMaxBytesAppliesFloor(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - freeBytes uint64 - }{ - {name: "100 MiB free", freeBytes: 104857600}, - {name: "zero free", freeBytes: 0}, - {name: "just below floor threshold", freeBytes: 699050665}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - got, err := cache.computeDefaultMaxBytes(t.Context(), - func(string) (uint64, error) { return tc.freeBytes, nil }) - if err != nil { - t.Fatalf("computeDefaultMaxBytes returned error: %v", err) - } - - if got != DefaultCacheMaxBytesFloor { - t.Errorf("computeDefaultMaxBytes = %d, want floor %d", - got, DefaultCacheMaxBytesFloor) - } - }) - } -} - -// TestComputeDefaultMaxBytesPropagatesProbeError verifies that a -// failing free-space probe produces an error naming cache_max_bytes, -// instead of a silently wrong default. -func TestComputeDefaultMaxBytesPropagatesProbeError(t *testing.T) { - t.Parallel() - - cache, _ := newEvictionTestCache(t, 1<<30) - - _, err := cache.computeDefaultMaxBytes(t.Context(), - func(string) (uint64, error) { return 0, errTestStatfsFailed }) - if err == nil { - t.Fatal("probe failure must produce an error, got nil") - } - - t.Logf("got expected error: %v", err) - - if !strings.Contains(err.Error(), "cache_max_bytes") { - t.Errorf("error %q does not name cache_max_bytes", err.Error()) - } -} - -// TestNewCacheComputesDefaultMaxBytesWhenAsked verifies that with -// UseDefaultMaxBytes set, the cache's limit becomes the computed -// default, and that the probe is pointed at /cache/, which -// must be created first so statfs measures the right filesystem. -func TestNewCacheComputesDefaultMaxBytesWhenAsked(t *testing.T) { - t.Parallel() - - stateDir := t.TempDir() - wantCacheDir := filepath.Join(stateDir, "cache") - - var probedPath string - - // 4 GiB free -> 3 GiB default. - probe := func(path string) (uint64, error) { - probedPath = path - - info, err := os.Stat(path) - if err != nil || !info.IsDir() { - t.Errorf("cache directory %q was not created before probing: info=%v err=%v", - path, info, err) - } - - return 4294967296, nil - } - - cache, err := newCache(evictionTestDB(t), CacheConfig{ - StateDir: stateDir, - UseDefaultMaxBytes: true, - }, prometheus.NewRegistry(), probe) - if err != nil { - t.Fatalf("newCache returned error: %v", err) - } - - if cache.config.MaxBytes != 3221225472 { - t.Errorf("MaxBytes = %d, want computed default 3221225472", - cache.config.MaxBytes) - } - - if probedPath != wantCacheDir { - t.Errorf("free space probed at %q, want cache directory %q", - probedPath, wantCacheDir) - } -} - -// TestNewCacheKeepsExplicitMaxBytes verifies that without -// UseDefaultMaxBytes the cache keeps MaxBytes exactly as given and -// never consults the free-space probe. -func TestNewCacheKeepsExplicitMaxBytes(t *testing.T) { - t.Parallel() - - probe := func(string) (uint64, error) { - t.Error("free-space probe must not be consulted for explicit values") - - return 0, errTestProbeNotExpected - } - - cache, err := newCache(evictionTestDB(t), CacheConfig{ - StateDir: t.TempDir(), - MaxBytes: 1024, - }, prometheus.NewRegistry(), probe) - if err != nil { - t.Fatalf("newCache returned error: %v", err) - } - - if cache.config.MaxBytes != 1024 { - t.Errorf("MaxBytes = %d, want explicit 1024 (no floor, no recompute)", - cache.config.MaxBytes) - } -} diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index 7fb3e0e..226e79e 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -455,7 +455,7 @@ func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) { svc, fixtures, fetcher := setupHeldFetchService(t) svc.processor = imageprocessor.New( - imageprocessor.Params{MaxConcurrentProcessing: 1}, + imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}, ) holdProcessingSlot(t, svc.processor) diff --git a/internal/imgcache/counters_internal_test.go b/internal/imgcache/counters_internal_test.go index 7886cbf..cb11aa3 100644 --- a/internal/imgcache/counters_internal_test.go +++ b/internal/imgcache/counters_internal_test.go @@ -36,7 +36,8 @@ func TestNewCache_RegistersItsCountersOnItsRegistry(t *testing.T) { registry := prometheus.NewRegistry() - _, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry) + _, err := NewCache(evictionTestDB(t), + CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) if err != nil { t.Fatalf("NewCache() error = %v", err) } @@ -68,12 +69,14 @@ func TestNewCache_FailsWhenItsCountersCannotBeRegistered(t *testing.T) { registry := prometheus.NewRegistry() - _, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry) + _, err := NewCache(evictionTestDB(t), + CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) if err != nil { t.Fatalf("NewCache() of the first cache error = %v", err) } - _, err = NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, registry) + _, err = NewCache(evictionTestDB(t), + CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) t.Logf("NewCache() of the second cache error = %v", err) var alreadyRegistered prometheus.AlreadyRegisteredError diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 6f3cc9f..2b1feb0 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -52,18 +52,25 @@ const sqliteTimestampLayout = "2006-01-02 15:04:05" const tempFilePrefix = ".tmp-" // UsageBytes returns the total number of bytes of cache content -// tracked in the database (source content blobs plus processed -// variants). It reads the total the database keeps up to date as rows -// are added and removed, so it neither scans the cache directories nor -// sums the tables. +// tracked in the database, as the package function UsageBytes does, or 0 +// when the disk cache is off. func (c *Cache) UsageBytes(ctx context.Context) (int64, error) { - if c.disabled { + if c.config.MaxBytes == 0 { return 0, nil } + return UsageBytes(ctx, c.db) +} + +// UsageBytes returns the total number of bytes of cache content tracked +// in db (source content blobs plus processed variants). It reads the +// total the database keeps up to date as rows are added and removed, so +// it neither scans the cache directories nor sums the tables. pixad reads +// it before it builds the cache, to work out the default cache_max_bytes. +func UsageBytes(ctx context.Context, db *sql.DB) (int64, error) { var total int64 - err := c.db.QueryRowContext(ctx, + err := db.QueryRowContext(ctx, `SELECT total_size_bytes FROM cache_usage WHERE id = 1`, ).Scan(&total) if err != nil { @@ -86,9 +93,9 @@ type evictionCandidate struct { // EvictToLimit evicts least-recently-used cache entries until total // tracked usage is at or below the configured MaxBytes limit. It is a -// no-op when the cache is disabled or no limit is configured. +// no-op when the cache is disabled. func (c *Cache) EvictToLimit(ctx context.Context) error { - if c.disabled || c.config.MaxBytes <= 0 { + if c.config.MaxBytes == 0 { return nil } @@ -326,7 +333,7 @@ func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) // storing request. The notification channel has capacity one and drops // when a wakeup is already pending. func (c *Cache) notifyWritePressure() { - if c.disabled || c.config.MaxBytes <= 0 { + if c.config.MaxBytes == 0 { return } @@ -485,7 +492,7 @@ func (c *Cache) runCleanExpired(ctx context.Context) { // Once ctx is cancelled, it stops at the next file or row and returns // ctx's error. func (c *Cache) reconcileAccounting(ctx context.Context) error { - if c.disabled { + if c.config.MaxBytes == 0 { return nil } diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 99ba09e..fec51d9 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -66,15 +66,11 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) { tmpDir := t.TempDir() db := evictionTestDB(t) - // maxBytes zero mirrors the production mapping of - // cache_max_bytes: 0 (handlers sets DisableDiskCache); at the - // CacheConfig layer itself a zero MaxBytes means "no limit" for - // backwards compatibility with existing fixtures. + // maxBytes zero turns the disk cache off, as cache_max_bytes: 0 does. cache, err := NewCache(db, CacheConfig{ - StateDir: tmpDir, - NegativeTTL: 5 * time.Minute, - MaxBytes: maxBytes, - DisableDiskCache: maxBytes == 0, + StateDir: tmpDir, + NegativeTTL: 5 * time.Minute, + MaxBytes: maxBytes, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imgcache/max_concurrent_processing_internal_test.go index 2ff8372..1de6011 100644 --- a/internal/imgcache/max_concurrent_processing_internal_test.go +++ b/internal/imgcache/max_concurrent_processing_internal_test.go @@ -66,7 +66,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { svc, fixtures := SetupTestService(t) svc.processor = imageprocessor.New( - imageprocessor.Params{MaxConcurrentProcessing: 1}, + imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}, ) // A first request caches the photo as a source. diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index 3aa7a10..709f090 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -20,6 +20,7 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatal(err) @@ -69,6 +70,7 @@ func TestNegativeCache_Expired(t *testing.T) { cache, err := NewCache(db, CacheConfig{ StateDir: dir, NegativeTTL: 1 * time.Millisecond, // very short TTL + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatal(err) @@ -177,11 +179,11 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { t.Parallel() tests := []struct { - name string - disableDiskCache bool + name string + maxBytes int64 }{ - {name: "disk cache enabled", disableDiskCache: false}, - {name: "disk cache disabled", disableDiskCache: true}, + {name: "disk cache enabled", maxBytes: 1 << 30}, + {name: "disk cache disabled", maxBytes: 0}, } for _, tt := range tests { @@ -189,9 +191,9 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { t.Parallel() cache, err := NewCache(evictionTestDB(t), CacheConfig{ - StateDir: t.TempDir(), - NegativeTTL: time.Second, - DisableDiskCache: tt.disableDiskCache, + StateDir: t.TempDir(), + NegativeTTL: time.Second, + MaxBytes: tt.maxBytes, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) @@ -266,7 +268,8 @@ func negativeCacheRowCount(tb testing.TB, cache *Cache, sourcePath string) int { func TestCleanExpiredDeletesInBatches(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, + cache, err := NewCache(evictionTestDB(t), + CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) @@ -321,7 +324,8 @@ func TestCleanExpiredDeletesInBatches(t *testing.T) { func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, + cache, err := NewCache(evictionTestDB(t), + CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) @@ -352,7 +356,8 @@ func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) { func TestCleanExpiredStopsOnceContextEnds(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{StateDir: t.TempDir()}, + cache, err := NewCache(evictionTestDB(t), + CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index 7d03c0a..ece9311 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -43,18 +43,20 @@ type Service struct { type ServiceConfig struct { // Cache is the cache instance Cache *Cache - // FetcherConfig configures the upstream fetcher built when Fetcher is - // not set. Its AllowHTTP and MaxResponseSize are used either way. - FetcherConfig *httpfetcher.Config - // Fetcher is an optional custom fetcher (for testing) + // Fetcher fetches source images from upstream hosts Fetcher httpfetcher.Fetcher + // Processor converts and resizes the images + Processor *imageprocessor.ImageProcessor // SigningKey is the HMAC signing key (empty disables signing) SigningKey string // Allowlist is the list of hosts that don't require signatures Allowlist []string - // MaxConcurrentProcessing is the most images processed at once; zero - // uses the image processor's default, half the CPUs - MaxConcurrentProcessing int + // AllowHTTP makes the service fetch sources over http instead of + // https (testing only), so the fetcher must allow http too + AllowHTTP bool + // MaxResponseSize is the largest source accepted, in bytes; a cached + // source larger than this is fetched again + MaxResponseSize int64 // Logger for logging Logger *slog.Logger } @@ -75,32 +77,11 @@ func NewService(cfg *ServiceConfig) (*Service, error) { return nil, errSigningKeyRequired } - // Resolve fetcher config for defaults - fetcherCfg := cfg.FetcherConfig - if fetcherCfg == nil { - fetcherCfg = httpfetcher.DefaultConfig() - } - - // Use custom fetcher if provided, otherwise create HTTP fetcher - var fetcher httpfetcher.Fetcher - if cfg.Fetcher != nil { - fetcher = cfg.Fetcher - } else { - fetcher = httpfetcher.New(fetcherCfg) - } - - signer := signature.New(cfg.SigningKey) - log := cfg.Logger if log == nil { log = slog.Default() } - allowHTTP := false - if cfg.FetcherConfig != nil { - allowHTTP = cfg.FetcherConfig.AllowHTTP - } - // JPEG XL is the default output format, so pixad does not start // without it. err := imageprocessor.CheckJPEGXLSupport() @@ -108,21 +89,15 @@ func NewService(cfg *ServiceConfig) (*Service, error) { return nil, err } - maxResponseSize := fetcherCfg.MaxResponseSize - processor := imageprocessor.New(imageprocessor.Params{ - MaxInputBytes: maxResponseSize, - MaxConcurrentProcessing: cfg.MaxConcurrentProcessing, - }) - return &Service{ cache: cfg.Cache, - fetcher: fetcher, - processor: processor, - signer: signer, + fetcher: cfg.Fetcher, + processor: cfg.Processor, + signer: signature.New(cfg.SigningKey), allowlist: allowlist.New(cfg.Allowlist), log: log, - allowHTTP: allowHTTP, - maxResponseSize: maxResponseSize, + allowHTTP: cfg.AllowHTTP, + maxResponseSize: cfg.MaxResponseSize, }, nil } diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index bd5ad8b..31c2c79 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" ) // Shared test data literals, extracted as constants for goconst. @@ -185,16 +186,22 @@ func SetupTestService( cache, err := NewCache(db, CacheConfig{ StateDir: tmpDir, NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, }, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) } svc, err := NewService(&ServiceConfig{ - Cache: cache, - Fetcher: httpfetcher.NewMock(mockFS), - SigningKey: cfg.signingKey, - Allowlist: cfg.allowlist, + Cache: cache, + Fetcher: httpfetcher.NewMock(mockFS), + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + SigningKey: cfg.signingKey, + Allowlist: cfg.allowlist, + MaxResponseSize: 50 << 20, }) if err != nil { t.Fatalf("failed to create service: %v", err) diff --git a/internal/providers/cache_max_bytes_internal_test.go b/internal/providers/cache_max_bytes_internal_test.go new file mode 100644 index 0000000..8b0fc03 --- /dev/null +++ b/internal/providers/cache_max_bytes_internal_test.go @@ -0,0 +1,315 @@ +package providers + +import ( + "context" + "database/sql" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/logger" +) + +// testSigningKey is a throwaway signing key. +const testSigningKey = "test-signing-key-0123456789abcdef" + +// Static errors returned by the stub free-space probes below. +var ( + errTestStatfsFailed = errors.New("statfs failed") + errTestProbeNotExpected = errors.New("probe must not be called") +) + +// newTestDB opens an in-memory database with pixa's schema, on one +// connection, as pixad's database has. +func newTestDB(t *testing.T) *sql.DB { + t.Helper() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("failed to open test db: %v", err) + } + + db.SetMaxOpenConns(1) + + err = database.ApplyMigrations(context.Background(), db, nil) + if err != nil { + t.Fatalf("failed to apply migrations: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + return db +} + +// TestCacheMaxBytesFromConfig checks the disk cache limit worked out from +// cache_max_bytes: omitted, it is the default from the free space; 0 turns +// the disk cache off; a positive value is the limit, unchanged. +func TestCacheMaxBytesFromConfig(t *testing.T) { + t.Parallel() + + const oneGiB = 1 << 30 + + cases := []struct { + name string + cacheMaxBytes int64 + cacheMaxBytesExplicit bool + wantMaxBytes int64 + }{ + { + name: "cache_max_bytes omitted", + cacheMaxBytes: 0, + cacheMaxBytesExplicit: false, + // 75% of the 4 GiB the probe reports free + wantMaxBytes: 3221225472, + }, + { + name: "cache_max_bytes: 0", + cacheMaxBytes: 0, + cacheMaxBytesExplicit: true, + wantMaxBytes: 0, + }, + { + name: "cache_max_bytes: 1 GiB", + cacheMaxBytes: oneGiB, + cacheMaxBytesExplicit: true, + wantMaxBytes: oneGiB, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cfg := &config.Config{ + StateDir: t.TempDir(), + CacheMaxBytes: tc.cacheMaxBytes, + CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit, + } + + got, err := cacheMaxBytes(t.Context(), cfg, newTestDB(t), + slog.New(slog.DiscardHandler), + func(string) (uint64, error) { return 4294967296, nil }) + if err != nil { + t.Fatalf("cacheMaxBytes() error = %v", err) + } + + t.Logf("MaxBytes = %d", got) + + if got != tc.wantMaxBytes { + t.Errorf("MaxBytes = %d, want %d", got, tc.wantMaxBytes) + } + }) + } +} + +// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes builds and starts the +// cache once with cache_max_bytes omitted and once with cache_max_bytes: +// 0, and checks by whether the cache directories were created that the +// disk cache is on in the first case and off in the second. +func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + cacheMaxBytesExplicit bool + wantDiskCache bool + }{ + {name: "cache_max_bytes omitted", cacheMaxBytesExplicit: false, wantDiskCache: true}, + {name: "cache_max_bytes: 0", cacheMaxBytesExplicit: true, wantDiskCache: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + CacheMaxBytes: 0, + CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit, + } + + lc := fxtest.NewLifecycle(t) + + log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}}) + if err != nil { + t.Fatalf("logger.New() error = %v", err) + } + + db, err := database.New(lc, database.Params{Logger: log, Config: cfg}) + if err != nil { + t.Fatalf("database.New() error = %v", err) + } + + _, err = newCache(lc, cacheParams{ + Config: cfg, + Database: db, + Logger: log, + MetricsRegistry: prometheus.NewRegistry(), + }) + if err != nil { + t.Fatalf("newCache() error = %v", err) + } + + lc.RequireStart() + t.Cleanup(lc.RequireStop) + + _, err = os.Stat(filepath.Join(stateDir, "cache", "variants")) + gotDiskCache := err == nil + + if gotDiskCache != tc.wantDiskCache { + t.Errorf("cache directories created = %v, want %v", + gotDiskCache, tc.wantDiskCache) + } + }) + } +} + +// TestDefaultCacheMaxBytesCountsWhatTheCacheHolds verifies that the +// default limit is 75% of the free space plus what the cache already +// holds, so a cache filled to its limit keeps that limit across a +// restart instead of shrinking to 75% of the space left free. +func TestDefaultCacheMaxBytesCountsWhatTheCacheHolds(t *testing.T) { + t.Parallel() + + cfg := &config.Config{StateDir: t.TempDir()} + db := newTestDB(t) + log := slog.New(slog.DiscardHandler) + + // Empty cache, 4 GiB free -> 3 GiB default. + got, err := cacheMaxBytes(t.Context(), cfg, db, log, + func(string) (uint64, error) { return 4294967296, nil }) + if err != nil { + t.Fatalf("cacheMaxBytes returned error: %v", err) + } + + t.Logf("default for an empty cache with 4 GiB free: %d", got) + + if got != 3221225472 { + t.Errorf("default for an empty cache = %d, want 3221225472 (75%% of 4 GiB)", + got) + } + + // The cache now holds those 3 GiB, which leaves 1 GiB free. + _, err = db.ExecContext(t.Context(), + `INSERT INTO variant_content (cache_key, size_bytes) VALUES (?, ?)`, + "aabbccdd0001", 3221225472, + ) + if err != nil { + t.Fatalf("failed to insert variant accounting row: %v", err) + } + + got, err = cacheMaxBytes(t.Context(), cfg, db, log, + func(string) (uint64, error) { return 1073741824, nil }) + if err != nil { + t.Fatalf("cacheMaxBytes returned error: %v", err) + } + + t.Logf("default for a cache holding 3 GiB with 1 GiB free: %d", got) + + if got != 3221225472 { + t.Errorf("default for a cache holding 3 GiB with 1 GiB free = %d, "+ + "want 3221225472 (75%% of 1 GiB + 3 GiB)", got) + } +} + +// TestDefaultCacheMaxBytesPropagatesProbeError verifies that a failing +// free-space probe produces an error naming cache_max_bytes, instead of +// a silently wrong default. +func TestDefaultCacheMaxBytesPropagatesProbeError(t *testing.T) { + t.Parallel() + + _, err := cacheMaxBytes(t.Context(), &config.Config{StateDir: t.TempDir()}, + newTestDB(t), slog.New(slog.DiscardHandler), + func(string) (uint64, error) { return 0, errTestStatfsFailed }) + if err == nil { + t.Fatal("probe failure must produce an error, got nil") + } + + t.Logf("got expected error: %v", err) + + if !strings.Contains(err.Error(), "cache_max_bytes") { + t.Errorf("error %q does not name cache_max_bytes", err.Error()) + } +} + +// TestDefaultCacheMaxBytesProbesTheCacheDirectory verifies that with +// cache_max_bytes omitted the limit is the computed default, and that the +// probe is pointed at /cache/, which must be created first so +// statfs measures the right filesystem. +func TestDefaultCacheMaxBytesProbesTheCacheDirectory(t *testing.T) { + t.Parallel() + + stateDir := t.TempDir() + wantCacheDir := filepath.Join(stateDir, "cache") + + var probedPath string + + // 4 GiB free -> 3 GiB default. + probe := func(path string) (uint64, error) { + probedPath = path + + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + t.Errorf("cache directory %q was not created before probing: info=%v err=%v", + path, info, err) + } + + return 4294967296, nil + } + + got, err := cacheMaxBytes(t.Context(), &config.Config{StateDir: stateDir}, + newTestDB(t), slog.New(slog.DiscardHandler), probe) + if err != nil { + t.Fatalf("cacheMaxBytes returned error: %v", err) + } + + if got != 3221225472 { + t.Errorf("MaxBytes = %d, want computed default 3221225472", got) + } + + if probedPath != wantCacheDir { + t.Errorf("free space probed at %q, want cache directory %q", + probedPath, wantCacheDir) + } +} + +// TestExplicitCacheMaxBytesIsKept verifies that an explicit +// cache_max_bytes is kept exactly as given and the free-space probe is +// never consulted. +func TestExplicitCacheMaxBytesIsKept(t *testing.T) { + t.Parallel() + + probe := func(string) (uint64, error) { + t.Error("free-space probe must not be consulted for explicit values") + + return 0, errTestProbeNotExpected + } + + cfg := &config.Config{ + StateDir: t.TempDir(), + CacheMaxBytes: 1024, + CacheMaxBytesExplicit: true, + } + + got, err := cacheMaxBytes(t.Context(), cfg, newTestDB(t), + slog.New(slog.DiscardHandler), probe) + if err != nil { + t.Fatalf("cacheMaxBytes returned error: %v", err) + } + + if got != 1024 { + t.Errorf("MaxBytes = %d, want explicit 1024 (no floor, no recompute)", got) + } +} diff --git a/internal/providers/fetcher_internal_test.go b/internal/providers/fetcher_internal_test.go new file mode 100644 index 0000000..75f2b02 --- /dev/null +++ b/internal/providers/fetcher_internal_test.go @@ -0,0 +1,103 @@ +package providers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/netip" + "path/filepath" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/globals" + "sneak.berlin/go/pixa/internal/handlers" + "sneak.berlin/go/pixa/internal/healthcheck" + "sneak.berlin/go/pixa/internal/logger" +) + +// TestFetcherFromConfigRefusesBlockedNetworks builds the handlers as pixad +// does, in an fx app with the providers of pixad's fx app, and requests an +// image from 192.0.2.10, which is on the allowlist and in blocked_networks. +// The URL check accepts that address; only the dialer that refuses internal +// addresses checks blocked_networks, so the answer is 403 only if the +// fetcher this package builds from the config connects with that dialer. +// Any other dialer would try to connect until the upstream fetch timeout, +// which is short so that the test then fails quickly. +func TestFetcherFromConfigRefusesBlockedNetworks(t *testing.T) { + t.Parallel() + + const host = "192.0.2.10" + + stateDir := t.TempDir() + cfg := &config.Config{ + SigningKey: testSigningKey, + StateDir: stateDir, + DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"), + AllowlistHosts: []string{host}, + BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}, + UpstreamFetchTimeout: 2 * time.Second, + // With no connection slots, the fetch would fail before dialing. + UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost, + UpstreamConnections: config.DefaultUpstreamConnections, + UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize, + MaxConcurrentProcessing: 1, + } + + var h *handlers.Handlers + + app := fxtest.New(t, + fx.Supply(cfg), + fx.Provide(globals.New, logger.New, database.New, healthcheck.New, + handlers.New, + func() prometheus.Registerer { return prometheus.NewRegistry() }), + fx.Provide(Constructors()...), + fx.Populate(&h), + ) + app.RequireStart() + t.Cleanup(app.RequireStop) + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + + target := "/v1/image/" + host + "/images/photo.jpg/50x50.jpeg" + rec := httptest.NewRecorder() + r.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil)) + t.Logf("GET %s: %d %s", target, rec.Code, rec.Body) + + if rec.Code != http.StatusForbidden { + t.Errorf("status = %d, want %d", rec.Code, http.StatusForbidden) + } + + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q, want application/json", ct) + } + + var body struct { + Error string `json:"error"` + Status int `json:"status"` + Timestamp string `json:"timestamp"` + } + + err := json.NewDecoder(rec.Body).Decode(&body) + if err != nil { + t.Fatalf("decoding response body: %v", err) + } + + if body.Error != "forbidden" || body.Status != http.StatusForbidden { + t.Errorf("body error = %q, status = %d; want %q, %d", + body.Error, body.Status, "forbidden", http.StatusForbidden) + } + + _, err = time.Parse(time.RFC3339, body.Timestamp) + if err != nil { + t.Errorf("body timestamp %q is not RFC 3339: %v", body.Timestamp, err) + } +} diff --git a/internal/providers/providers.go b/internal/providers/providers.go new file mode 100644 index 0000000..b098b4e --- /dev/null +++ b/internal/providers/providers.go @@ -0,0 +1,256 @@ +// Package providers builds from the config the parts pixad serves images +// and logins with, and provides them to the fx graph: +// - the image cache, with its default limit +// - the fetcher +// - the processor +// - the image service +// - the session manager +// - the encrypted URL generator +package providers + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "syscall" + + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/fx" + "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/session" +) + +// Constructors returns the constructors of the parts this package builds, +// for fx.Provide. The fx graph must also provide the prometheus.Registerer +// the cache registers its counters on. +func Constructors() []any { + return []any{ + newCache, + newFetcherConfig, + httpfetcher.New, + newProcessor, + newImageService, + newSessionManager, + newEncryptedURLGenerator, + } +} + +// cacheParams defines dependencies for newCache. +type cacheParams struct { + fx.In + + Config *config.Config + Database *database.Database + Logger *logger.Logger + MetricsRegistry prometheus.Registerer +} + +// newCache builds the image cache with the disk cache limit cacheMaxBytes +// returns. Its start hook starts the cache's background goroutines. Its +// stop hook stops them, which writes the pending access times. The +// database is open for both: it opens before the cache is built and +// closes after this stop hook. +func newCache(lc fx.Lifecycle, params cacheParams) (*imgcache.Cache, error) { + cfg := params.Config + db := params.Database.DB() + + maxBytes, err := cacheMaxBytes( + context.Background(), cfg, db, params.Logger.Get(), defaultFreeSpaceProbe) + if err != nil { + return nil, err + } + + cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ + StateDir: cfg.StateDir, + NegativeTTL: imgcache.DefaultNegativeTTL, + MaxBytes: maxBytes, + Logger: params.Logger.Get(), + }, params.MetricsRegistry) + if err != nil { + return nil, err + } + + // The start hook takes no context: the goroutines it starts outlive it, + // and the stop hook stops them. + lc.Append(fx.StartStopHook( + func() { + // Background eviction: startup reconciliation, then periodic + // and write-pressure passes, and the removal of expired + // negative cache rows, which is all it does when the disk + // cache is off. + cache.StartEviction(imgcache.DefaultEvictionInterval) + + // Writes the access times that requests hand over, so that no + // request waits for the database to record them + cache.StartPendingAccessTimeWrites() + }, + func(ctx context.Context) error { + return errors.Join( + cache.StopEviction(ctx), + cache.StopPendingAccessTimeWrites(ctx), + ) + }, + )) + + return cache, nil +} + +// freeSpaceProbeFunc reports the number of free bytes available on the +// filesystem containing path. It is a function type so tests can pass a +// fake probe instead of depending on the host disk. +type freeSpaceProbeFunc func(path string) (uint64, error) + +var errNegativeBlockSize = errors.New("statfs reported negative block size") + +// defaultFreeSpaceProbe reports free filesystem bytes via statfs on the +// given path, as available to unprivileged processes. +func defaultFreeSpaceProbe(path string) (uint64, error) { + var stat syscall.Statfs_t + + err := syscall.Statfs(path, &stat) + if err != nil { + return 0, err + } + + if stat.Bsize < 0 { + return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path) + } + + blockSize := uint64(stat.Bsize) + + return stat.Bavail * blockSize, nil +} + +// cacheMaxBytes returns cache_max_bytes as set. When the key is omitted, +// it returns config.DefaultCacheMaxBytes of the free bytes probe reports +// for /cache/ and the bytes the cache holds, as db records +// them. It creates that directory first, so the probe measures the +// filesystem that holds the cache. +func cacheMaxBytes( + ctx context.Context, + cfg *config.Config, + db *sql.DB, + log *slog.Logger, + probe freeSpaceProbeFunc, +) (int64, error) { + if cfg.CacheMaxBytesExplicit { + return cfg.CacheMaxBytes, nil + } + + cacheDir := filepath.Join(cfg.StateDir, "cache") + + err := os.MkdirAll(cacheDir, imgcache.StorageDirPerm) + if err != nil { + return 0, fmt.Errorf("failed to create cache directory: %w", err) + } + + freeBytes, err := probe(cacheDir) + if err != nil { + return 0, fmt.Errorf( + "default cache_max_bytes: cannot determine free space for %q: %w", + cacheDir, err) + } + + heldBytes, err := imgcache.UsageBytes(ctx, db) + if err != nil { + return 0, err + } + + limit := config.DefaultCacheMaxBytes(freeBytes, heldBytes) + + log.Info("computed default cache size limit from free space and cache contents", + "cache_max_bytes", limit, + "cache_dir", cacheDir, + ) + + return limit, nil +} + +// newFetcherConfig builds the upstream fetcher's settings from cfg, and +// httpfetcher.New builds the fetcher from them. The settings are a value +// of their own in the fx graph, so that a test can replace the dial +// function alone. +func newFetcherConfig(cfg *config.Config) *httpfetcher.Config { + return &httpfetcher.Config{ + Timeout: cfg.UpstreamFetchTimeout, + MaxResponseSize: cfg.UpstreamMaxResponseSize, + AllowHTTP: cfg.AllowHTTP, + MaxConnectionsPerHost: cfg.UpstreamConnectionsPerHost, + MaxConnections: cfg.UpstreamConnections, + BlockedNetworks: cfg.BlockedNetworks, + } +} + +// newProcessor builds the image processor from cfg. Its input limit is +// the largest upstream response the fetcher accepts. +func newProcessor(cfg *config.Config) *imageprocessor.ImageProcessor { + return imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: cfg.UpstreamMaxResponseSize, + MaxConcurrentProcessing: cfg.MaxConcurrentProcessing, + }) +} + +// imageServiceParams defines dependencies for newImageService. +type imageServiceParams struct { + fx.In + + Config *config.Config + Cache *imgcache.Cache + Fetcher *httpfetcher.HTTPFetcher + Processor *imageprocessor.ImageProcessor + Logger *logger.Logger +} + +// newImageService builds the image service from the cache, the fetcher +// and the processor, with the signing key, the allowlist and the upstream +// settings of the config. +func newImageService( + _ fx.Lifecycle, params imageServiceParams, +) (*imgcache.Service, error) { + cfg := params.Config + + return imgcache.NewService(&imgcache.ServiceConfig{ + Cache: params.Cache, + Fetcher: params.Fetcher, + Processor: params.Processor, + SigningKey: cfg.SigningKey, + Allowlist: cfg.AllowlistHosts, + AllowHTTP: cfg.AllowHTTP, + MaxResponseSize: cfg.UpstreamMaxResponseSize, + Logger: params.Logger.Get(), + }) +} + +// sessionManagerParams defines dependencies for newSessionManager. +type sessionManagerParams struct { + fx.In + + Config *config.Config + Database *database.Database +} + +// newSessionManager builds the session manager. Session cookies are +// always Secure, HttpOnly and SameSite=Strict. The random value that each +// logout replaces is saved in the database, so it outlasts a restart. +func newSessionManager( + _ fx.Lifecycle, params sessionManagerParams, +) (*session.Manager, error) { + return session.NewManagerWithDatabase( + context.Background(), params.Config.SigningKey, params.Database.DB()) +} + +// newEncryptedURLGenerator builds the encrypted URL generator, with a key +// derived from the signing key. +func newEncryptedURLGenerator(cfg *config.Config) (*encurl.Generator, error) { + return encurl.NewGenerator(cfg.SigningKey) +} diff --git a/internal/server/image_proxy_internal_test.go b/internal/server/image_proxy_internal_test.go index ace59d5..86d598e 100644 --- a/internal/server/image_proxy_internal_test.go +++ b/internal/server/image_proxy_internal_test.go @@ -32,6 +32,7 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/middleware" + "sneak.berlin/go/pixa/internal/providers" ) // upstreamHost is the upstream host of the image URLs below. It is a @@ -138,22 +139,21 @@ func startImageProxy( AllowHTTP: true, // A limit of its own, so the cache does not size itself from the // host's free disk space. - CacheMaxBytes: 64 << 20, - CacheMaxBytesExplicit: true, - UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize, - DownstreamTimeout: config.DefaultDownstreamTimeout, + CacheMaxBytes: 64 << 20, + CacheMaxBytesExplicit: true, + UpstreamFetchTimeout: config.DefaultUpstreamFetchTimeout, + UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize, + UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost, + UpstreamConnections: config.DefaultUpstreamConnections, + MaxConcurrentProcessing: 2, + DownstreamTimeout: config.DefaultDownstreamTimeout, } - fetcherCfg := httpfetcher.DefaultConfig() - fetcherCfg.AllowHTTP = true - fetcherCfg.DialContext = func( - ctx context.Context, network, _ string, - ) (net.Conn, error) { + dialUpstream := func(ctx context.Context, network, _ string) (net.Conn, error) { var dialer net.Dialer return dialer.DialContext(ctx, network, upstream.Listener.Addr().String()) } - fetcher := httpfetcher.New(fetcherCfg) var ( h *handlers.Handlers @@ -170,9 +170,14 @@ func startImageProxy( healthcheck.New, handlers.New, middleware.New, - func() httpfetcher.Fetcher { return fetcher }, func() prometheus.Registerer { return prometheus.NewRegistry() }, ), + fx.Provide(providers.Constructors()...), + fx.Decorate(func(fetcherConfig *httpfetcher.Config) *httpfetcher.Config { + fetcherConfig.DialContext = dialUpstream + + return fetcherConfig + }), fx.Populate(&h, &mw, &db), ) app.RequireStart() @@ -188,10 +193,10 @@ func startImageProxy( // startImageProxyWithBlockedNetworks starts the components pixad's fx app // builds, from a config with a fresh state directory and blockedNetworks as -// blocked_networks. Unlike startImageProxy it provides no upstream fetcher, so -// the handlers build the one pixad uses, which checks every address it -// connects to against the built-in blocked ranges and blocked_networks. It -// returns the server with its routes. +// blocked_networks. Unlike startImageProxy it keeps pixad's dial function, +// which checks every address the fetcher connects to against the built-in +// blocked ranges and blocked_networks. It returns the server with its +// routes. func startImageProxyWithBlockedNetworks( t *testing.T, blockedNetworks []netip.Prefix, ) *Server { @@ -207,8 +212,9 @@ func startImageProxyWithBlockedNetworks( // fails within this time at the latest. UpstreamFetchTimeout: 2 * time.Second, // With no connection slots, the fetch would fail before connecting. - UpstreamConnections: config.DefaultUpstreamConnections, - DownstreamTimeout: config.DefaultDownstreamTimeout, + UpstreamConnectionsPerHost: config.DefaultUpstreamConnectionsPerHost, + UpstreamConnections: config.DefaultUpstreamConnections, + DownstreamTimeout: config.DefaultDownstreamTimeout, } var ( @@ -227,6 +233,7 @@ func startImageProxyWithBlockedNetworks( middleware.New, func() prometheus.Registerer { return prometheus.NewRegistry() }, ), + fx.Provide(providers.Constructors()...), fx.Populate(&h, &mw), ) app.RequireStart() diff --git a/internal/server/login_rate_limit_internal_test.go b/internal/server/login_rate_limit_internal_test.go index e55fb79..ebae939 100644 --- a/internal/server/login_rate_limit_internal_test.go +++ b/internal/server/login_rate_limit_internal_test.go @@ -12,16 +12,7 @@ import ( "strings" "testing" - "github.com/prometheus/client_golang/prometheus" - "go.uber.org/fx/fxtest" - "sneak.berlin/go/pixa/internal/config" - "sneak.berlin/go/pixa/internal/database" - "sneak.berlin/go/pixa/internal/globals" - "sneak.berlin/go/pixa/internal/handlers" - "sneak.berlin/go/pixa/internal/healthcheck" - "sneak.berlin/go/pixa/internal/logger" - "sneak.berlin/go/pixa/internal/middleware" ) // testSigningKey is a throwaway signing key; submitting it logs in. @@ -61,45 +52,7 @@ func newTestServer(t *testing.T) *Server { DownstreamTimeout: config.DefaultDownstreamTimeout, } - lc := fxtest.NewLifecycle(t) - - log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}}) - if err != nil { - t.Fatalf("logger.New() error = %v", err) - } - - db, err := database.New(lc, database.Params{Logger: log, Config: cfg}) - if err != nil { - t.Fatalf("database.New() error = %v", err) - } - - hc, err := healthcheck.New(lc, healthcheck.Params{ - Globals: &globals.Globals{}, Config: cfg, Logger: log, Database: db, - }) - if err != nil { - t.Fatalf("healthcheck.New() error = %v", err) - } - - h, err := handlers.New(lc, handlers.Params{ - Logger: log, Healthcheck: hc, Database: db, Config: cfg, - MetricsRegistry: prometheus.NewRegistry(), - }) - if err != nil { - t.Fatalf("handlers.New() error = %v", err) - } - - mw, err := middleware.New(lc, middleware.Params{Logger: log, Config: cfg}) - if err != nil { - t.Fatalf("middleware.New() error = %v", err) - } - - lc.RequireStart() - t.Cleanup(lc.RequireStop) - - s := &Server{config: cfg, mw: mw, h: h} - s.SetupRoutes() - - return s + return startPixa(t, cfg) } // clientRequest builds a request for / arriving from remoteAddr, carrying diff --git a/internal/server/logout_internal_test.go b/internal/server/logout_internal_test.go index d518743..941268f 100644 --- a/internal/server/logout_internal_test.go +++ b/internal/server/logout_internal_test.go @@ -20,6 +20,7 @@ import ( "sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/middleware" + "sneak.berlin/go/pixa/internal/providers" "sneak.berlin/go/pixa/internal/session" ) @@ -29,7 +30,7 @@ var logoutFormPattern = regexp.MustCompile( `(?s)action="/logout".*?name="gorilla\.csrf\.Token" value="([^"]+)"`) // startPixa starts the components pixad's fx app builds, from cfg, and -// returns the server with its routes, as newTestServer does. +// returns the server with its routes. func startPixa(t *testing.T, cfg *config.Config) *Server { t.Helper() @@ -49,6 +50,7 @@ func startPixa(t *testing.T, cfg *config.Config) *Server { middleware.New, func() prometheus.Registerer { return prometheus.NewRegistry() }, ), + fx.Provide(providers.Constructors()...), fx.Populate(&h, &mw), ) app.RequireStart() From 7b6712538ef418cb369a9d6727f242d3f40840ff Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 13:49:12 +0100 Subject: [PATCH 104/131] Take the signature, the expiry and the URL scheme off the image request (closes #314) The image request now holds only its source and its transform, exactly what the variant key covers. The /v1/image/ handler checks the allowlist and verifies the signature itself with Signer.Verify(req, sig, expires). Both image handlers keep the URL's expiry as a local value for Cache-Control. The image service holds no signer, allowlist or AllowHTTP, and ValidateRequest is gone. The fetcher takes the source and builds its URL with Source.URL from its own AllowHTTP, so no URL string is built and parsed again. chooseAutoFormat returns the format it chooses instead of writing into the request. The signed URL format and every route's answers are unchanged. Model: opus-5-5 --- TODO.md | 21 ++ internal/encurl/encurl.go | 33 +- internal/encurl/encurl_test.go | 35 +- internal/handlers/auth.go | 31 +- internal/handlers/cache_hit_internal_test.go | 14 +- .../handlers/first_bytes_internal_test.go | 19 +- internal/handlers/format_auto.go | 23 +- .../handlers/format_auto_internal_test.go | 9 +- internal/handlers/handlers.go | 8 + internal/handlers/handlers_internal_test.go | 16 +- internal/handlers/image.go | 72 ++-- .../image_cache_control_internal_test.go | 14 +- .../handlers/image_errors_internal_test.go | 48 +-- .../handlers/image_signature_internal_test.go | 47 +-- internal/handlers/imageenc.go | 33 +- internal/handlers/imageenc_internal_test.go | 5 +- .../handlers/imageenc_token_internal_test.go | 9 +- .../handlers/jpegxl_default_internal_test.go | 5 +- .../referer_blocklist_internal_test.go | 16 +- internal/handlers/request_id_internal_test.go | 6 +- .../validate_signature_internal_test.go | 211 ++++++++++++ .../httpfetcher/dial_context_internal_test.go | 4 +- internal/httpfetcher/fetch_internal_test.go | 64 +++- internal/httpfetcher/httpfetcher.go | 65 ++-- .../httpfetcher/httpfetcher_internal_test.go | 47 +-- .../max_connections_internal_test.go | 50 +-- internal/httpfetcher/mock.go | 46 +-- .../httpfetcher/request_id_internal_test.go | 2 +- internal/imagerequest/imagerequest.go | 40 +-- .../imagerequest/sourceurl_internal_test.go | 46 ++- internal/imagerequest/urlparser.go | 13 +- .../concurrent_misses_internal_test.go | 10 +- internal/imgcache/service.go | 65 +--- internal/imgcache/service_internal_test.go | 312 +----------------- internal/imgcache/stats_internal_test.go | 2 +- internal/imgcache/testutil_internal_test.go | 58 +--- internal/providers/providers.go | 10 +- .../server/computed_side_internal_test.go | 3 +- internal/server/fit_outside_internal_test.go | 3 +- internal/server/source_url_internal_test.go | 5 +- internal/server/svg_as_avif_internal_test.go | 3 +- internal/signature/escaping_test.go | 13 +- internal/signature/golden_test.go | 7 +- internal/signature/quality_fit_test.go | 19 +- internal/signature/signature.go | 35 +- internal/signature/signature_test.go | 98 +++--- 46 files changed, 770 insertions(+), 925 deletions(-) create mode 100644 internal/handlers/validate_signature_internal_test.go diff --git a/TODO.md b/TODO.md index 2f56525..f02b7c0 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,27 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 the image request holds only what names the cached object (closes + #314): its source and its transform, which the variant key covers. The signed + URL format, the signing algorithm and every route's answers are unchanged. + - The `/v1/image/` handler checks `allowlist_hosts` and verifies the + signature itself, with `Signer.Verify(req, sig, expires)`. + - Each image handler keeps its URL's expiry as a value of its own, for + `Cache-Control`. `encurl`'s `Generate` takes the expiry and its `Parse` + returns it. + - The fetcher takes the source and builds its URL with `Source.URL`, over + http only when the fetcher's own `AllowHTTP` is set. `validateURL` takes + that URL, so no URL string is built and parsed again. + - `chooseAutoFormat` returns the format it chooses, and the handler puts it + into the request it parsed. + - The image service serves images through `Get`, and has `WaitForProcessing` + for shutdown. + - Removed: + - `Signature`, `Expires`, `AllowHTTP` and `SourceURL` of the image + request + - `Service.ValidateRequest`, and the service's signer, allowlist and + `AllowHTTP` + - `extractHost` of the fetcher and `urlToFSPath` of the mock fetcher - 2026-10-26 pixad's fx graph builds the image pipeline (closes #313). `database.New` now opens the database, so the parts below can read it while they are built; its stop hook still runs after the cache's. The cache's diff --git a/internal/encurl/encurl.go b/internal/encurl/encurl.go index e754ec9..2f06726 100644 --- a/internal/encurl/encurl.go +++ b/internal/encurl/encurl.go @@ -59,12 +59,14 @@ func NewGenerator(signingKey string) (*Generator, error) { return &Generator{key: key}, nil } -// Generate creates an encrypted URL token from the request's source, -// transform and expiry, each as the request holds it; an empty format, a -// quality of 0 or an empty fit mode is left out of the token, and a zero -// Expires makes a token that never expires. The token is CBOR-encoded, +// Generate creates an encrypted URL token from the request's source and +// transform, each as the request holds it, and from expires; an empty format, +// a quality of 0 or an empty fit mode is left out of the token, and a zero +// expires makes a token that never expires. The token is CBOR-encoded, // encrypted with NaCl secretbox, and base64url-encoded. -func (g *Generator) Generate(req *imagerequest.Request) (string, error) { +func (g *Generator) Generate( + req *imagerequest.Request, expires time.Time, +) (string, error) { p := &payload{ SourceHost: req.Source.Host, SourcePath: req.Source.Path, @@ -76,8 +78,8 @@ func (g *Generator) Generate(req *imagerequest.Request) (string, error) { FitMode: req.Transform.FitMode, } - if !req.Expires.IsZero() { - p.ExpiresAt = req.Expires.Unix() + if !expires.IsZero() { + p.ExpiresAt = expires.Unix() } // CBOR encode the payload @@ -91,19 +93,19 @@ func (g *Generator) Generate(req *imagerequest.Request) (string, error) { } // Parse decrypts and validates an encrypted URL token and returns the -// request it holds, with default values for omitted optional fields. A token -// that never expires gives the zero Expires. +// request it holds, with default values for omitted optional fields, and its +// expiry. A token that never expires gives the zero time. // Returns ErrExpired if the token has expired, or other errors for invalid tokens. -func (g *Generator) Parse(token string) (*imagerequest.Request, error) { +func (g *Generator) Parse(token string) (*imagerequest.Request, time.Time, error) { // Decrypt data, err := seal.Decrypt(g.key, token) if err != nil { if errors.Is(err, seal.ErrDecryptionFailed) || errors.Is(err, seal.ErrInvalidPayload) { - return nil, ErrDecryptFailed + return nil, time.Time{}, ErrDecryptFailed } - return nil, err + return nil, time.Time{}, err } // CBOR decode @@ -111,12 +113,12 @@ func (g *Generator) Parse(token string) (*imagerequest.Request, error) { err = cbor.Unmarshal(data, &p) if err != nil { - return nil, ErrInvalidFormat + return nil, time.Time{}, ErrInvalidFormat } // Check expiration (0 = never expires) if p.ExpiresAt != 0 && time.Now().Unix() > p.ExpiresAt { - return nil, ErrExpired + return nil, time.Time{}, ErrExpired } format := p.Format @@ -151,6 +153,5 @@ func (g *Generator) Parse(token string) (*imagerequest.Request, error) { Quality: quality, FitMode: fitMode, }, - Expires: expires, - }, nil + }, expires, nil } diff --git a/internal/encurl/encurl_test.go b/internal/encurl/encurl_test.go index 338f678..9ce3100 100644 --- a/internal/encurl/encurl_test.go +++ b/internal/encurl/encurl_test.go @@ -36,10 +36,10 @@ func TestGenerator_GenerateAndParse(t *testing.T) { Quality: 90, FitMode: imagerequest.FitContain, }, - Expires: time.Now().Add(time.Hour), } + expires := time.Now().Add(time.Hour) - token, err := gen.Generate(req) + token, err := gen.Generate(req, expires) if err != nil { t.Fatalf("Generate() error = %v", err) } @@ -48,7 +48,7 @@ func TestGenerator_GenerateAndParse(t *testing.T) { t.Fatal("Generate() returned empty token") } - parsed, err := gen.Parse(token) + parsed, parsedExpires, err := gen.Parse(token) if err != nil { t.Fatalf("Parse() error = %v", err) } @@ -91,8 +91,8 @@ func TestGenerator_GenerateAndParse(t *testing.T) { parsed.Transform.FitMode, req.Transform.FitMode) } - if parsed.Expires.Unix() != req.Expires.Unix() { - t.Errorf("ExpiresAt = %d, want %d", parsed.Expires.Unix(), req.Expires.Unix()) + if parsedExpires.Unix() != expires.Unix() { + t.Errorf("ExpiresAt = %d, want %d", parsedExpires.Unix(), expires.Unix()) } } @@ -106,15 +106,14 @@ func TestGenerator_Parse_Expired(t *testing.T) { Host: testSourceHost, Path: testSourcePath, }, - Expires: time.Now().Add(-time.Hour), // Already expired } - token, err := gen.Generate(req) + token, err := gen.Generate(req, time.Now().Add(-time.Hour)) // Already expired if err != nil { t.Fatalf("Generate() error = %v", err) } - _, err = gen.Parse(token) + _, _, err = gen.Parse(token) if err == nil { t.Error("Parse() should fail for expired token") } @@ -129,7 +128,7 @@ func TestGenerator_Parse_InvalidToken(t *testing.T) { gen, _ := encurl.NewGenerator("test-signing-key-12345") - _, err := gen.Parse("not-a-valid-token") + _, _, err := gen.Parse("not-a-valid-token") if err == nil { t.Error("Parse() should fail for invalid token") } @@ -145,10 +144,9 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) { Host: testSourceHost, Path: testSourcePath, }, - Expires: time.Now().Add(time.Hour), } - token, _ := gen.Generate(req) + token, _ := gen.Generate(req, time.Now().Add(time.Hour)) // Tamper with the token tampered := []byte(token) @@ -156,7 +154,7 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) { tampered[10] ^= 0x01 } - _, err := gen.Parse(string(tampered)) + _, _, err := gen.Parse(string(tampered)) if err == nil { t.Error("Parse() should fail for tampered token") } @@ -173,12 +171,11 @@ func TestGenerator_Parse_WrongKey(t *testing.T) { Host: testSourceHost, Path: testSourcePath, }, - Expires: time.Now().Add(time.Hour), } - token, _ := gen1.Generate(req) + token, _ := gen1.Generate(req, time.Now().Add(time.Hour)) - _, err := gen2.Parse(token) + _, _, err := gen2.Parse(token) if err == nil { t.Error("Parse() should fail with different signing key") } @@ -195,13 +192,12 @@ func TestGenerator_Parse_Defaults(t *testing.T) { Host: testSourceHost, Path: testSourcePath, }, - Expires: time.Now().Add(time.Hour), - }) + }, time.Now().Add(time.Hour)) if err != nil { t.Fatalf("Generate() error = %v", err) } - req, err := gen.Parse(token) + req, _, err := gen.Parse(token) if err != nil { t.Fatalf("Parse() error = %v", err) } @@ -232,10 +228,9 @@ func TestGenerator_TokenIsURLSafe(t *testing.T) { Host: testSourceHost, Path: testSourcePath, }, - Expires: time.Now().Add(time.Hour), } - token, _ := gen.Generate(req) + token, _ := gen.Generate(req, time.Now().Add(time.Hour)) // Check that token only contains URL-safe characters for _, c := range token { diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index a5a6702..1dda36d 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -153,20 +153,20 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc { return } - // The image URL must fetch the source URL as it was written - req.AllowHTTP = s.config.AllowHTTP - fetchURL := req.SourceURL() + // The image URL must fetch the source URL as it was written. The + // fetcher fetches over http only when allow_http is set. + fetchURL := req.Source.URL(s.config.AllowHTTP) if !fetchedAsWritten(parsed, fetchURL) { s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+ - "written: pixa would fetch "+fetchURL, + "written: pixa would fetch "+fetchURL.String(), r.Form, http.StatusBadRequest) return } // Generate encrypted token - token, err := s.encGen.Generate(req) + token, err := s.encGen.Generate(req, expiresAt) if err != nil { s.log.Error("failed to generate encrypted URL", "error", err) s.renderGeneratorWithForm(w, r, "Failed to generate URL", r.Form, @@ -226,25 +226,23 @@ func parseSourceURL(sourceURL string) (*url.URL, error) { // in RawPath when it is not the decoded path percent-encoded again. // EscapedPath ignores RawPath when it also holds a character net/url encodes, // such as a space, so RawPath is compared itself. -func fetchedAsWritten(source *url.URL, fetchURL string) bool { +func fetchedAsWritten(source, fetchURL *url.URL) bool { writtenPath := source.RawPath if writtenPath == "" { writtenPath = source.EscapedPath() } - fetched, err := url.Parse(fetchURL) - - return err == nil && source.User == nil && !source.ForceQuery && - fetched.EscapedPath() == writtenPath && - fetched.RawQuery == source.RawQuery + return source.User == nil && !source.ForceQuery && + fetchURL.EscapedPath() == writtenPath && + fetchURL.RawQuery == source.RawQuery } // buildGenerateRequest parses the numeric form fields and assembles the -// image request the encrypted URL's token is made from. ttl=0 means never -// expires (Expires stays the zero time). A non-numeric or out-of-range -// width, height, quality or ttl, or an unrecognized fit mode, is a client -// error naming the offending field. The format field is passed through -// unchecked. +// image request the encrypted URL's token is made from, and its expiry. +// ttl=0 means never expires (the expiry is the zero time). A non-numeric or +// out-of-range width, height, quality or ttl, or an unrecognized fit mode, is +// a client error naming the offending field. The format field is passed +// through unchecked. func buildGenerateRequest( parsed *url.URL, form url.Values, ) (*imagerequest.Request, time.Time, int, error) { @@ -294,7 +292,6 @@ func buildGenerateRequest( Quality: quality, FitMode: fitMode, }, - Expires: expiresAt, } return req, expiresAt, ttl, nil diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go index 56d1dda..bbc16e1 100644 --- a/internal/handlers/cache_hit_internal_test.go +++ b/internal/handlers/cache_hit_internal_test.go @@ -19,11 +19,13 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/signature" ) // countingHost is the allowlisted host whose photoPath the server of @@ -137,8 +139,6 @@ func newCountingDatabaseServer( MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: testSigningKey, - Allowlist: []string{countingHost}, MaxResponseSize: 50 << 20, Logger: slog.New(slog.DiscardHandler), }) @@ -159,15 +159,17 @@ func newCountingDatabaseServer( Quality: 85, FitMode: imagerequest.FitCover, }, - }) + }, time.Time{}) if err != nil { tb.Fatalf("encGen.Generate() error = %v", err) } h := &Handlers{ - log: slog.New(slog.DiscardHandler), - imgSvc: svc, - encGen: encGen, + log: slog.New(slog.DiscardHandler), + imgSvc: svc, + encGen: encGen, + signer: signature.New(testSigningKey), + allowlist: allowlist.New([]string{countingHost}), } r := chi.NewRouter() diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go index 4a31a7b..69a3751 100644 --- a/internal/handlers/first_bytes_internal_test.go +++ b/internal/handlers/first_bytes_internal_test.go @@ -13,11 +13,13 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/signature" ) // mismatchedSourcePath is the path of the source that @@ -27,8 +29,9 @@ import ( const mismatchedSourcePath = "/images/source.png" // newRoutesFetchingThrough returns both image routes, and the Handlers behind -// them, whose image service fetches through fetcher and has allowlistedHost on -// its allowlist. The handlers and the image service log to log. +// them, whose image service fetches through fetcher, and which have +// allowlistedHost on their allowlist. The handlers and the image service log +// to log. func newRoutesFetchingThrough( t *testing.T, fetcher httpfetcher.Fetcher, log *slog.Logger, ) (*Handlers, http.Handler) { @@ -50,8 +53,6 @@ func newRoutesFetchingThrough( MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: testSigningKey, - Allowlist: []string{allowlistedHost}, MaxResponseSize: 50 << 20, Logger: log, }) @@ -64,7 +65,13 @@ func newRoutesFetchingThrough( t.Fatalf("encurl.NewGenerator() error = %v", err) } - h := &Handlers{log: log, imgSvc: svc, encGen: encGen} + h := &Handlers{ + log: log, + imgSvc: svc, + encGen: encGen, + signer: signature.New(testSigningKey), + allowlist: allowlist.New([]string{allowlistedHost}), + } r := chi.NewRouter() r.Get("/v1/image/*", h.HandleImage()) @@ -93,7 +100,7 @@ func mismatchedSourceEncURL(t *testing.T, h *Handlers) string { Size: imagerequest.Size{Width: 50, Height: 50}, Format: imagerequest.FormatJPEG, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/format_auto.go b/internal/handlers/format_auto.go index 0dda8f8..96a1b06 100644 --- a/internal/handlers/format_auto.go +++ b/internal/handlers/format_auto.go @@ -18,16 +18,17 @@ var ( "image/jxl, image/avif, image/webp or image/jpeg") ) -// chooseAutoFormat replaces the format auto in req with the format -// formatForAccept chooses from r's Accept header, and adds Vary: Accept to the +// chooseAutoFormat returns the format to serve for the requested format: for +// the format auto, the format formatForAccept chooses from r's Accept header, +// and any other format as it is. For auto it adds Vary: Accept to the // response, which then depends on that header. It answers 400 for an Accept // header that is not valid and 406 for one that allows none of the formats, -// and reports whether req can be served. Any other format is left as it is. +// and reports whether the image can be served. func (s *Handlers) chooseAutoFormat( - w http.ResponseWriter, r *http.Request, req *imagerequest.Request, -) bool { - if req.Transform.Format != imagerequest.FormatAuto { - return true + w http.ResponseWriter, r *http.Request, requested imagerequest.Format, +) (imagerequest.Format, bool) { + if requested != imagerequest.FormatAuto { + return requested, true } w.Header().Add("Vary", "Accept") @@ -36,18 +37,16 @@ func (s *Handlers) chooseAutoFormat( if errors.Is(err, errNotAcceptable) { s.respondError(w, err.Error(), http.StatusNotAcceptable) - return false + return "", false } if err != nil { s.respondError(w, err.Error(), http.StatusBadRequest) - return false + return "", false } - req.Transform.Format = format - - return true + return format, true } // formatForAccept returns the format an auto URL is served in for the Accept diff --git a/internal/handlers/format_auto_internal_test.go b/internal/handlers/format_auto_internal_test.go index c543902..5a99f1e 100644 --- a/internal/handlers/format_auto_internal_test.go +++ b/internal/handlers/format_auto_internal_test.go @@ -75,9 +75,9 @@ func TestFormatForAccept(t *testing.T) { // autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL, // both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in -// the format auto. The first is signed with testSigningKey, the key of the -// image service newSignedHostServer builds, and the second is made with h's -// generator. +// the format auto. The first is signed with testSigningKey, the key the +// handlers that newSignedHostServer builds check signatures with. The second +// is made with h's generator. func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { t.Helper() @@ -101,8 +101,7 @@ func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { Size: imagerequest.Size{Width: 50, Height: 50}, Format: imagerequest.FormatAuto, }, - Expires: time.Now().Add(time.Minute), - }) + }, time.Now().Add(time.Minute)) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 555e9c6..db24c4f 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -16,6 +16,7 @@ import ( "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/session" + "sneak.berlin/go/pixa/internal/signature" ) // Params defines dependencies for Handlers. @@ -41,6 +42,11 @@ type Handlers struct { encGen *encurl.Generator csrfProtect func(http.Handler) http.Handler + // signer checks the signature of a /v1/image/ URL with signing_key. + signer *signature.Signer + // allowlist matches the hosts of allowlist_hosts, which a /v1/image/ + // URL may name without a signature. + allowlist *allowlist.HostAllowList // refererBlocklist matches the hosts of referer_blocklist; its IsAllowed // reports whether a URL's host is on that list. refererBlocklist *allowlist.HostAllowList @@ -61,6 +67,8 @@ func New(_ fx.Lifecycle, params Params) (*Handlers, error) { sessMgr: params.SessionManager, encGen: params.EncryptedURLGenerator, csrfProtect: csrfProtect, + signer: signature.New(params.Config.SigningKey), + allowlist: allowlist.New(params.Config.AllowlistHosts), refererBlocklist: allowlist.New(params.Config.RefererBlocklist), }, nil } diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index e463dda..9568849 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -19,10 +19,13 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/signature" ) // testFixtures contains test data. @@ -62,8 +65,6 @@ func setupTestHandler(t *testing.T) *testFixtures { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: "test-signing-key-must-be-32-chars", - Allowlist: []string{goodHost}, MaxResponseSize: 50 << 20, }) if err != nil { @@ -71,8 +72,10 @@ func setupTestHandler(t *testing.T) *testFixtures { } h := &Handlers{ - imgSvc: svc, - log: slog.Default(), + imgSvc: svc, + log: slog.Default(), + signer: signature.New("test-signing-key-must-be-32-chars"), + allowlist: allowlist.New([]string{goodHost}), } return &testFixtures{ @@ -127,10 +130,9 @@ func newMockFetcher(fs fs.FS) *mockFetcher { } func (f *mockFetcher) Fetch( - _ context.Context, url string, + _ context.Context, source imagerequest.Source, ) (*httpfetcher.FetchResult, error) { - // Remove https:// prefix - path := url[8:] // Remove "https://" + path := source.Host + source.Path data, err := fs.ReadFile(f.fs, path) if err != nil { diff --git a/internal/handlers/image.go b/internal/handlers/image.go index b11b0f5..12665e1 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/go-chi/chi/v5/middleware" @@ -27,13 +28,13 @@ func (s *Handlers) HandleImage() http.HandlerFunc { return } - req, ok := s.parseImageRequest(w, r) + req, sig, expires, ok := s.parseImageRequest(w, r) if !ok { return } // Validate signature if required - err := s.imgSvc.ValidateRequest(req) + err := s.validateSignature(req, sig, expires) if err != nil { s.log.Warn("signature validation failed", "host", req.Source.Host, @@ -46,10 +47,13 @@ func (s *Handlers) HandleImage() http.HandlerFunc { } // The signature covers the format auto, not the format chosen - if !s.chooseAutoFormat(w, r, req) { + format, ok := s.chooseAutoFormat(w, r, req.Transform.Format) + if !ok { return } + req.Transform.Format = format + // Get cache key for logging cacheKey := imagerequest.CacheKey(req) @@ -65,7 +69,7 @@ func (s *Handlers) HandleImage() http.HandlerFunc { defer func() { _ = resp.Content.Close() }() - s.writeImageResponse(w, r, req, resp, cacheKey, startTime) + s.writeImageResponse(w, r, req, expires, resp, cacheKey, startTime) } } @@ -82,11 +86,11 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc { } // parseImageRequest parses the image URL's path and query parameters into -// an image request. On invalid input it writes an error response and -// returns false. +// an image request, and returns it with the URL's signature and expiry. On +// invalid input it writes an error response and returns false. func (s *Handlers) parseImageRequest( w http.ResponseWriter, r *http.Request, -) (*imagerequest.Request, bool) { +) (*imagerequest.Request, string, time.Time, bool) { // The path is parsed decoded once, as r.URL.Path holds it, and not from // chi's wildcard: chi routes on the path as the client wrote it whenever // that is not Go's own encoding of it, so its wildcard would still hold @@ -100,7 +104,7 @@ func (s *Handlers) parseImageRequest( ) s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } // Parse signature params from query string. r.URL.Query() would silently @@ -113,7 +117,7 @@ func (s *Handlers) parseImageRequest( s.respondError(w, fmt.Sprintf("invalid query string %q: %v", r.URL.RawQuery, err), http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } for name, values := range query { @@ -121,17 +125,15 @@ func (s *Handlers) parseImageRequest( s.respondError(w, fmt.Sprintf("invalid %s: given more than once", name), http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } } - req.Signature = query.Get("sig") - - req.Expires, err = parseExpires(query) + expires, err := parseExpires(query) if err != nil { s.respondError(w, err.Error(), http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } // Parse optional quality and fit params. Only a q missing from the URL is @@ -143,7 +145,7 @@ func (s *Handlers) parseImageRequest( s.respondError(w, `invalid q: not a number, got ""`, http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } req.Transform.Quality, err = parseFormInt(query, "q", @@ -152,7 +154,7 @@ func (s *Handlers) parseImageRequest( s.respondError(w, fmt.Sprintf("%v, got %q", err, qStr), http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } // Only a fit missing from the URL is cover. A fit in the URL that is not a @@ -162,7 +164,7 @@ func (s *Handlers) parseImageRequest( if query.Has("fit") && req.Transform.FitMode == "" { s.respondError(w, `invalid fit: not a fit mode, got ""`, http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } // Default fit mode if not set @@ -179,10 +181,35 @@ func (s *Handlers) parseImageRequest( s.respondError(w, "invalid image request: "+err.Error(), http.StatusBadRequest) - return nil, false + return nil, "", time.Time{}, false } - return req, true + return req, query.Get("sig"), expires, true +} + +// validateSignature returns nil when the source host of req is on +// allowlist_hosts and names no port. Otherwise it verifies sig, the URL's +// signature of req, and expires, the URL's expiry. +func (s *Handlers) validateSignature( + req *imagerequest.Request, sig string, expires time.Time, +) error { + // An allowlist entry names no port, so a host followed by ":", with or + // without a port, needs a signature. Go's URL type reads the port as the + // fetcher does: in an IPv6 address written without brackets, such as + // 2001:db8::1:8443, it is what follows the last ":". This is checked here + // and not in allowlist.IsAllowed, which also matches referer_blocklist, + // where a Referer must be blocked whatever its port. Port and IsAllowed + // read only the host of a URL. + sourceURL := &url.URL{Host: req.Source.Host} + hostHasPort := sourceURL.Port() != "" || + strings.HasSuffix(sourceURL.Host, ":") + + if !hostHasPort && s.allowlist.IsAllowed(sourceURL) { + return nil + } + + // Signature required for non-allowed hosts + return s.signer.Verify(req, sig, expires) } // parseExpires reads the exp query parameter, a Unix time in seconds. An exp @@ -341,10 +368,11 @@ func notModified(w http.ResponseWriter, r *http.Request, etag string) bool { } // writeImageResponse writes headers and streams the image content, -// handling conditional and HEAD requests. +// handling conditional and HEAD requests. expires is the URL's expiry, which +// ends the Cache-Control max-age. func (s *Handlers) writeImageResponse( w http.ResponseWriter, r *http.Request, - req *imagerequest.Request, resp *imgcache.ImageResponse, + req *imagerequest.Request, expires time.Time, resp *imgcache.ImageResponse, cacheKey imagerequest.VariantKey, startTime time.Time, ) { // Set response headers @@ -355,7 +383,7 @@ func (s *Handlers) writeImageResponse( } // Cache control headers - w.Header().Set("Cache-Control", cacheControl(req.Expires)) + w.Header().Set("Cache-Control", cacheControl(expires)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) if notModified(w, r, resp.ETag) { diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index b27465d..7059ac7 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -14,10 +14,12 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/signature" ) // photoPath is the path of the JPEG that newSignedHostServer serves. @@ -52,7 +54,6 @@ func newSignedHostServer( MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: testSigningKey, MaxResponseSize: 50 << 20, Logger: log, }) @@ -66,9 +67,11 @@ func newSignedHostServer( } h := &Handlers{ - log: log, - imgSvc: svc, - encGen: encGen, + log: log, + imgSvc: svc, + encGen: encGen, + signer: signature.New(testSigningKey), + allowlist: allowlist.New(nil), } r := chi.NewRouter() @@ -202,8 +205,7 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { Size: imagerequest.Size{Width: 50, Height: 50}, Format: imagerequest.FormatJPEG, }, - Expires: tt.expires, - }) + }, tt.expires) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index e4143ef..0591044 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" @@ -26,7 +27,7 @@ import ( const allowlistedHost = "allowed.example.com" // newImageRoute returns the image route of a Handlers whose service fetches -// with fetcher and checks signatures with testSigningKey. +// with fetcher, and which checks signatures with testSigningKey. func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { t.Helper() @@ -46,15 +47,18 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: testSigningKey, - Allowlist: []string{allowlistedHost}, MaxResponseSize: 50 << 20, }) if err != nil { t.Fatalf("failed to create service: %v", err) } - h := &Handlers{imgSvc: svc, log: slog.New(slog.DiscardHandler)} + h := &Handlers{ + imgSvc: svc, + log: slog.New(slog.DiscardHandler), + signer: signature.New(testSigningKey), + allowlist: allowlist.New([]string{allowlistedHost}), + } r := chi.NewRouter() r.Get("/v1/image/*", h.HandleImage()) @@ -63,7 +67,7 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { } // newPhotoFetcher returns a mock fetcher that serves a JPEG at photoPath on -// each of hosts, and answers any other URL with an upstream error. +// each of hosts, and answers any other source with an upstream error. func newPhotoFetcher(t *testing.T, hosts ...string) *httpfetcher.MockFetcher { t.Helper() @@ -90,10 +94,10 @@ func photoURLWithSig(host, sig string, expires time.Time) string { return fmt.Sprintf("%s?sig=%s&exp=%d", photoURL(host), sig, expires.Unix()) } -// photoSignature returns the signature of photoURL(host) at the default -// quality and fit, made with key and expiring at expires. -func photoSignature(key, host string, expires time.Time) string { - return signature.New(key).Sign(&imagerequest.Request{ +// photoRequest returns the image request of photoURL(host) at the default +// quality and fit. +func photoRequest(host string) *imagerequest.Request { + return &imagerequest.Request{ Source: imagerequest.Source{ Host: host, Path: photoPath, @@ -104,8 +108,13 @@ func photoSignature(key, host string, expires time.Time) string { Quality: 85, FitMode: imagerequest.FitCover, }, - Expires: expires, - }) + } +} + +// photoSignature returns the signature of photoURL(host) at the default +// quality and fit, made with key and expiring at expires. +func photoSignature(key, host string, expires time.Time) string { + return signature.New(key).Sign(photoRequest(host), expires) } // sendGet sends a GET for target to route and returns the response. @@ -286,10 +295,9 @@ func TestHandleImage_AllowlistOrSignature(t *testing.T) { // host can be, and checks that each is refused with 400 naming the host as // decoded: allowlistedHost with a character after it that a URL host cannot // hold, percent-encoded, an IPv6 address without brackets, and brackets -// around something that is not an IPv6 address. A "?" or "#" after -// allowlistedHost would otherwise end the host when the source URL is read, -// so the request would pass the allowlist and be fetched and cached under a -// host that is not the one fetched. +// around something that is not an IPv6 address. Later steps put the host +// unparsed into the allowlist check, the cache key and the upstream URL, so +// such a host is refused first. func TestHandleImage_InvalidHost_Returns400(t *testing.T) { t.Parallel() @@ -360,10 +368,9 @@ func TestHandleImage_HostForms_Served(t *testing.T) { // TestHandleImage_HashInSourcePathOrQuery_Returns400 sends image URLs whose // source path or source query holds a "#", written as %23, and checks that -// each is refused with 400, the one signed for that source path included. The -// source URL would otherwise end at the "#" when it is fetched, so each text -// after it would fetch the image at photoPath and cache it under keys of its -// own. +// each is refused with 400, the one signed for that source path included. +// A "#" starts a URL's fragment, so neither the source path nor the source +// query may hold one. func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) { t.Parallel() @@ -381,8 +388,7 @@ func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) { Quality: 85, FitMode: imagerequest.FitCover, }, - Expires: exp, - }) + }, exp) tests := []struct { name string diff --git a/internal/handlers/image_signature_internal_test.go b/internal/handlers/image_signature_internal_test.go index 3d2b766..eaed6ee 100644 --- a/internal/handlers/image_signature_internal_test.go +++ b/internal/handlers/image_signature_internal_test.go @@ -38,40 +38,44 @@ func getImage(t *testing.T, fix *testFixtures, target string) int { } // generateSignedURL returns the /v1/image/ URL of req, signed with key and -// expiring after ttl, and sets req's expiry. An unset quality or fit of req is -// first set to 85 or cover, the values the route uses when a URL has no q or -// fit. The URL carries q and fit next to sig and exp, so the route verifies it -// for the quality and fit it was signed with. +// expiring after ttl. It signs a copy of req, so req itself is not changed. +// An unset quality or fit in the copy is set to 85 or cover, the values the +// route uses when a URL has no q or fit. The URL carries q and fit next to sig +// and exp, so the route verifies it for the quality and fit it was signed +// with. func generateSignedURL( key string, req *imagerequest.Request, ttl time.Duration, ) string { - if req.Transform.Quality == 0 { - req.Transform.Quality = 85 + signed := *req + + if signed.Transform.Quality == 0 { + signed.Transform.Quality = 85 } - if req.Transform.FitMode == "" { - req.Transform.FitMode = imagerequest.FitCover + if signed.Transform.FitMode == "" { + signed.Transform.FitMode = imagerequest.FitCover } - req.Expires = time.Now().Add(ttl) - sig := signature.New(key).Sign(req) + expires := time.Now().Add(ttl) + sig := signature.New(key).Sign(&signed, expires) // A source query follows the source path after a "?". The image route // reads the path decoded once, so the whole path is written in Go's own // percent-encoding of a URL path, which writes "?" as %3F and "%" as %25, // and leaves "/", ";", ",", "=" and "&" as they are. - sourcePathAndQuery := req.Source.Path - if req.Source.Query != "" { - sourcePathAndQuery += "?" + req.Source.Query + sourcePathAndQuery := signed.Source.Path + if signed.Source.Query != "" { + sourcePathAndQuery += "?" + signed.Source.Query } unescapedPath := fmt.Sprintf("/v1/image/%s%s/%dx%d.%s", - req.Source.Host, sourcePathAndQuery, - req.Transform.Size.Width, req.Transform.Size.Height, req.Transform.Format) + signed.Source.Host, sourcePathAndQuery, + signed.Transform.Size.Width, signed.Transform.Size.Height, + signed.Transform.Format) return fmt.Sprintf("%s?sig=%s&exp=%d&q=%d&fit=%s", - (&url.URL{Path: unescapedPath}).EscapedPath(), sig, req.Expires.Unix(), - req.Transform.Quality, req.Transform.FitMode) + (&url.URL{Path: unescapedPath}).EscapedPath(), sig, expires.Unix(), + signed.Transform.Quality, signed.Transform.FitMode) } // TestHandleImage_SignatureCoversQualityAndFit signs a URL for quality 85 @@ -93,8 +97,7 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) { Quality: 85, FitMode: imagerequest.FitCover, }, - Expires: expires, - }) + }, expires) signedURL := fmt.Sprintf("/v1/image/%s/images/photo.jpg/50x50.jpeg?sig=%s&exp=%d", signedHost, sig, expires.Unix()) @@ -277,8 +280,7 @@ func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) { Quality: 85, FitMode: imagerequest.FitCover, }, - Expires: expires, - }) + }, expires) tests := []struct { name string @@ -344,8 +346,7 @@ func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) { Quality: 85, FitMode: imagerequest.FitCover, }, - Expires: expires, - }) + }, expires) target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d", signedHost, tt.writtenPathAndQuery, sig, expires.Unix()) diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index e5a94d4..e823498 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -31,11 +31,18 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { ctx := r.Context() start := time.Now() - req, ok := s.parseImageEncRequest(w, r) - if !ok || !s.chooseAutoFormat(w, r, req) { + req, expires, ok := s.parseImageEncRequest(w, r) + if !ok { return } + format, ok := s.chooseAutoFormat(w, r, req.Transform.Format) + if !ok { + return + } + + req.Transform.Format = format + // Log the request s.log.Debug("encrypted image request", "host", req.Source.Host, @@ -64,7 +71,7 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { } // Cache headers: max-age ends at the URL's expiry - w.Header().Set("Cache-Control", cacheControl(req.Expires)) + w.Header().Set("Cache-Control", cacheControl(expires)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) if notModified(w, r, resp.ETag) { @@ -102,34 +109,34 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { } // parseImageEncRequest decrypts the token of an encrypted image URL into an -// image request and checks it. On a token that is missing, does not decrypt, -// has expired or asks for something not valid, it writes an error response -// and returns false. +// image request and checks it, and returns it with the token's expiry. On a +// token that is missing, does not decrypt, has expired or asks for something +// not valid, it writes an error response and returns false. func (s *Handlers) parseImageEncRequest( w http.ResponseWriter, r *http.Request, -) (*imagerequest.Request, bool) { +) (*imagerequest.Request, time.Time, bool) { // Extract token from URL token := chi.URLParam(r, "token") if token == "" { s.respondError(w, "missing token", http.StatusBadRequest) - return nil, false + return nil, time.Time{}, false } // Decrypt and validate the payload - req, err := s.encGen.Parse(token) + req, expires, err := s.encGen.Parse(token) if err != nil { if errors.Is(err, encurl.ErrExpired) { s.log.Debug("encrypted URL expired", "error", err) s.respondError(w, "URL has expired", http.StatusGone) - return nil, false + return nil, time.Time{}, false } s.log.Debug("failed to decrypt URL", "error", err) s.respondError(w, "invalid encrypted URL", http.StatusBadRequest) - return nil, false + return nil, time.Time{}, false } // Apply the same dimension and fit-mode bounds as the plain image @@ -142,10 +149,10 @@ func (s *Handlers) parseImageEncRequest( s.respondError(w, "invalid encrypted URL: "+err.Error(), http.StatusBadRequest) - return nil, false + return nil, time.Time{}, false } - return req, true + return req, expires, true } // handleImageError converts image service errors to HTTP responses. diff --git a/internal/handlers/imageenc_internal_test.go b/internal/handlers/imageenc_internal_test.go index ec034c5..e939e13 100644 --- a/internal/handlers/imageenc_internal_test.go +++ b/internal/handlers/imageenc_internal_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/go-chi/chi/v5" @@ -62,7 +63,7 @@ func TestHandleImageEnc_OverLimitDimension_Returns400(t *testing.T) { Transform: imagerequest.Transform{ Size: imagerequest.Size{Width: 100000, Height: 100000}, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } @@ -91,7 +92,7 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) { Size: imagerequest.Size{Width: 800, Height: 600}, FitMode: imagerequest.Fit("bogus"), }, - }) + }, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/imageenc_token_internal_test.go b/internal/handlers/imageenc_token_internal_test.go index f647a11..920cc12 100644 --- a/internal/handlers/imageenc_token_internal_test.go +++ b/internal/handlers/imageenc_token_internal_test.go @@ -70,15 +70,12 @@ func TestHandleImageEnc_RejectedToken(t *testing.T) { }, } - valid, err := h.encGen.Generate(&photo) + valid, err := h.encGen.Generate(&photo, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } - expiredPhoto := photo - expiredPhoto.Expires = time.Now().Add(-time.Minute) - - expired, err := h.encGen.Generate(&expiredPhoto) + expired, err := h.encGen.Generate(&photo, time.Now().Add(-time.Minute)) if err != nil { t.Fatalf("Generate() error = %v", err) } @@ -88,7 +85,7 @@ func TestHandleImageEnc_RejectedToken(t *testing.T) { t.Fatalf("encurl.NewGenerator() error = %v", err) } - otherKey, err := otherGen.Generate(&photo) + otherKey, err := otherGen.Generate(&photo, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/jpegxl_default_internal_test.go b/internal/handlers/jpegxl_default_internal_test.go index 219c0f0..9a5502e 100644 --- a/internal/handlers/jpegxl_default_internal_test.go +++ b/internal/handlers/jpegxl_default_internal_test.go @@ -69,8 +69,7 @@ func TestImageWithoutFormat_SignedAsJXL(t *testing.T) { Quality: encurl.DefaultQuality, FitMode: imagerequest.FitCover, }, - Expires: expires, - }) + }, expires) query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix()) for _, size := range []string{"50x50.jxl", "50x50"} { @@ -94,7 +93,7 @@ func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) { Transform: imagerequest.Transform{ Size: imagerequest.Size{Width: 50, Height: 50}, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index f3160a5..a81a7f5 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -15,7 +15,9 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/signature" ) // blockedReferer is a page on leech.example, which newRefererRoutes puts on @@ -31,11 +33,11 @@ type countingFetcher struct { // Fetch counts the fetch and passes it on. func (f *countingFetcher) Fetch( - ctx context.Context, url string, + ctx context.Context, source imagerequest.Source, ) (*httpfetcher.FetchResult, error) { f.fetches.Add(1) - return f.Fetcher.Fetch(ctx, url) + return f.Fetcher.Fetch(ctx, source) } // newRefererRoutes returns both image routes of a Handlers whose @@ -65,8 +67,6 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: testSigningKey, - Allowlist: []string{allowlistedHost}, MaxResponseSize: 50 << 20, }) if err != nil { @@ -79,9 +79,11 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) } h := &Handlers{ - log: slog.New(slog.DiscardHandler), - imgSvc: svc, - encGen: encGen, + log: slog.New(slog.DiscardHandler), + imgSvc: svc, + encGen: encGen, + signer: signature.New(testSigningKey), + allowlist: allowlist.New([]string{allowlistedHost}), refererBlocklist: allowlist.New( []string{"leech.example", ".hotlinker.example"}), } diff --git a/internal/handlers/request_id_internal_test.go b/internal/handlers/request_id_internal_test.go index 110d45c..72da7e6 100644 --- a/internal/handlers/request_id_internal_test.go +++ b/internal/handlers/request_id_internal_test.go @@ -18,8 +18,8 @@ import ( // signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the // JPEG at photoPath on signedHost at 50x50. It is signed with testSigningKey, -// the key of the image service newSignedHostServer builds. It takes a Handlers -// it does not use, so that it has the type of encPhotoURL. +// the key the handlers that newSignedHostServer builds check signatures with. +// It takes a Handlers it does not use, so that it has the type of encPhotoURL. func signedPhotoURL(t *testing.T, _ *Handlers) string { t.Helper() @@ -49,7 +49,7 @@ func encPhotoURL(t *testing.T, h *Handlers) string { Size: imagerequest.Size{Width: 50, Height: 50}, Format: imagerequest.FormatJPEG, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) } diff --git a/internal/handlers/validate_signature_internal_test.go b/internal/handlers/validate_signature_internal_test.go new file mode 100644 index 0000000..0b13996 --- /dev/null +++ b/internal/handlers/validate_signature_internal_test.go @@ -0,0 +1,211 @@ +package handlers + +import ( + "testing" + "time" + + "sneak.berlin/go/pixa/internal/allowlist" + "sneak.berlin/go/pixa/internal/signature" +) + +// Addresses the port tests below put on the allowlist. +const ( + allowlistedIPv4 = "203.0.113.5" + allowlistedIPv6 = "2001:db8::1" +) + +// newSignatureChecker returns Handlers that check signatures with +// testSigningKey and have allowlistHosts on their allowlist, as New builds +// them from signing_key and allowlist_hosts. +func newSignatureChecker(allowlistHosts ...string) *Handlers { + return &Handlers{ + signer: signature.New(testSigningKey), + allowlist: allowlist.New(allowlistHosts), + } +} + +func TestValidateSignature_NonAllowlistedHost_NoSignature(t *testing.T) { + t.Parallel() + + h := newSignatureChecker(allowlistedHost) + + // Should fail validation - not allowlisted and no signature + err := h.validateSignature(photoRequest(signedHost), "", time.Time{}) + if err == nil { + t.Error("validateSignature() expected error for non-allowlisted " + + "host without signature") + } +} + +func TestValidateSignature_NonAllowlistedHost_ValidSignature(t *testing.T) { + t.Parallel() + + h := newSignatureChecker(allowlistedHost) + req := photoRequest(signedHost) + + // Generate a valid signature + expires := time.Now().Add(time.Hour) + sig := signature.New(testSigningKey).Sign(req, expires) + + // Should pass validation + err := h.validateSignature(req, sig, expires) + if err != nil { + t.Errorf("validateSignature() error = %v", err) + } +} + +func TestValidateSignature_NonAllowlistedHost_ExpiredSignature(t *testing.T) { + t.Parallel() + + h := newSignatureChecker(allowlistedHost) + req := photoRequest(signedHost) + + // Generate an expired signature + expires := time.Now().Add(-time.Hour) + sig := signature.New(testSigningKey).Sign(req, expires) + + // Should fail validation + err := h.validateSignature(req, sig, expires) + if err == nil { + t.Error("validateSignature() expected error for expired signature") + } +} + +func TestValidateSignature_NonAllowlistedHost_InvalidSignature(t *testing.T) { + t.Parallel() + + h := newSignatureChecker(allowlistedHost) + + // Should fail validation + err := h.validateSignature(photoRequest(signedHost), "invalid-signature", + time.Now().Add(time.Hour)) + if err == nil { + t.Error("validateSignature() expected error for invalid signature") + } +} + +// TestValidateSignature_NoAllowlist checks that with no host on the +// allowlist, a request without a signature fails validation. +func TestValidateSignature_NoAllowlist(t *testing.T) { + t.Parallel() + + h := newSignatureChecker() + + err := h.validateSignature(photoRequest(signedHost), "", time.Time{}) + if err == nil { + t.Error("validateSignature() expected error when no host is allowlisted") + } +} + +// TestValidateSignature_SignatureExactHostMatch verifies that +// validateSignature enforces exact host matching for signatures. A +// signature for one host must not verify for a different host, even +// if they share a domain suffix. +func TestValidateSignature_SignatureExactHostMatch(t *testing.T) { + t.Parallel() + + h := newSignatureChecker() + + // Sign a request for signedHost + signedReq := photoRequest(signedHost) + expires := time.Now().Add(time.Hour) + sig := signature.New(testSigningKey).Sign(signedReq, expires) + + // The original request should pass validation + t.Run("exact host passes", func(t *testing.T) { + t.Parallel() + + err := h.validateSignature(signedReq, sig, expires) + if err != nil { + t.Errorf("validateSignature() exact host failed: %v", err) + } + }) + + // Try to reuse the signature with different hosts + tests := []struct { + name string + host string + }{ + {"parent domain", "example.com"}, + {"sibling subdomain", "images.example.com"}, + {"deeper subdomain", "a." + signedHost}, + {"evil suffix domain", signedHost + ".evil.com"}, + {"prefixed host", "evil" + signedHost}, + } + + for _, tt := range tests { + t.Run(tt.name+" rejected", func(t *testing.T) { + t.Parallel() + + err := h.validateSignature(photoRequest(tt.host), sig, expires) + if err == nil { + t.Errorf("validateSignature() should reject signature for "+ + "host %q (signed for %q)", tt.host, signedHost) + } + }) + } +} + +// TestValidateSignature_HostWithPortNeedsSignature verifies that a host +// followed by a port, or by a colon with no port, matches no allowlist entry, +// so that a request for it without a signature is refused, while the same +// host with no port is allowlisted. +func TestValidateSignature_HostWithPortNeedsSignature(t *testing.T) { + t.Parallel() + + h := newSignatureChecker(allowlistedHost, allowlistedIPv4, allowlistedIPv6) + + tests := []struct { + host string + allowlisted bool + }{ + {allowlistedHost, true}, + {allowlistedHost + ":8443", false}, + {allowlistedHost + ":443", false}, + {allowlistedHost + ":", false}, + {allowlistedIPv4, true}, + {allowlistedIPv4 + ":8080", false}, + {"[" + allowlistedIPv6 + "]", true}, + {"[" + allowlistedIPv6 + "]:8443", false}, + } + + for _, tt := range tests { + t.Run(tt.host, func(t *testing.T) { + t.Parallel() + + err := h.validateSignature(photoRequest(tt.host), "", time.Time{}) + t.Logf("validateSignature() for host %q with no signature: %v", + tt.host, err) + + if allowlisted := err == nil; allowlisted != tt.allowlisted { + t.Errorf("host %q served without a signature = %v, want %v", + tt.host, allowlisted, tt.allowlisted) + } + }) + } +} + +// TestValidateSignature_UnbracketedIPv6PortNeedsSignature verifies that an +// allowlisted IPv6 address written without brackets and followed by a port, +// or by a colon with no port, needs a signature. Go's URL type reads +// 2001:db8::1:8443 as the host 2001:db8::1 with the port 8443. +func TestValidateSignature_UnbracketedIPv6PortNeedsSignature(t *testing.T) { + t.Parallel() + + h := newSignatureChecker(allowlistedIPv6) + + for _, host := range []string{allowlistedIPv6 + ":8443", allowlistedIPv6 + ":"} { + t.Run(host, func(t *testing.T) { + t.Parallel() + + err := h.validateSignature(photoRequest(host), "", time.Time{}) + t.Logf("validateSignature() for host %q with no signature: %v", + host, err) + + if err == nil { + t.Errorf("host %q served without a signature, want refused", + host) + } + }) + } +} diff --git a/internal/httpfetcher/dial_context_internal_test.go b/internal/httpfetcher/dial_context_internal_test.go index 036bd78..b632a1f 100644 --- a/internal/httpfetcher/dial_context_internal_test.go +++ b/internal/httpfetcher/dial_context_internal_test.go @@ -51,12 +51,12 @@ func TestDialContextReplacesOnlyTheDialer(t *testing.T) { t.Errorf("body = %q, want %q", body, imagePayload) } - _, err := f.Fetch(testContext(t), "http://127.0.0.1/image") + _, err := f.Fetch(testContext(t), imageSourceOn("127.0.0.1")) if !errors.Is(err, ErrSSRFBlocked) { t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err) } - _, err = f.Fetch(testContext(t), upstreamURL("/redirect/private")) + _, err = f.Fetch(testContext(t), upstreamSource("/redirect/private")) if !errors.Is(err, ErrSSRFBlocked) { t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err) } diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index f2f3e83..0d515d7 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -14,6 +14,8 @@ import ( "sync" "testing" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // testPublicHost is a TEST-NET-1 (RFC 5737) literal. isPrivateIP treats it as @@ -29,9 +31,9 @@ const imagePayload = "fake-jpeg-bytes" // would mean SSRF protection let a forbidden target reach the transport. var errUnexpectedDial = errors.New("unexpected dial target") -// upstreamURL builds a fetch URL on the fake public host for the given path. -func upstreamURL(path string) string { - return "http://" + testPublicHost + path +// upstreamSource returns the source at path on the fake public host. +func upstreamSource(path string) imagerequest.Source { + return imagerequest.Source{Host: testPublicHost, Path: path} } // recordingDialer records every address the transport asks it to dial and @@ -158,7 +160,7 @@ func testContext(t *testing.T) context.Context { func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult { t.Helper() - res, err := f.Fetch(testContext(t), upstreamURL(path)) + res, err := f.Fetch(testContext(t), upstreamSource(path)) if err != nil { t.Fatalf("Fetch(%s) error = %v", path, err) } @@ -170,7 +172,7 @@ func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult { func fetchExpectError(t *testing.T, f *HTTPFetcher, path string) error { t.Helper() - res, err := f.Fetch(testContext(t), upstreamURL(path)) + res, err := f.Fetch(testContext(t), upstreamSource(path)) if err == nil { _ = res.Content.Close() @@ -222,7 +224,7 @@ func TestFetchRedirectToPrivateIPBlocked(t *testing.T) { srv := startUpstream(t) f, dialer := newServerFetcher(t, srv, nil) - _, err := f.Fetch(testContext(t), upstreamURL("/redirect/private")) + _, err := f.Fetch(testContext(t), upstreamSource("/redirect/private")) if !errors.Is(err, ErrSSRFBlocked) { t.Fatalf("Fetch() error = %v, want ErrSSRFBlocked", err) } @@ -590,3 +592,53 @@ func TestSSRFSafeDialerAllowsPublicTarget(t *testing.T) { t.Errorf("public target was SSRF-blocked: %v", err) } } + +// TestFetchSchemeFollowsAllowHTTP checks that the fetcher fetches a source +// over https, on port 443, unless AllowHTTP is set, and then over http, on +// port 80. The test server speaks plain HTTP, so only the fetch over http +// succeeds. +func TestFetchSchemeFollowsAllowHTTP(t *testing.T) { + t.Parallel() + + tests := []struct { + allowHTTP bool + wantAddr string + }{ + {false, net.JoinHostPort(testPublicHost, "443")}, + {true, net.JoinHostPort(testPublicHost, "80")}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("AllowHTTP %v", tt.allowHTTP), func(t *testing.T) { + t.Parallel() + + srv := startUpstream(t) + dialer := &recordingDialer{target: srv.Listener.Addr().String()} + + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + AllowHTTP: tt.allowHTTP, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + DialContext: dialer.dialContext, + }) + + res, err := f.Fetch(testContext(t), upstreamSource("/image")) + if err == nil { + _ = res.Content.Close() + } + + dialed := dialer.dialedAddrs() + t.Logf("Fetch(): error %v, dialed %v", err, dialed) + + if served := err == nil; served != tt.allowHTTP { + t.Errorf("served = %v, want %v", served, tt.allowHTTP) + } + + if len(dialed) != 1 || dialed[0] != tt.wantAddr { + t.Errorf("dialed %v, want only %s", dialed, tt.wantAddr) + } + }) + } +} diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index c9d2eae..08eeee1 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -20,6 +20,7 @@ import ( "time" "github.com/go-chi/chi/v5/middleware" + "sneak.berlin/go/pixa/internal/imagerequest" ) // Fetcher configuration constants. @@ -91,8 +92,8 @@ var ( // Fetcher retrieves content from upstream origins. type Fetcher interface { - // Fetch retrieves content from the given URL. - Fetch(ctx context.Context, url string) (*FetchResult, error) + // Fetch retrieves the content of the source image. + Fetch(ctx context.Context, source imagerequest.Source) (*FetchResult, error) } // FetchResult contains the result of fetching from upstream. @@ -139,7 +140,8 @@ type Config struct { Timeout time.Duration // MaxResponseSize is the maximum allowed response body size. MaxResponseSize int64 - // AllowHTTP allows non-TLS connections (for testing only). + // AllowHTTP makes the fetcher fetch sources over http instead of https, + // and follow redirects to http (for testing only). AllowHTTP bool // MaxConnectionsPerHost limits concurrent connections to each upstream host. MaxConnectionsPerHost int @@ -211,7 +213,7 @@ func New(config *Config) *HTTPFetcher { } // Validate the redirect target - err := validateURL(req.Context(), req.URL.String(), config.AllowHTTP) + err := validateURL(req.Context(), req.URL, config.AllowHTTP) if err != nil { return fmt.Errorf("redirect blocked: %w", err) } @@ -229,18 +231,22 @@ func New(config *Config) *HTTPFetcher { } } -// Fetch retrieves content from the given URL with SSRF protection. When -// MaxConnections fetches are already in progress, it waits up to -// ConnectionWaitTimeout for one to finish, then fails with -// ErrTooManyConnections. -func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, error) { +// Fetch retrieves the source image with SSRF protection, over https, or over +// http when AllowHTTP is set. When MaxConnections fetches are already in +// progress, it waits up to ConnectionWaitTimeout for one to finish, then +// fails with ErrTooManyConnections. +func (f *HTTPFetcher) Fetch( + ctx context.Context, source imagerequest.Source, +) (*FetchResult, error) { + sourceURL := source.URL(f.config.AllowHTTP) + // Validate URL before making request - err := validateURL(ctx, url, f.config.AllowHTTP) + err := validateURL(ctx, sourceURL, f.config.AllowHTTP) if err != nil { return nil, err } - release, err := f.acquireConnection(ctx, extractHost(url)) + release, err := f.acquireConnection(ctx, source.Host) if err != nil { return nil, err } @@ -254,14 +260,9 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro } }() - parsedURL, err := neturl.Parse(url) - if err != nil { - return nil, fmt.Errorf("failed to parse URL: %w", err) - } - req := &http.Request{ Method: http.MethodGet, - URL: parsedURL, + URL: sourceURL, Header: make(http.Header), } @@ -455,19 +456,14 @@ func (f *HTTPFetcher) isAllowedContentType(contentType string) bool { } // validateURL checks if a URL is safe to fetch (not internal/private). -func validateURL(ctx context.Context, rawURL string, allowHTTP bool) error { - if !allowHTTP && !strings.HasPrefix(rawURL, "https://") { +func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) error { + if !allowHTTP && fetchURL.Scheme != "https" { return ErrUnsupportedScheme } - parsedURL, err := neturl.Parse(rawURL) - if err != nil { - return fmt.Errorf("%w: %w", ErrInvalidHost, err) - } - // Hostname drops any port, and the brackets of an IPv6 address whether // or not a port follows them - host := parsedURL.Hostname() + host := fetchURL.Hostname() if host == "" { return ErrInvalidHost } @@ -493,25 +489,6 @@ func validateURL(ctx context.Context, rawURL string, allowHTTP bool) error { return nil } -// extractHost extracts the host from a URL string. -func extractHost(rawURL string) string { - // Simple extraction without full URL parsing - url := rawURL - if idx := strings.Index(url, "://"); idx != -1 { - url = url[idx+3:] - } - - if idx := strings.Index(url, "/"); idx != -1 { - url = url[:idx] - } - - if idx := strings.Index(url, "?"); idx != -1 { - url = url[:idx] - } - - return url -} - // isLocalhost checks if the host is localhost. func isLocalhost(host string) bool { host = strings.ToLower(host) diff --git a/internal/httpfetcher/httpfetcher_internal_test.go b/internal/httpfetcher/httpfetcher_internal_test.go index 7341d28..8b71419 100644 --- a/internal/httpfetcher/httpfetcher_internal_test.go +++ b/internal/httpfetcher/httpfetcher_internal_test.go @@ -5,9 +5,12 @@ import ( "errors" "io" "net" + "net/url" "testing" "testing/fstest" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // testHost is the hostname used by mock fetch tests. @@ -49,33 +52,6 @@ func TestIsAllowedContentType(t *testing.T) { } } -func TestExtractHost(t *testing.T) { - t.Parallel() - - tests := []struct { - url string - want string - }{ - {"https://example.com/path", testHost}, - {"http://example.com:8080/path", "example.com:8080"}, - {"https://example.com", testHost}, - {"https://example.com?q=1", testHost}, - {"example.com/path", testHost}, - {"", ""}, - } - - for _, tc := range tests { - t.Run(tc.url, func(t *testing.T) { - t.Parallel() - - got := extractHost(tc.url) - if got != tc.want { - t.Errorf("extractHost(%q) = %q, want %q", tc.url, got, tc.want) - } - }) - } -} - func TestIsLocalhost(t *testing.T) { t.Parallel() @@ -188,7 +164,7 @@ func TestIsPrivateIPJudgesEmbeddedIPv4Address(t *testing.T) { func TestValidateURL_RejectsNonHTTPS(t *testing.T) { t.Parallel() - err := validateURL(t.Context(), "http://example.com/path", false) + err := validateURL(t.Context(), &url.URL{Scheme: "http", Host: testHost}, false) if !errors.Is(err, ErrUnsupportedScheme) { t.Errorf("validateURL http = %v, want ErrUnsupportedScheme", err) } @@ -198,7 +174,8 @@ func TestValidateURL_AllowsHTTPWhenConfigured(t *testing.T) { t.Parallel() // Use a host that won't resolve (explicit .invalid TLD) so we don't hit DNS. - err := validateURL(t.Context(), "http://nonexistent.invalid/path", true) + err := validateURL(t.Context(), + &url.URL{Scheme: "http", Host: "nonexistent.invalid"}, true) // We expect a host resolution error, not ErrUnsupportedScheme. if errors.Is(err, ErrUnsupportedScheme) { t.Error("validateURL with AllowHTTP should not return ErrUnsupportedScheme") @@ -208,7 +185,7 @@ func TestValidateURL_AllowsHTTPWhenConfigured(t *testing.T) { func TestValidateURL_RejectsLocalhost(t *testing.T) { t.Parallel() - err := validateURL(t.Context(), "https://localhost/path", false) + err := validateURL(t.Context(), &url.URL{Scheme: "https", Host: "localhost"}, false) if !errors.Is(err, ErrSSRFBlocked) { t.Errorf("validateURL localhost = %v, want ErrSSRFBlocked", err) } @@ -217,7 +194,7 @@ func TestValidateURL_RejectsLocalhost(t *testing.T) { func TestValidateURL_EmptyHost(t *testing.T) { t.Parallel() - err := validateURL(t.Context(), "https:///path", false) + err := validateURL(t.Context(), &url.URL{Scheme: "https"}, false) if !errors.Is(err, ErrInvalidHost) { t.Errorf("validateURL empty host = %v, want ErrInvalidHost", err) } @@ -232,7 +209,8 @@ func TestMockFetcher_FetchesFile(t *testing.T) { m := NewMock(mockFS) - result, err := m.Fetch(context.Background(), "https://example.com/images/photo.jpg") + result, err := m.Fetch(context.Background(), + imagerequest.Source{Host: testHost, Path: "/images/photo.jpg"}) if err != nil { t.Fatalf("Fetch() error = %v", err) } @@ -263,7 +241,8 @@ func TestMockFetcher_MissingFileReturnsUpstreamError(t *testing.T) { mockFS := fstest.MapFS{} m := NewMock(mockFS) - _, err := m.Fetch(context.Background(), "https://example.com/missing.jpg") + _, err := m.Fetch(context.Background(), + imagerequest.Source{Host: testHost, Path: "/missing.jpg"}) if !errors.Is(err, ErrUpstreamError) { t.Errorf("Fetch() error = %v, want ErrUpstreamError", err) } @@ -280,7 +259,7 @@ func TestMockFetcher_RespectsContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := m.Fetch(ctx, "https://example.com/photo.jpg") + _, err := m.Fetch(ctx, imagerequest.Source{Host: testHost, Path: "/photo.jpg"}) if !errors.Is(err, context.Canceled) { t.Errorf("Fetch() error = %v, want context.Canceled", err) } diff --git a/internal/httpfetcher/max_connections_internal_test.go b/internal/httpfetcher/max_connections_internal_test.go index d414178..bb80228 100644 --- a/internal/httpfetcher/max_connections_internal_test.go +++ b/internal/httpfetcher/max_connections_internal_test.go @@ -8,14 +8,20 @@ import ( "sync" "testing" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) -// imageURLOnPort is the fake upstream's image route on testPublicHost at +// imageSourceOn is the fake upstream's image route on host. +func imageSourceOn(host string) imagerequest.Source { + return imagerequest.Source{Host: host, Path: "/image"} +} + +// imageSourceOnPort is the fake upstream's image route on testPublicHost at // port. Each port is a different host to the per-host limit, while the test // dialer sends every port to the one test server. -func imageURLOnPort(port int) string { - return "http://" + net.JoinHostPort(testPublicHost, strconv.Itoa(port)) + - "/image" +func imageSourceOnPort(port int) imagerequest.Source { + return imageSourceOn(net.JoinHostPort(testPublicHost, strconv.Itoa(port))) } // TestFetchLimitsConnectionsToAllHostsTogether checks that MaxConnections @@ -39,12 +45,12 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond - first, err := f.Fetch(testContext(t), imageURLOnPort(81)) + first, err := f.Fetch(testContext(t), imageSourceOnPort(81)) if err != nil { t.Fatalf("first Fetch() error = %v", err) } - second, err := f.Fetch(testContext(t), imageURLOnPort(82)) + second, err := f.Fetch(testContext(t), imageSourceOnPort(82)) if err != nil { t.Fatalf("second Fetch() error = %v", err) } @@ -53,7 +59,7 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { start := time.Now() - _, err = f.Fetch(testContext(t), imageURLOnPort(83)) + _, err = f.Fetch(testContext(t), imageSourceOnPort(83)) if !errors.Is(err, ErrTooManyConnections) { t.Fatalf("third Fetch() error = %v, want ErrTooManyConnections", err) } @@ -72,7 +78,7 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { t.Fatalf("close first body: %v", err) } - third, err := f.Fetch(testContext(t), imageURLOnPort(83)) + third, err := f.Fetch(testContext(t), imageSourceOnPort(83)) if err != nil { t.Fatalf("Fetch() after a response was closed: error = %v", err) } @@ -99,7 +105,7 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) - first, err := f.Fetch(testContext(t), imageURLOnPort(81)) + first, err := f.Fetch(testContext(t), imageSourceOnPort(81)) if err != nil { t.Fatalf("first Fetch() error = %v", err) } @@ -109,7 +115,7 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - _, err = f.Fetch(ctx, imageURLOnPort(82)) + _, err = f.Fetch(ctx, imageSourceOnPort(82)) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("second Fetch() error = %v, want context.DeadlineExceeded", err) } @@ -134,7 +140,7 @@ func TestFetchRemovesIdleHostSemaphores(t *testing.T) { for i := range 100 { wg.Go(func() { - res, err := f.Fetch(ctx, imageURLOnPort(1+i%50)) + res, err := f.Fetch(ctx, imageSourceOnPort(1+i%50)) if err != nil { t.Errorf("Fetch() error = %v", err) @@ -173,12 +179,12 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond - open, err := f.Fetch(testContext(t), imageURLOnPort(81)) + open, err := f.Fetch(testContext(t), imageSourceOnPort(81)) if err != nil { t.Fatalf("first Fetch() error = %v", err) } - _, err = f.Fetch(testContext(t), imageURLOnPort(82)) + _, err = f.Fetch(testContext(t), imageSourceOnPort(82)) if !errors.Is(err, ErrTooManyConnections) { t.Fatalf("Fetch() from another host: error = %v, "+ "want ErrTooManyConnections", err) @@ -187,7 +193,7 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - _, err = f.Fetch(ctx, imageURLOnPort(81)) + _, err = f.Fetch(ctx, imageSourceOnPort(81)) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("Fetch() from the busy host: error = %v, "+ "want context.DeadlineExceeded", err) @@ -199,7 +205,7 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - _, err = f.Fetch(ctx, imageURLOnPort(83)) + _, err = f.Fetch(ctx, imageSourceOnPort(83)) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("Fetch() from a host with nothing open: error = %v, "+ "want context.DeadlineExceeded", err) @@ -222,15 +228,15 @@ func TestFetchReleasesConnectionOnError(t *testing.T) { t.Parallel() cases := []struct { - name string - url string - want error + name string + source imagerequest.Source + want error }{ - {"upstream answers 500", upstreamURL("/status/500"), ErrUpstreamError}, - {"upstream sends HTML", upstreamURL("/html"), ErrInvalidContentType}, + {"upstream answers 500", upstreamSource("/status/500"), ErrUpstreamError}, + {"upstream sends HTML", upstreamSource("/html"), ErrInvalidContentType}, // 198.51.100.7 (TEST-NET-2) passes the SSRF checks, and the test // dialer refuses every host but testPublicHost. - {"connecting fails", "http://198.51.100.7/image", errUnexpectedDial}, + {"connecting fails", imageSourceOn("198.51.100.7"), errUnexpectedDial}, } for _, tc := range cases { @@ -249,7 +255,7 @@ func TestFetchReleasesConnectionOnError(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond - _, err := f.Fetch(testContext(t), tc.url) + _, err := f.Fetch(testContext(t), tc.source) if !errors.Is(err, tc.want) { t.Fatalf("Fetch() error = %v, want %v", err, tc.want) } diff --git a/internal/httpfetcher/mock.go b/internal/httpfetcher/mock.go index 2a3b11b..40ac3e6 100644 --- a/internal/httpfetcher/mock.go +++ b/internal/httpfetcher/mock.go @@ -7,14 +7,14 @@ import ( "io/fs" "net/http" "strings" -) -// errEmptyURLPath is returned when a mock URL has no usable path. -var errEmptyURLPath = errors.New("empty URL path") + "sneak.berlin/go/pixa/internal/imagerequest" +) // MockFetcher implements Fetcher using an embedded filesystem. // Files are organized as: hostname/path/to/file.ext -// URLs like https://example.com/images/photo.jpg map to example.com/images/photo.jpg. +// The source with host example.com and path /images/photo.jpg maps to +// example.com/images/photo.jpg; the source query is ignored. type MockFetcher struct { fs fs.FS } @@ -24,8 +24,10 @@ func NewMock(fsys fs.FS) *MockFetcher { return &MockFetcher{fs: fsys} } -// Fetch retrieves content from the mock filesystem. -func (m *MockFetcher) Fetch(ctx context.Context, url string) (*FetchResult, error) { +// Fetch retrieves the source's content from the mock filesystem. +func (m *MockFetcher) Fetch( + ctx context.Context, source imagerequest.Source, +) (*FetchResult, error) { // Check context cancellation select { case <-ctx.Done(): @@ -33,11 +35,7 @@ func (m *MockFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro default: } - // Parse URL to get filesystem path - path, err := urlToFSPath(url) - if err != nil { - return nil, err - } + path := source.Host + source.Path // Open the file f, err := m.fs.Open(path) @@ -68,32 +66,6 @@ func (m *MockFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro }, nil } -// urlToFSPath converts a URL to a filesystem path. -// https://example.com/images/photo.jpg -> example.com/images/photo.jpg -func urlToFSPath(rawURL string) (string, error) { - // Strip scheme - url := rawURL - if idx := strings.Index(url, "://"); idx != -1 { - url = url[idx+3:] - } - - // Remove query string - if idx := strings.Index(url, "?"); idx != -1 { - url = url[:idx] - } - - // Remove fragment - if idx := strings.Index(url, "#"); idx != -1 { - url = url[:idx] - } - - if url == "" { - return "", errEmptyURLPath - } - - return url, nil -} - // detectContentTypeFromPath returns the MIME type based on file extension. func detectContentTypeFromPath(path string) string { path = strings.ToLower(path) diff --git a/internal/httpfetcher/request_id_internal_test.go b/internal/httpfetcher/request_id_internal_test.go index a3c998e..62a654d 100644 --- a/internal/httpfetcher/request_id_internal_test.go +++ b/internal/httpfetcher/request_id_internal_test.go @@ -34,7 +34,7 @@ func TestFetchSendsRequestID(t *testing.T) { ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID) - res, err := f.Fetch(ctx, upstreamURL("/image")) + res, err := f.Fetch(ctx, upstreamSource("/image")) if err != nil { t.Fatalf("Fetch() error = %v", err) } diff --git a/internal/imagerequest/imagerequest.go b/internal/imagerequest/imagerequest.go index ecc86b4..d9b2bac 100644 --- a/internal/imagerequest/imagerequest.go +++ b/internal/imagerequest/imagerequest.go @@ -1,13 +1,12 @@ // Package imagerequest holds the image request type, its parts, its checks, -// the parsing of the /v1/image/ path, the variant key and the source key. It -// imports no other package of this repo, so every package that handles an -// image request can import it. +// the parsing of the /v1/image/ path, the variant key, the source key and the +// source's upstream URL. It imports no other package of this repo, so every +// package that handles an image request can import it. package imagerequest import ( "errors" "net/url" - "time" ) // ErrInvalidFitMode is returned when an invalid fit mode is provided. @@ -85,19 +84,13 @@ func ValidateImageRequest(req *Request) error { } // Request represents a request for a processed image: the source to fetch -// and the transform to apply to it. +// and the transform to apply to it. It holds exactly what the variant key +// covers. type Request struct { // Source is what to fetch Source Source // Transform is what to do to it Transform Transform - // Signature is the HMAC signature for non-allowlisted hosts - Signature string - // Expires is when the URL expires: the exp of a signed URL, or the expiry - // of an encrypted URL; the zero time if it has none - Expires time.Time - // AllowHTTP indicates whether HTTP (non-TLS) is allowed for this request - AllowHTTP bool } // Source is the image a request fetches from the origin. @@ -125,23 +118,20 @@ type Transform struct { FitMode Fit } -// SourceURL returns the full upstream URL to fetch: the source path -// percent-encoded again, as net/url writes a path, so each source path is one -// upstream path, and the source query as it is, with the characters -// escapeQuery encodes percent-encoded. Uses http:// scheme when AllowHTTP is -// true, otherwise https://. -func (r *Request) SourceURL() string { +// URL returns the source's upstream URL, with the scheme http when allowHTTP +// is true and https otherwise. net/url writes its path percent-encoded again, +// so each source path is one upstream path. Its query is the source query as +// it is, with the characters escapeQuery encodes percent-encoded. +func (s Source) URL(allowHTTP bool) *url.URL { scheme := "https" - if r.AllowHTTP { + if allowHTTP { scheme = "http" } - sourceURL := url.URL{ + return &url.URL{ Scheme: scheme, - Host: r.Source.Host, - Path: r.Source.Path, - RawQuery: escapeQuery(r.Source.Query), + Host: s.Host, + Path: s.Path, + RawQuery: escapeQuery(s.Query), } - - return sourceURL.String() } diff --git a/internal/imagerequest/sourceurl_internal_test.go b/internal/imagerequest/sourceurl_internal_test.go index 6e6b671..0b953fd 100644 --- a/internal/imagerequest/sourceurl_internal_test.go +++ b/internal/imagerequest/sourceurl_internal_test.go @@ -2,57 +2,49 @@ package imagerequest import "testing" -func TestRequest_SourceURL_DefaultHTTPS(t *testing.T) { +func TestSource_URL_DefaultHTTPS(t *testing.T) { t.Parallel() - req := &Request{ - Source: Source{ - Host: testHostCDN, - Path: testPathCat, - Query: "v=2", - }, + source := Source{ + Host: testHostCDN, + Path: testPathCat, + Query: "v=2", } - got := req.SourceURL() + got := source.URL(false).String() want := "https://cdn.example.com/photos/cat.jpg?v=2" if got != want { - t.Errorf("SourceURL() = %q, want %q", got, want) + t.Errorf("URL() = %q, want %q", got, want) } } -func TestRequest_SourceURL_AllowHTTP(t *testing.T) { +func TestSource_URL_AllowHTTP(t *testing.T) { t.Parallel() - req := &Request{ - Source: Source{ - Host: "localhost:8080", - Path: testPathCat, - }, - AllowHTTP: true, + source := Source{ + Host: "localhost:8080", + Path: testPathCat, } - got := req.SourceURL() + got := source.URL(true).String() want := "http://localhost:8080/photos/cat.jpg" if got != want { - t.Errorf("SourceURL() = %q, want %q", got, want) + t.Errorf("URL() = %q, want %q", got, want) } } -func TestRequest_SourceURL_AllowHTTPFalse(t *testing.T) { +func TestSource_URL_AllowHTTPFalse(t *testing.T) { t.Parallel() - req := &Request{ - Source: Source{ - Host: testHostCDN, - Path: "/img.jpg", - }, - AllowHTTP: false, + source := Source{ + Host: testHostCDN, + Path: "/img.jpg", } - got := req.SourceURL() + got := source.URL(false).String() if got != "https://cdn.example.com/img.jpg" { - t.Errorf("SourceURL() = %q, want https scheme", got) + t.Errorf("URL() = %q, want https scheme", got) } } diff --git a/internal/imagerequest/urlparser.go b/internal/imagerequest/urlparser.go index 9e8cc4c..09d7bd5 100644 --- a/internal/imagerequest/urlparser.go +++ b/internal/imagerequest/urlparser.go @@ -126,15 +126,15 @@ func parseImageComponents(remainder string) (*Request, error) { return nil, ErrMissingHost } - // Only the forms hostRegex describes are accepted. A "?", "#" or "@" in - // the host would make Go's URL parser, which the allowlist check and the - // fetcher use, read a different host from the one allowlisted and cached + // Later steps put the host unparsed into the allowlist check, the cache key + // and the upstream URL, so only the forms hostRegex describes are accepted hostMatch := hostRegex.FindStringSubmatch(host) if hostMatch == nil { return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host) } - // Go's URL parser refuses brackets around anything but an IPv6 address + // The allowlist check and the fetcher drop the brackets, so brackets + // around anything but an IPv6 address would give a host a second spelling if inBrackets := hostMatch[1]; inBrackets != "" { addr, err := netip.ParseAddr(inBrackets) if err != nil || !addr.Is6() { @@ -147,9 +147,8 @@ func parseImageComponents(remainder string) (*Request, error) { // host host = strings.ToLower(host) - // Go's URL parser, which the fetcher uses, reads a "#" as the start of a - // fragment and drops the rest, so each text after it would be another - // spelling, cached under its own keys, of one upstream request + // A "#" starts a URL's fragment, so neither the source path nor the source + // query may hold one if strings.Contains(path, "#") { return nil, fmt.Errorf( `%w: the source path or query holds a "#"`, ErrInvalidPath) diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imgcache/concurrent_misses_internal_test.go index 226e79e..594e4fb 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imgcache/concurrent_misses_internal_test.go @@ -36,7 +36,7 @@ type heldFetcher struct { } func (f *heldFetcher) Fetch( - ctx context.Context, url string, + ctx context.Context, source imagerequest.Source, ) (*httpfetcher.FetchResult, error) { f.fetches.Add(1) @@ -48,7 +48,7 @@ func (f *heldFetcher) Fetch( return nil, ctx.Err() } - return f.upstream.Fetch(ctx, url) + return f.upstream.Fetch(ctx, source) } // releaseFetches lets every held fetch, and every later one, go on. @@ -79,8 +79,8 @@ func setupHeldFetchService(t *testing.T) (*Service, *TestFixtures, *heldFetcher) } // photoVariant asks for the test photo, 100x100, at 50x25 as a JPEG of the -// given quality and fit mode. Each call returns a new request, as Get writes -// to the request it is given. +// given quality and fit mode. Each call returns a new request, which a test +// may change. func photoVariant( fixtures *TestFixtures, quality int, fit imagerequest.Fit, ) *imagerequest.Request { @@ -489,7 +489,7 @@ func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) { type panickingFetcher struct{} func (panickingFetcher) Fetch( - context.Context, string, + context.Context, imagerequest.Source, ) (*httpfetcher.FetchResult, error) { panic("upstream fetcher panicked") } diff --git a/internal/imgcache/service.go b/internal/imgcache/service.go index ece9311..35d762a 100644 --- a/internal/imgcache/service.go +++ b/internal/imgcache/service.go @@ -7,21 +7,17 @@ import ( "fmt" "io" "log/slog" - "net/url" "runtime/debug" - "strings" "time" "github.com/dustin/go-humanize" "github.com/getsentry/sentry-go" "github.com/go-chi/chi/v5/middleware" "golang.org/x/sync/singleflight" - "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/magic" - "sneak.berlin/go/pixa/internal/signature" ) // Service orchestrates the cache, fetcher, and processor. @@ -29,10 +25,7 @@ type Service struct { cache *Cache fetcher httpfetcher.Fetcher processor *imageprocessor.ImageProcessor - signer *signature.Signer - allowlist *allowlist.HostAllowList log *slog.Logger - allowHTTP bool maxResponseSize int64 // variantsInProgress lets the requests that miss the same variant at the // same time share one fetch and one transcode. @@ -47,13 +40,6 @@ type ServiceConfig struct { Fetcher httpfetcher.Fetcher // Processor converts and resizes the images Processor *imageprocessor.ImageProcessor - // SigningKey is the HMAC signing key (empty disables signing) - SigningKey string - // Allowlist is the list of hosts that don't require signatures - Allowlist []string - // AllowHTTP makes the service fetch sources over http instead of - // https (testing only), so the fetcher must allow http too - AllowHTTP bool // MaxResponseSize is the largest source accepted, in bytes; a cached // source larger than this is fetched again MaxResponseSize int64 @@ -61,11 +47,8 @@ type ServiceConfig struct { Logger *slog.Logger } -// Static errors for service construction. -var ( - errCacheRequired = errors.New("cache is required") - errSigningKeyRequired = errors.New("signing key is required") -) +// errCacheRequired is returned when the service is built without a cache. +var errCacheRequired = errors.New("cache is required") // NewService creates a new image service. func NewService(cfg *ServiceConfig) (*Service, error) { @@ -73,10 +56,6 @@ func NewService(cfg *ServiceConfig) (*Service, error) { return nil, errCacheRequired } - if cfg.SigningKey == "" { - return nil, errSigningKeyRequired - } - log := cfg.Logger if log == nil { log = slog.Default() @@ -93,10 +72,7 @@ func NewService(cfg *ServiceConfig) (*Service, error) { cache: cfg.Cache, fetcher: cfg.Fetcher, processor: cfg.Processor, - signer: signature.New(cfg.SigningKey), - allowlist: allowlist.New(cfg.Allowlist), log: log, - allowHTTP: cfg.AllowHTTP, maxResponseSize: cfg.MaxResponseSize, }, nil } @@ -123,9 +99,6 @@ var ErrProcessingTimedOut = errors.New( func (s *Service) Get( ctx context.Context, req *imagerequest.Request, ) (*ImageResponse, error) { - // Propagate AllowHTTP setting to the request - req.AllowHTTP = s.allowHTTP - cacheKey := imagerequest.CacheKey(req) reader, size, contentType, err := s.cache.GetVariant(cacheKey) @@ -169,33 +142,6 @@ func (s *Service) WaitForProcessing(ctx context.Context) int { return s.processor.WaitForProcessing(ctx) } -// ValidateRequest validates the request signature if required. -func (s *Service) ValidateRequest(req *imagerequest.Request) error { - // Check if host is allowed (no signature required) - sourceURL := req.SourceURL() - - parsedURL, err := url.Parse(sourceURL) - if err != nil { - return fmt.Errorf("invalid source URL: %w", err) - } - - // An allowlist entry names no port, so a host followed by ":", with or - // without a port, needs a signature. The port is read from the parsed - // URL, as the fetcher reads it: in an IPv6 address written without - // brackets, such as 2001:db8::1:8443, it is what follows the last ":". - // This is checked here and not in allowlist.IsAllowed, which also matches - // referer_blocklist, where a Referer must be blocked whatever its port. - hostHasPort := parsedURL.Port() != "" || - strings.HasSuffix(parsedURL.Host, ":") - - if !hostHasPort && s.allowlist.IsAllowed(parsedURL) { - return nil - } - - // Signature required for non-allowed hosts - return s.signer.Verify(req) -} - // errPanicked is returned when processing a variant panicked. var errPanicked = errors.New("panic while processing image") @@ -388,11 +334,10 @@ func (s *Service) fetchAndProcess( } // Fetch from upstream - sourceURL := req.SourceURL() + s.log.Debug("fetching from upstream", "host", req.Source.Host, + "path", req.Source.Path, "query", req.Source.Query) - s.log.Debug("fetching from upstream", "url", sourceURL) - - fetchResult, err := s.fetcher.Fetch(ctx, sourceURL) + fetchResult, err := s.fetcher.Fetch(ctx, req.Source) if err != nil { // Store negative cache for certain errors if isNegativeCacheable(err) { diff --git a/internal/imgcache/service_internal_test.go b/internal/imgcache/service_internal_test.go index 31573e1..ed8ae02 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imgcache/service_internal_test.go @@ -9,17 +9,15 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/magic" - "sneak.berlin/go/pixa/internal/signature" ) // Test data literals used repeatedly in this file (goconst). const ( testPathPhoto = "/images/photo.jpg" testPathUpload = "/uploads/image.jpg" - testSigningKey = "test-signing-key-12345" ) -func TestService_Get_AllowlistedHost(t *testing.T) { +func TestService_Get_GoodHost(t *testing.T) { t.Parallel() svc, fixtures := SetupTestService(t) @@ -60,36 +58,12 @@ func TestService_Get_AllowlistedHost(t *testing.T) { } } -func TestService_Get_NonAllowlistedHost_NoSignature(t *testing.T) { +// TestService_Get_OtherHost checks that the image service serves a second host +// as it serves GoodHost. +func TestService_Get_OtherHost(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t, WithSigningKey("test-key")) - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: fixtures.OtherHost, - Path: testPathUpload, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Should fail validation - not allowlisted and no signature - err := svc.ValidateRequest(req) - if err == nil { - t.Error("ValidateRequest() expected error for non-allowlisted host without signature") - } -} - -func TestService_Get_NonAllowlistedHost_ValidSignature(t *testing.T) { - t.Parallel() - - signingKey := testSigningKey - svc, fixtures := SetupTestService(t, WithSigningKey(signingKey)) + svc, fixtures := SetupTestService(t) ctx := context.Background() req := &imagerequest.Request{ @@ -105,17 +79,6 @@ func TestService_Get_NonAllowlistedHost_ValidSignature(t *testing.T) { }, } - // Generate a valid signature - signer := signature.New(signingKey) - req.Expires = time.Now().Add(time.Hour) - req.Signature = signer.Sign(req) - - // Should pass validation - err := svc.ValidateRequest(req) - if err != nil { - t.Errorf("ValidateRequest() error = %v", err) - } - // Should fetch successfully resp, err := svc.Get(ctx, req) if err != nil { @@ -134,244 +97,6 @@ func TestService_Get_NonAllowlistedHost_ValidSignature(t *testing.T) { } } -func TestService_Get_NonAllowlistedHost_ExpiredSignature(t *testing.T) { - t.Parallel() - - signingKey := testSigningKey - svc, fixtures := SetupTestService(t, WithSigningKey(signingKey)) - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: fixtures.OtherHost, - Path: testPathUpload, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Generate an expired signature - signer := signature.New(signingKey) - req.Expires = time.Now().Add(-time.Hour) // Already expired - req.Signature = signer.Sign(req) - - // Should fail validation - err := svc.ValidateRequest(req) - if err == nil { - t.Error("ValidateRequest() expected error for expired signature") - } -} - -func TestService_Get_NonAllowlistedHost_InvalidSignature(t *testing.T) { - t.Parallel() - - signingKey := testSigningKey - svc, fixtures := SetupTestService(t, WithSigningKey(signingKey)) - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: fixtures.OtherHost, - Path: testPathUpload, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - // Set an invalid signature - req.Signature = "invalid-signature" - req.Expires = time.Now().Add(time.Hour) - - // Should fail validation - err := svc.ValidateRequest(req) - if err == nil { - t.Error("ValidateRequest() expected error for invalid signature") - } -} - -// TestService_ValidateRequest_SignatureExactHostMatch verifies that -// ValidateRequest enforces exact host matching for signatures. A -// signature for one host must not verify for a different host, even -// if they share a domain suffix. -func TestService_ValidateRequest_SignatureExactHostMatch(t *testing.T) { - t.Parallel() - - signingKey := "test-signing-key-must-be-32-chars" - svc, _ := SetupTestService(t, - WithSigningKey(signingKey), - WithNoAllowlist(), - ) - - signer := signature.New(signingKey) - - // Sign a request for "cdn.example.com" - signedReq := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: testHostCDN, - Path: testPathCat, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - Expires: time.Now().Add(time.Hour), - } - signedReq.Signature = signer.Sign(signedReq) - - // The original request should pass validation - t.Run("exact host passes", func(t *testing.T) { - t.Parallel() - - err := svc.ValidateRequest(signedReq) - if err != nil { - t.Errorf("ValidateRequest() exact host failed: %v", err) - } - }) - - // Try to reuse the signature with different hosts - tests := []struct { - name string - host string - }{ - {"parent domain", testHostExample}, - {"sibling subdomain", "images.example.com"}, - {"deeper subdomain", "a.cdn.example.com"}, - {"evil suffix domain", "cdn.example.com.evil.com"}, - {"prefixed host", "evilcdn.example.com"}, - } - - for _, tt := range tests { - t.Run(tt.name+" rejected", func(t *testing.T) { - t.Parallel() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: tt.host, - Path: signedReq.Source.Path, - Query: signedReq.Source.Query, - }, - Transform: signedReq.Transform, - Expires: signedReq.Expires, - Signature: signedReq.Signature, - } - - err := svc.ValidateRequest(req) - if err == nil { - t.Errorf( - "ValidateRequest() should reject signature for host %q (signed for %q)", - tt.host, signedReq.Source.Host) - } - }) - } -} - -// TestService_ValidateRequest_HostWithPortNeedsSignature verifies that a host -// followed by a port, or by a colon with no port, matches no allowlist entry, -// so that a request for it without a signature is refused, while the same -// host with no port is allowlisted. -func TestService_ValidateRequest_HostWithPortNeedsSignature(t *testing.T) { - t.Parallel() - - svc, _ := SetupTestService(t, - WithSigningKey(testSigningKey), - WithAllowlist(testHostCDN, "203.0.113.5", "2001:db8::1"), - ) - - tests := []struct { - host string - allowlisted bool - }{ - {testHostCDN, true}, - {"cdn.example.com:8443", false}, - {"cdn.example.com:443", false}, - {"cdn.example.com:", false}, - {"203.0.113.5", true}, - {"203.0.113.5:8080", false}, - {"[2001:db8::1]", true}, - {"[2001:db8::1]:8443", false}, - } - - for _, tt := range tests { - t.Run(tt.host, func(t *testing.T) { - t.Parallel() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: tt.host, - Path: testPathCat, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - err := svc.ValidateRequest(req) - t.Logf("ValidateRequest() for host %q with no signature: %v", - tt.host, err) - - if allowlisted := err == nil; allowlisted != tt.allowlisted { - t.Errorf("host %q served without a signature = %v, want %v", - tt.host, allowlisted, tt.allowlisted) - } - }) - } -} - -// TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature verifies that -// an allowlisted IPv6 address written without brackets and followed by a -// port, or by a colon with no port, needs a signature. Go's URL parser, which -// the fetcher uses, reads 2001:db8::1:8443 as the host 2001:db8::1 with the -// port 8443. -func TestService_ValidateRequest_UnbracketedIPv6PortNeedsSignature( - t *testing.T, -) { - t.Parallel() - - svc, _ := SetupTestService(t, - WithSigningKey(testSigningKey), - WithAllowlist("2001:db8::1"), - ) - - for _, host := range []string{"2001:db8::1:8443", "2001:db8::1:"} { - t.Run(host, func(t *testing.T) { - t.Parallel() - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: host, - Path: testPathCat, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - err := svc.ValidateRequest(req) - t.Logf("ValidateRequest() for host %q with no signature: %v", - host, err) - - if err == nil { - t.Errorf("host %q served without a signature, want refused", - host) - } - }) - } -} - func TestService_Get_InvalidFile(t *testing.T) { t.Parallel() @@ -692,33 +417,6 @@ func TestService_Get_DifferentSizes(t *testing.T) { } } -func TestService_ValidateRequest_NoSigningKey(t *testing.T) { - t.Parallel() - - // Service with no signing key - all non-allowlisted requests should fail - svc, fixtures := SetupTestService(t, WithNoAllowlist()) - - req := &imagerequest.Request{ - Source: imagerequest.Source{ - Host: fixtures.OtherHost, - Path: testPathUpload, - }, - Transform: imagerequest.Transform{ - Size: imagerequest.Size{Width: 50, Height: 50}, - Format: imagerequest.FormatJPEG, - Quality: 85, - FitMode: imagerequest.FitCover, - }, - } - - err := svc.ValidateRequest(req) - if err == nil { - t.Error( - "ValidateRequest() expected error when no signing key and host not allowlisted", - ) - } -} - func TestService_Get_ContextCancellation(t *testing.T) { t.Parallel() diff --git a/internal/imgcache/stats_internal_test.go b/internal/imgcache/stats_internal_test.go index 6a90fbe..6e4912a 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imgcache/stats_internal_test.go @@ -93,7 +93,7 @@ type fakeUpstream struct { } func (u *fakeUpstream) Fetch( - context.Context, string, + context.Context, imagerequest.Source, ) (*httpfetcher.FetchResult, error) { return &httpfetcher.FetchResult{ Content: io.NopCloser(u), diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index 31c2c79..ec44fb3 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -31,11 +31,11 @@ const ( // TestFixtures contains paths to test files in the mock filesystem. type TestFixtures struct { // Valid image files - GoodHostJPEG string // allowlisted host, valid JPEG - GoodHostPNG string // allowlisted host, valid PNG - GoodHostGIF string // allowlisted host, valid GIF - OtherHostJPEG string // non-allowlisted host, valid JPEG - OtherHostPNG string // non-allowlisted host, valid PNG + GoodHostJPEG string // valid JPEG on GoodHost + GoodHostPNG string // valid PNG on GoodHost + GoodHostGIF string // valid GIF on GoodHost + OtherHostJPEG string // valid JPEG on OtherHost + OtherHostPNG string // valid PNG on OtherHost // Invalid/edge case files InvalidFile string // file with wrong magic bytes @@ -43,8 +43,8 @@ type TestFixtures struct { TextFile string // text file masquerading as image // Hostnames - GoodHost string // allowlisted hostname - OtherHost string // non-allowlisted hostname + GoodHost string // a host the mock fetcher serves + OtherHost string // a second host the mock fetcher serves } // DefaultFixtures returns the standard test fixture paths. @@ -161,22 +161,11 @@ func generateTestGIF(t *testing.T, width, height int, c color.Color) []byte { } // SetupTestService creates a Service with mock fetcher for testing. -func SetupTestService( - t *testing.T, opts ...TestServiceOption, -) (*Service, *TestFixtures) { +func SetupTestService(t *testing.T) (*Service, *TestFixtures) { t.Helper() mockFS, fixtures := NewTestFS(t) - cfg := &testServiceConfig{ - allowlist: []string{fixtures.GoodHost}, - signingKey: "test-signing-key-must-be-32-chars", - } - - for _, opt := range opts { - opt(cfg) - } - // Create temp directory for cache tmpDir := t.TempDir() @@ -199,8 +188,6 @@ func SetupTestService( MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2, }), - SigningKey: cfg.signingKey, - Allowlist: cfg.allowlist, MaxResponseSize: 50 << 20, }) if err != nil { @@ -228,32 +215,3 @@ func setupServiceTestDB(t *testing.T) *sql.DB { return db } - -type testServiceConfig struct { - allowlist []string - signingKey string -} - -// TestServiceOption configures the test service. -type TestServiceOption func(*testServiceConfig) - -// WithAllowlist sets the allowlist for the test service. -func WithAllowlist(hosts ...string) TestServiceOption { - return func(c *testServiceConfig) { - c.allowlist = hosts - } -} - -// WithSigningKey sets the signing key for the test service. -func WithSigningKey(key string) TestServiceOption { - return func(c *testServiceConfig) { - c.signingKey = key - } -} - -// WithNoAllowlist removes all allowlisted hosts. -func WithNoAllowlist() TestServiceOption { - return func(c *testServiceConfig) { - c.allowlist = nil - } -} diff --git a/internal/providers/providers.go b/internal/providers/providers.go index b098b4e..7d2a951 100644 --- a/internal/providers/providers.go +++ b/internal/providers/providers.go @@ -212,21 +212,15 @@ type imageServiceParams struct { } // newImageService builds the image service from the cache, the fetcher -// and the processor, with the signing key, the allowlist and the upstream -// settings of the config. +// and the processor, with the largest upstream response the config allows. func newImageService( _ fx.Lifecycle, params imageServiceParams, ) (*imgcache.Service, error) { - cfg := params.Config - return imgcache.NewService(&imgcache.ServiceConfig{ Cache: params.Cache, Fetcher: params.Fetcher, Processor: params.Processor, - SigningKey: cfg.SigningKey, - Allowlist: cfg.AllowlistHosts, - AllowHTTP: cfg.AllowHTTP, - MaxResponseSize: cfg.UpstreamMaxResponseSize, + MaxResponseSize: params.Config.UpstreamMaxResponseSize, Logger: params.Logger.Get(), }) } diff --git a/internal/server/computed_side_internal_test.go b/internal/server/computed_side_internal_test.go index ad81740..95fb696 100644 --- a/internal/server/computed_side_internal_test.go +++ b/internal/server/computed_side_internal_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" @@ -76,7 +77,7 @@ func TestImageProxyRefusesComputedSideAboveMaximum(t *testing.T) { Size: imagerequest.Size{Width: tt.width, Height: tt.height}, Format: imagerequest.FormatJPEG, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("encurl Generate: %v", err) } diff --git a/internal/server/fit_outside_internal_test.go b/internal/server/fit_outside_internal_test.go index cbc6a80..1c095fd 100644 --- a/internal/server/fit_outside_internal_test.go +++ b/internal/server/fit_outside_internal_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" @@ -66,7 +67,7 @@ func TestImageProxyRefusesFitOutsideAboveMaximum(t *testing.T) { Format: imagerequest.FormatJPEG, FitMode: imagerequest.FitOutside, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("encurl Generate: %v", err) } diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index 550b8e5..68676d8 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -524,8 +524,7 @@ func signedImageURL(t *testing.T, sourceHost string) string { Quality: 85, FitMode: imagerequest.FitCover, }, - Expires: expires, - }) + }, expires) // Go's percent-encoding of a URL path writes the brackets of the IPv6 // host as %5B and %5D. @@ -549,7 +548,7 @@ func encryptedImageURL(t *testing.T, sourceHost string) string { token, err := generator.Generate(&imagerequest.Request{ Source: imagerequest.Source{Host: sourceHost, Path: ipv6SourcePath}, Transform: imagerequest.Transform{Format: imagerequest.FormatPNG}, - }) + }, time.Time{}) if err != nil { t.Fatalf("encurl Generate: %v", err) } diff --git a/internal/server/svg_as_avif_internal_test.go b/internal/server/svg_as_avif_internal_test.go index af75b24..0993c16 100644 --- a/internal/server/svg_as_avif_internal_test.go +++ b/internal/server/svg_as_avif_internal_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" @@ -45,7 +46,7 @@ func TestImageProxyRefusesSVGServedAsAVIF(t *testing.T) { Size: imagerequest.Size{Width: 32, Height: 24}, Format: format, }, - }) + }, time.Time{}) if err != nil { t.Fatalf("encurl Generate: %v", err) } diff --git a/internal/signature/escaping_test.go b/internal/signature/escaping_test.go index 032df24..30d9603 100644 --- a/internal/signature/escaping_test.go +++ b/internal/signature/escaping_test.go @@ -28,20 +28,20 @@ func TestSigner_Verify_ColonCannotMovePathQueryBoundary(t *testing.T) { Quality: 85, FitMode: testFitCover, }, - Expires: time.Now().Add(1 * time.Hour), } - signed.Signature = signer.Sign(signed) + expires := time.Now().Add(1 * time.Hour) + sig := signer.Sign(signed, expires) replayed := *signed replayed.Source.Path = "/a" replayed.Source.Query = "b:" t.Logf("signature for path %q query %q: %s", - signed.Source.Path, signed.Source.Query, signed.Signature) + signed.Source.Path, signed.Source.Query, sig) t.Logf("signature for path %q query %q: %s", - replayed.Source.Path, replayed.Source.Query, signer.Sign(&replayed)) + replayed.Source.Path, replayed.Source.Query, signer.Sign(&replayed, expires)) - err := signer.Verify(&replayed) + err := signer.Verify(&replayed, sig, expires) if !errors.Is(err, signature.ErrInvalid) { t.Errorf("Verify() of path %q query %q with the signature for "+ "path %q = %v, want %v", @@ -106,10 +106,9 @@ func TestSigner_EscapedGoldenVectors(t *testing.T) { Quality: 85, FitMode: testFitCover, }, - Expires: time.Unix(goldenExpiresUnix, 0), } - gotSignature := signer.Sign(req) + gotSignature := signer.Sign(req, time.Unix(goldenExpiresUnix, 0)) t.Logf("Sign() of path %q query %q = %s", tt.path, tt.query, gotSignature) if gotSignature != tt.wantSignature { diff --git a/internal/signature/golden_test.go b/internal/signature/golden_test.go index f3d6e16..9447c9b 100644 --- a/internal/signature/golden_test.go +++ b/internal/signature/golden_test.go @@ -19,8 +19,8 @@ type goldenVector struct { name string req imagerequest.Request // wantSignature is the exact base64url (RFC 4648 URL-safe, - // padded) HMAC-SHA256 signature for the request with Expires - // set to goldenExpiresUnix. + // padded) HMAC-SHA256 signature for the request expiring at + // goldenExpiresUnix. wantSignature string } @@ -124,9 +124,8 @@ func TestSigner_GoldenVectors(t *testing.T) { t.Parallel() signReq := tt.req - signReq.Expires = time.Unix(goldenExpiresUnix, 0) - gotSignature := signer.Sign(&signReq) + gotSignature := signer.Sign(&signReq, time.Unix(goldenExpiresUnix, 0)) if gotSignature != tt.wantSignature { t.Errorf("Sign() = %q, want %q (signed byte format changed?)", gotSignature, tt.wantSignature) diff --git a/internal/signature/quality_fit_test.go b/internal/signature/quality_fit_test.go index 8821b7b..ff27efa 100644 --- a/internal/signature/quality_fit_test.go +++ b/internal/signature/quality_fit_test.go @@ -9,10 +9,12 @@ import ( "sneak.berlin/go/pixa/internal/signature" ) -// signedQualityFitRequest returns a request signed for quality 85 and fit -// mode "cover", the effective defaults the handler applies before -// verification. -func signedQualityFitRequest(signer *signature.Signer) *imagerequest.Request { +// signedQualityFitRequest returns a request, its signature for quality 85 and +// fit mode "cover", the effective defaults the handler applies before +// verification, and the expiry it was signed with. +func signedQualityFitRequest( + signer *signature.Signer, +) (*imagerequest.Request, string, time.Time) { req := &imagerequest.Request{ Source: imagerequest.Source{ Host: testHost, @@ -24,11 +26,10 @@ func signedQualityFitRequest(signer *signature.Signer) *imagerequest.Request { Quality: 85, FitMode: testFitCover, }, - Expires: time.Now().Add(1 * time.Hour), } - req.Signature = signer.Sign(req) + expires := time.Now().Add(1 * time.Hour) - return req + return req, signer.Sign(req, expires), expires } // TestSigner_Verify_QualityAndFitAreSigned proves that quality and fit are @@ -60,10 +61,10 @@ func TestSigner_Verify_QualityAndFitAreSigned(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - req := signedQualityFitRequest(signer) + req, sig, expires := signedQualityFitRequest(signer) tt.tamper(req) - err := signer.Verify(req) + err := signer.Verify(req, sig, expires) if !errors.Is(err, signature.ErrInvalid) { t.Errorf("Verify() = %v, want %v", err, signature.ErrInvalid) } diff --git a/internal/signature/signature.go b/internal/signature/signature.go index 6982756..6ec50aa 100644 --- a/internal/signature/signature.go +++ b/internal/signature/signature.go @@ -34,14 +34,14 @@ func New(secretKey string) *Signer { } } -// Sign generates an HMAC-SHA256 signature for the given request. -// The signature covers: host + path + query + width + height + format + -// expiration + quality + fit. The quality and fit mode are the effective -// values the request resolves to: callers pass the defaults when the request -// omits those parameters, so an omitted quality or fit mode signs +// Sign generates an HMAC-SHA256 signature for the given request, expiring at +// expires. The signature covers: host + path + query + width + height + +// format + expiration + quality + fit. The quality and fit mode are the +// effective values the request resolves to: callers pass the defaults when +// the request omits those parameters, so an omitted quality or fit mode signs // identically to that same value stated explicitly. -func (s *Signer) Sign(req *imagerequest.Request) string { - data := s.buildSignatureData(req) +func (s *Signer) Sign(req *imagerequest.Request, expires time.Time) string { + data := s.buildSignatureData(req, expires) mac := hmac.New(sha256.New, s.secretKey) mac.Write([]byte(data)) sig := mac.Sum(nil) @@ -49,28 +49,31 @@ func (s *Signer) Sign(req *imagerequest.Request) string { return base64.URLEncoding.EncodeToString(sig) } -// Verify checks if the signature on the request is valid and not expired. +// Verify checks that sig is the signature of the request expiring at expires, +// and that expires has not passed. // Signatures are exact-match only: every component of the signed data // (host, path, query, dimensions, format, expiration, quality, fit) must // match exactly. // No suffix matching, wildcard matching, or partial matching is supported. // A signature for "cdn.example.com" will NOT verify for "example.com" or // "other.cdn.example.com", and vice versa. -func (s *Signer) Verify(req *imagerequest.Request) error { +func (s *Signer) Verify( + req *imagerequest.Request, sig string, expires time.Time, +) error { // Check expiration first - if req.Expires.IsZero() { + if expires.IsZero() { return ErrMissingExpiration } - if time.Now().After(req.Expires) { + if time.Now().After(expires) { return ErrExpired } // Compute expected signature - expected := s.Sign(req) + expected := s.Sign(req, expires) // Constant-time comparison to prevent timing attacks - if !hmac.Equal([]byte(req.Signature), []byte(expected)) { + if !hmac.Equal([]byte(sig), []byte(expected)) { return ErrInvalid } @@ -85,7 +88,9 @@ func (s *Signer) Verify(req *imagerequest.Request) error { // expansion is performed. Quality and fit are the effective transform // values, so replaying a signed URL with a different quality or fit mode // fails verification. -func (s *Signer) buildSignatureData(req *imagerequest.Request) string { +func (s *Signer) buildSignatureData( + req *imagerequest.Request, expires time.Time, +) string { return fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%d:%s", req.Source.Host, escapePercentAndColon(req.Source.Path), @@ -93,7 +98,7 @@ func (s *Signer) buildSignatureData(req *imagerequest.Request) string { req.Transform.Size.Width, req.Transform.Size.Height, req.Transform.Format, - req.Expires.Unix(), + expires.Unix(), req.Transform.Quality, req.Transform.FitMode, ) diff --git a/internal/signature/signature_test.go b/internal/signature/signature_test.go index dfe08a2..b8b2c75 100644 --- a/internal/signature/signature_test.go +++ b/internal/signature/signature_test.go @@ -34,11 +34,11 @@ func TestSigner_Sign(t *testing.T) { Size: imagerequest.Size{Width: 800, Height: 600}, Format: testFormatWebP, }, - Expires: time.Unix(1704067200, 0), // Fixed timestamp for reproducibility } + expires := time.Unix(1704067200, 0) // Fixed timestamp for reproducibility - sig1 := signer.Sign(req) - sig2 := signer.Sign(req) + sig1 := signer.Sign(req, expires) + sig2 := signer.Sign(req, expires) // Same input should produce same signature if sig1 != sig2 { @@ -62,18 +62,17 @@ func TestSigner_Sign(t *testing.T) { Size: imagerequest.Size{Width: 800, Height: 600}, Format: testFormatWebP, }, - Expires: time.Unix(1704067200, 0), } - sig3 := signer.Sign(req2) + sig3 := signer.Sign(req2, expires) if sig1 == sig3 { t.Error("Sign() produced same signature for different input") } } -// validVerifyRequest returns a fully-populated request that verifies -// successfully once signed. -func validVerifyRequest() *imagerequest.Request { +// validVerifyRequest returns a fully-populated request, and an expiry an hour +// ahead, that verify successfully once signed. +func validVerifyRequest() (*imagerequest.Request, time.Time) { return &imagerequest.Request{ Source: imagerequest.Source{ Host: testHost, @@ -83,13 +82,13 @@ func validVerifyRequest() *imagerequest.Request { Size: imagerequest.Size{Width: 800, Height: 600}, Format: testFormatWebP, }, - Expires: time.Now().Add(1 * time.Hour), - } + }, time.Now().Add(1 * time.Hour) } type verifyCase struct { - name string - setup func() *imagerequest.Request + name string + // setup returns the request, the signature and the expiry to verify + setup func() (*imagerequest.Request, string, time.Time) wantErr error } @@ -97,54 +96,49 @@ func verifyCases(signer *signature.Signer) []verifyCase { return []verifyCase{ { name: "valid signature", - setup: func() *imagerequest.Request { - req := validVerifyRequest() - req.Signature = signer.Sign(req) + setup: func() (*imagerequest.Request, string, time.Time) { + req, expires := validVerifyRequest() - return req + return req, signer.Sign(req, expires), expires }, wantErr: nil, }, { name: "expired signature", - setup: func() *imagerequest.Request { - req := validVerifyRequest() - req.Expires = time.Now().Add(-1 * time.Hour) - req.Signature = signer.Sign(req) + setup: func() (*imagerequest.Request, string, time.Time) { + req, _ := validVerifyRequest() + expires := time.Now().Add(-1 * time.Hour) - return req + return req, signer.Sign(req, expires), expires }, wantErr: signature.ErrExpired, }, { name: "invalid signature", - setup: func() *imagerequest.Request { - req := validVerifyRequest() - req.Signature = "invalid-signature" + setup: func() (*imagerequest.Request, string, time.Time) { + req, expires := validVerifyRequest() - return req + return req, "invalid-signature", expires }, wantErr: signature.ErrInvalid, }, { name: "missing expiration", - setup: func() *imagerequest.Request { - req := validVerifyRequest() - req.Expires = time.Time{} - req.Signature = "some-signature" + setup: func() (*imagerequest.Request, string, time.Time) { + req, _ := validVerifyRequest() - return req + return req, "some-signature", time.Time{} }, wantErr: signature.ErrMissingExpiration, }, { name: "tampered request", - setup: func() *imagerequest.Request { - req := validVerifyRequest() - req.Signature = signer.Sign(req) + setup: func() (*imagerequest.Request, string, time.Time) { + req, expires := validVerifyRequest() + sig := signer.Sign(req, expires) req.Source.Path = "/photos/secret.jpg" - return req + return req, sig, expires }, wantErr: signature.ErrInvalid, }, @@ -160,8 +154,8 @@ func TestSigner_Verify(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - req := tt.setup() - err := signer.Verify(req) + req, sig, expires := tt.setup() + err := signer.Verify(req, sig, expires) if tt.wantErr == nil { if err != nil { @@ -254,8 +248,9 @@ func TestSigner_Verify_ExactMatchOnly(t *testing.T) { signer := signature.New("test-secret-key") - // Base request that we'll sign, then tamper with individual fields. - baseReq := func() *imagerequest.Request { + // Base request that we'll sign, then tamper with individual fields; it + // is returned with its signature and expiry. + baseReq := func() (*imagerequest.Request, string, time.Time) { req := &imagerequest.Request{ Source: imagerequest.Source{ Host: testHost, @@ -266,21 +261,20 @@ func TestSigner_Verify_ExactMatchOnly(t *testing.T) { Size: imagerequest.Size{Width: 800, Height: 600}, Format: testFormatWebP, }, - Expires: time.Now().Add(1 * time.Hour), } - req.Signature = signer.Sign(req) + expires := time.Now().Add(1 * time.Hour) - return req + return req, signer.Sign(req, expires), expires } for _, tt := range exactMatchTamperCases() { t.Run(tt.name, func(t *testing.T) { t.Parallel() - req := baseReq() + req, sig, expires := baseReq() tt.tamper(req) - err := signer.Verify(req) + err := signer.Verify(req, sig, expires) if !errors.Is(err, signature.ErrInvalid) { t.Errorf("Verify() = %v, want %v", err, signature.ErrInvalid) } @@ -291,9 +285,9 @@ func TestSigner_Verify_ExactMatchOnly(t *testing.T) { t.Run("unmodified request passes", func(t *testing.T) { t.Parallel() - req := baseReq() + req, sig, expires := baseReq() - err := signer.Verify(req) + err := signer.Verify(req, sig, expires) if err != nil { t.Errorf("Verify() unmodified request failed: %v", err) } @@ -329,10 +323,9 @@ func TestSigner_Sign_ExactHostInData(t *testing.T) { Size: imagerequest.Size{Width: 800, Height: 600}, Format: testFormatWebP, }, - Expires: time.Unix(1704067200, 0), } - sig := signer.Sign(req) + sig := signer.Sign(req, time.Unix(1704067200, 0)) if existing, ok := sigs[sig]; ok { t.Errorf("hosts %q and %q produced the same signature", existing, host) } @@ -356,20 +349,20 @@ func TestSigner_DifferentKeys(t *testing.T) { Size: imagerequest.Size{Width: 800, Height: 600}, Format: testFormatWebP, }, - Expires: time.Now().Add(1 * time.Hour), } + expires := time.Now().Add(1 * time.Hour) // Sign with key 1 - req.Signature = signer1.Sign(req) + sig := signer1.Sign(req, expires) // Verify with key 1 should succeed - err := signer1.Verify(req) + err := signer1.Verify(req, sig, expires) if err != nil { t.Errorf("Verify() with same key failed: %v", err) } // Verify with key 2 should fail - err = signer2.Verify(req) + err = signer2.Verify(req, sig, expires) if !errors.Is(err, signature.ErrInvalid) { t.Errorf("Verify() with different key should fail, got: %v", err) } @@ -395,12 +388,11 @@ func TestSign_ReadmeQueryExample(t *testing.T) { Quality: 85, FitMode: testFitCover, }, - Expires: time.Unix(1704067200, 0), } const wantSig = "37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=" - gotSig := signer.Sign(&req) + gotSig := signer.Sign(&req, time.Unix(1704067200, 0)) if gotSig != wantSig { t.Errorf("Sign() = %q, want %q", gotSig, wantSig) } From 77d55abb7e66d45aa1acc980f59dc60a19967136 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 14:10:20 +0100 Subject: [PATCH 105/131] Derive every key from the signing key in one place (closes #315) seal.DeriveKeys derives the session hash and block keys, the encrypted URL key and the CSRF key from the signing key, with their salts declared beside it, and returns them as one seal.Keys. internal/providers calls it once at startup and provides the value to the fx graph. The session manager, the encrypted URL generator and the CSRF protection each take their own key. session.NewManager, encurl.NewGenerator, newCSRFProtect and handlers.New return no error, as nothing in them can fail now. Each key is derived as before, with seal.DeriveKey and the same salt. The signing key itself stays the HMAC key of signed URLs and the login password. README.md says under signing_key what it protects. Model: opus-5-5 --- README.md | 12 +++- TODO.md | 11 ++++ internal/encurl/encurl.go | 16 ++---- internal/encurl/encurl_test.go | 28 ++++++--- internal/handlers/auth_csrf_internal_test.go | 21 ++----- .../handlers/auth_login_log_internal_test.go | 7 ++- .../handlers/auth_session_internal_test.go | 8 ++- internal/handlers/bodylimit_internal_test.go | 21 ++----- internal/handlers/cache_hit_internal_test.go | 7 ++- internal/handlers/csrf.go | 20 ++----- .../handlers/first_bytes_internal_test.go | 7 ++- internal/handlers/handlers.go | 13 ++--- .../image_cache_control_internal_test.go | 7 ++- internal/handlers/imageenc_internal_test.go | 7 ++- .../handlers/imageenc_token_internal_test.go | 7 ++- .../referer_blocklist_internal_test.go | 7 ++- internal/providers/providers.go | 30 ++++++---- internal/seal/keys.go | 57 +++++++++++++++++++ internal/seal/keys_test.go | 42 ++++++++++++++ .../server/computed_side_internal_test.go | 7 ++- internal/server/fit_outside_internal_test.go | 7 ++- internal/server/source_url_internal_test.go | 7 ++- internal/server/svg_as_avif_internal_test.go | 7 ++- internal/session/session.go | 37 +++--------- .../session/session_cookie_attributes_test.go | 7 ++- internal/session/session_logout_test.go | 19 +++++-- internal/session/session_test.go | 28 ++++++--- 27 files changed, 295 insertions(+), 152 deletions(-) create mode 100644 internal/seal/keys.go create mode 100644 internal/seal/keys_test.go diff --git a/README.md b/README.md index d2170b9..787d958 100644 --- a/README.md +++ b/README.md @@ -765,7 +765,17 @@ Key settings in more detail: duration; default `60s`. The upstream fetch counts toward it, and so do the waits for an upstream connection and for a processing slot (up to 10 seconds each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds -- `signing_key` — HMAC secret for URL signatures +- `signing_key` — the secret that protects signed URLs, encrypted URLs and + logins. pixa uses it as it is for two things: + - the HMAC key of signed URLs (see Signature Specification) + - the password of the login page + + pixa derives every other key it uses from it at startup, with HKDF-SHA256: + - a key that signs the session cookie + - a key that encrypts the session cookie + - a key that encrypts the token of an encrypted URL + - a key that signs the CSRF token cookie of the login and generator pages + - `db_url` — the SQLite database to open; omitted, it is `file:/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the database in WAL mode. `state_dir` is percent-encoded in it, so a `%`, `?` or diff --git a/TODO.md b/TODO.md index f02b7c0..85667d1 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 every key derived from the signing key is derived in one place + (closes #315): `seal.DeriveKeys` returns them as one `seal.Keys`, with their + salts declared beside it. `internal/providers` calls it once at startup. Each + key is derived as before, with `seal.DeriveKey` and the same salt. + - The session manager, the encrypted URL generator and the CSRF protection + each take their own key instead of the signing key. + - `session.NewManager`, `encurl.NewGenerator`, `newCSRFProtect` and + `handlers.New` return no error, as nothing in them can fail now. + - The signing key itself stays the HMAC key of signed URLs and the login + password. + - `README.md` says under `signing_key` what it protects. - 2026-10-26 the image request holds only what names the cached object (closes #314): its source and its transform, which the variant key covers. The signed URL format, the signing algorithm and every route's answers are unchanged. diff --git a/internal/encurl/encurl.go b/internal/encurl/encurl.go index 2f06726..9d67d35 100644 --- a/internal/encurl/encurl.go +++ b/internal/encurl/encurl.go @@ -16,9 +16,6 @@ const ( DefaultQuality = 85 DefaultFormat = imagerequest.FormatJXL DefaultFitMode = imagerequest.FitCover - - // HKDF salt for URL encryption key derivation - urlKeySalt = "pixa-urlenc-v1" ) // Errors returned by encurl operations. @@ -48,15 +45,10 @@ type Generator struct { key [seal.KeySize]byte } -// NewGenerator creates an encrypted URL generator with a key derived -// from the signing key. -func NewGenerator(signingKey string) (*Generator, error) { - key, err := seal.DeriveKey([]byte(signingKey), urlKeySalt) - if err != nil { - return nil, err - } - - return &Generator{key: key}, nil +// NewGenerator creates an encrypted URL generator that encrypts its tokens +// with key. +func NewGenerator(key [seal.KeySize]byte) *Generator { + return &Generator{key: key} } // Generate creates an encrypted URL token from the request's source and diff --git a/internal/encurl/encurl_test.go b/internal/encurl/encurl_test.go index 9ce3100..7b19068 100644 --- a/internal/encurl/encurl_test.go +++ b/internal/encurl/encurl_test.go @@ -7,6 +7,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" ) // Shared test fixture strings. @@ -19,11 +20,13 @@ const ( func TestGenerator_GenerateAndParse(t *testing.T) { t.Parallel() - gen, err := encurl.NewGenerator("test-signing-key-12345") + keys, err := seal.DeriveKeys("test-signing-key-12345") if err != nil { - t.Fatalf("NewGenerator() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + gen := encurl.NewGenerator(keys.EncryptedURLKey) + req := &imagerequest.Request{ Source: imagerequest.Source{ Host: testSourceHost, @@ -99,7 +102,8 @@ func TestGenerator_GenerateAndParse(t *testing.T) { func TestGenerator_Parse_Expired(t *testing.T) { t.Parallel() - gen, _ := encurl.NewGenerator("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + gen := encurl.NewGenerator(keys.EncryptedURLKey) req := &imagerequest.Request{ Source: imagerequest.Source{ @@ -126,7 +130,8 @@ func TestGenerator_Parse_Expired(t *testing.T) { func TestGenerator_Parse_InvalidToken(t *testing.T) { t.Parallel() - gen, _ := encurl.NewGenerator("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + gen := encurl.NewGenerator(keys.EncryptedURLKey) _, _, err := gen.Parse("not-a-valid-token") if err == nil { @@ -137,7 +142,8 @@ func TestGenerator_Parse_InvalidToken(t *testing.T) { func TestGenerator_Parse_TamperedToken(t *testing.T) { t.Parallel() - gen, _ := encurl.NewGenerator("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + gen := encurl.NewGenerator(keys.EncryptedURLKey) req := &imagerequest.Request{ Source: imagerequest.Source{ @@ -163,8 +169,10 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) { func TestGenerator_Parse_WrongKey(t *testing.T) { t.Parallel() - gen1, _ := encurl.NewGenerator("signing-key-1") - gen2, _ := encurl.NewGenerator("signing-key-2") + keys1, _ := seal.DeriveKeys("signing-key-1") + keys2, _ := seal.DeriveKeys("signing-key-2") + gen1 := encurl.NewGenerator(keys1.EncryptedURLKey) + gen2 := encurl.NewGenerator(keys2.EncryptedURLKey) req := &imagerequest.Request{ Source: imagerequest.Source{ @@ -184,7 +192,8 @@ func TestGenerator_Parse_WrongKey(t *testing.T) { func TestGenerator_Parse_Defaults(t *testing.T) { t.Parallel() - gen, _ := encurl.NewGenerator("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + gen := encurl.NewGenerator(keys.EncryptedURLKey) // Request with only required fields - should get defaults token, err := gen.Generate(&imagerequest.Request{ @@ -221,7 +230,8 @@ func TestGenerator_Parse_Defaults(t *testing.T) { func TestGenerator_TokenIsURLSafe(t *testing.T) { t.Parallel() - gen, _ := encurl.NewGenerator("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + gen := encurl.NewGenerator(keys.EncryptedURLKey) req := &imagerequest.Request{ Source: imagerequest.Source{ diff --git a/internal/handlers/auth_csrf_internal_test.go b/internal/handlers/auth_csrf_internal_test.go index 161253b..6237798 100644 --- a/internal/handlers/auth_csrf_internal_test.go +++ b/internal/handlers/auth_csrf_internal_test.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -44,27 +45,17 @@ func newCSRFTestRouter(t *testing.T) (*Handlers, http.Handler) { cfg := &config.Config{SigningKey: testSigningKey, Debug: true} - sessMgr, err := session.NewManager(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("session.NewManager() error = %v", err) - } - - encGen, err := encurl.NewGenerator(testSigningKey) - if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) - } - - protect, err := newCSRFProtect(testSigningKey, cfg.Debug) - if err != nil { - t.Fatalf("newCSRFProtect() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } h := &Handlers{ log: slog.New(slog.DiscardHandler), config: cfg, - sessMgr: sessMgr, - encGen: encGen, - csrfProtect: protect, + sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), + encGen: encurl.NewGenerator(keys.EncryptedURLKey), + csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug), } r := chi.NewRouter() diff --git a/internal/handlers/auth_login_log_internal_test.go b/internal/handlers/auth_login_log_internal_test.go index 2c8e49f..71943a8 100644 --- a/internal/handlers/auth_login_log_internal_test.go +++ b/internal/handlers/auth_login_log_internal_test.go @@ -10,6 +10,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/config" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -22,15 +23,15 @@ func TestLoginLogLeavesOutSubmittedKey(t *testing.T) { var buf bytes.Buffer - sessMgr, err := session.NewManager(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("session.NewManager() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } h := &Handlers{ log: slog.New(slog.NewJSONHandler(&buf, nil)), config: &config.Config{SigningKey: testSigningKey}, - sessMgr: sessMgr, + sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), } submittedKeys := []string{wrongKey, testSigningKey} diff --git a/internal/handlers/auth_session_internal_test.go b/internal/handlers/auth_session_internal_test.go index d99ca86..8920ae5 100644 --- a/internal/handlers/auth_session_internal_test.go +++ b/internal/handlers/auth_session_internal_test.go @@ -11,6 +11,7 @@ import ( "time" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -232,8 +233,13 @@ func TestHandleLogout_SaveFails_Answers500(t *testing.T) { db := setupTestDB(t) + keys, err := seal.DeriveKeys(testSigningKey) + if err != nil { + t.Fatalf("seal.DeriveKeys() error = %v", err) + } + sessMgr, err := session.NewManagerWithDatabase( - t.Context(), testSigningKey, db) + t.Context(), keys.SessionHashKey, keys.SessionBlockKey, db) if err != nil { t.Fatalf("session.NewManagerWithDatabase() error = %v", err) } diff --git a/internal/handlers/bodylimit_internal_test.go b/internal/handlers/bodylimit_internal_test.go index 59e5fd7..496eccd 100644 --- a/internal/handlers/bodylimit_internal_test.go +++ b/internal/handlers/bodylimit_internal_test.go @@ -11,6 +11,7 @@ import ( "sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -32,27 +33,17 @@ func newBodyLimitTestRouter( cfg := &config.Config{SigningKey: testSigningKey, Debug: true} - sessMgr, err := session.NewManager(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("session.NewManager() error = %v", err) - } - - encGen, err := encurl.NewGenerator(testSigningKey) - if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) - } - - protect, err := newCSRFProtect(testSigningKey, cfg.Debug) - if err != nil { - t.Fatalf("newCSRFProtect() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } h := &Handlers{ log: slog.New(slog.DiscardHandler), config: cfg, - sessMgr: sessMgr, - encGen: encGen, - csrfProtect: protect, + sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), + encGen: encurl.NewGenerator(keys.EncryptedURLKey), + csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug), } r := chi.NewRouter() diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go index bbc16e1..f62a2b0 100644 --- a/internal/handlers/cache_hit_internal_test.go +++ b/internal/handlers/cache_hit_internal_test.go @@ -25,6 +25,7 @@ import ( "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/signature" ) @@ -146,11 +147,13 @@ func newCountingDatabaseServer( tb.Fatalf("imgcache.NewService() error = %v", err) } - encGen, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - tb.Fatalf("encurl.NewGenerator() error = %v", err) + tb.Fatalf("seal.DeriveKeys() error = %v", err) } + encGen := encurl.NewGenerator(keys.EncryptedURLKey) + token, err := encGen.Generate(&imagerequest.Request{ Source: imagerequest.Source{Host: countingHost, Path: photoPath}, Transform: imagerequest.Transform{ diff --git a/internal/handlers/csrf.go b/internal/handlers/csrf.go index 166abdc..03e4de7 100644 --- a/internal/handlers/csrf.go +++ b/internal/handlers/csrf.go @@ -9,15 +9,10 @@ import ( "sneak.berlin/go/pixa/internal/seal" ) -// csrfKeySalt provides domain separation for the CSRF authentication key, -// derived from the signing key so tokens survive restarts without extra -// configuration and never reuse the session or encrypted-URL key material. -const csrfKeySalt = "pixa-csrf-v1" - // newCSRFProtect builds the CSRF-protection middleware for the // state-mutating HTML form routes. The token lives in its own cookie, // independent of the session cookie, so it also protects the login POST -// where no session exists yet (login CSRF). +// where no session exists yet (login CSRF). key signs that cookie. // // When plaintext is true (local HTTP development), requests are marked // plaintext so the library neither demands an https Referer nor sets a @@ -25,13 +20,8 @@ const csrfKeySalt = "pixa-csrf-v1" // service runs behind a TLS-terminating proxy, so plaintext is false and // the library enforces its https Referer origin check. func newCSRFProtect( - signingKey string, plaintext bool, -) (func(http.Handler) http.Handler, error) { - key, err := seal.DeriveKey([]byte(signingKey), csrfKeySalt) - if err != nil { - return nil, err - } - + key [seal.KeySize]byte, plaintext bool, +) func(http.Handler) http.Handler { protect := csrf.Protect( key[:], csrf.Path("/"), @@ -40,7 +30,7 @@ func newCSRFProtect( ) if !plaintext { - return protect, nil + return protect } return func(next http.Handler) http.Handler { @@ -49,7 +39,7 @@ func newCSRFProtect( return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { protected.ServeHTTP(w, csrf.PlaintextHTTPRequest(r)) }) - }, nil + } } // CSRF returns the CSRF-protection middleware for the login and generator diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go index 69a3751..3a25b79 100644 --- a/internal/handlers/first_bytes_internal_test.go +++ b/internal/handlers/first_bytes_internal_test.go @@ -19,6 +19,7 @@ import ( "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/signature" ) @@ -60,11 +61,13 @@ func newRoutesFetchingThrough( t.Fatalf("imgcache.NewService() error = %v", err) } - encGen, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + encGen := encurl.NewGenerator(keys.EncryptedURLKey) + h := &Handlers{ log: log, imgSvc: svc, diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index db24c4f..50c4f1e 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -15,6 +15,7 @@ import ( "sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/signature" ) @@ -26,6 +27,7 @@ type Params struct { Logger *logger.Logger Healthcheck *healthcheck.Healthcheck Config *config.Config + Keys seal.Keys ImageService *imgcache.Service SessionManager *session.Manager @@ -53,12 +55,7 @@ type Handlers struct { } // New creates a new Handlers instance. -func New(_ fx.Lifecycle, params Params) (*Handlers, error) { - csrfProtect, err := newCSRFProtect(params.Config.SigningKey, params.Config.Debug) - if err != nil { - return nil, err - } - +func New(_ fx.Lifecycle, params Params) *Handlers { return &Handlers{ log: params.Logger.Get(), hc: params.Healthcheck, @@ -66,11 +63,11 @@ func New(_ fx.Lifecycle, params Params) (*Handlers, error) { imgSvc: params.ImageService, sessMgr: params.SessionManager, encGen: params.EncryptedURLGenerator, - csrfProtect: csrfProtect, + csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug), signer: signature.New(params.Config.SigningKey), allowlist: allowlist.New(params.Config.AllowlistHosts), refererBlocklist: allowlist.New(params.Config.RefererBlocklist), - }, nil + } } // WaitForProcessing waits until no image is being processed, or until ctx diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index 7059ac7..a561c8d 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -19,6 +19,7 @@ import ( "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/signature" ) @@ -61,11 +62,13 @@ func newSignedHostServer( t.Fatalf("imgcache.NewService() error = %v", err) } - encGen, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + encGen := encurl.NewGenerator(keys.EncryptedURLKey) + h := &Handlers{ log: log, imgSvc: svc, diff --git a/internal/handlers/imageenc_internal_test.go b/internal/handlers/imageenc_internal_test.go index e939e13..2da0124 100644 --- a/internal/handlers/imageenc_internal_test.go +++ b/internal/handlers/imageenc_internal_test.go @@ -12,6 +12,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" ) // newEncTestServer builds a router serving the encrypted-URL route with a @@ -21,11 +22,13 @@ import ( func newEncTestServer(t *testing.T) (*encurl.Generator, http.Handler) { t.Helper() - encGen, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + encGen := encurl.NewGenerator(keys.EncryptedURLKey) + h := &Handlers{ log: slog.New(slog.DiscardHandler), encGen: encGen, diff --git a/internal/handlers/imageenc_token_internal_test.go b/internal/handlers/imageenc_token_internal_test.go index 920cc12..09e449f 100644 --- a/internal/handlers/imageenc_token_internal_test.go +++ b/internal/handlers/imageenc_token_internal_test.go @@ -10,6 +10,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" ) // requireServedPhoto requires that rec answers 200 with the JPEG at photoPath @@ -80,11 +81,13 @@ func TestHandleImageEnc_RejectedToken(t *testing.T) { t.Fatalf("Generate() error = %v", err) } - otherGen, err := encurl.NewGenerator("another-signing-key-fedcba9876543210") + keys, err := seal.DeriveKeys("another-signing-key-fedcba9876543210") if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + otherGen := encurl.NewGenerator(keys.EncryptedURLKey) + otherKey, err := otherGen.Generate(&photo, time.Time{}) if err != nil { t.Fatalf("Generate() error = %v", err) diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index a81a7f5..51fa8ec 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -17,6 +17,7 @@ import ( "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/signature" ) @@ -73,11 +74,13 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) t.Fatalf("imgcache.NewService() error = %v", err) } - encGen, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + encGen := encurl.NewGenerator(keys.EncryptedURLKey) + h := &Handlers{ log: slog.New(slog.DiscardHandler), imgSvc: svc, diff --git a/internal/providers/providers.go b/internal/providers/providers.go index 7d2a951..2a11e61 100644 --- a/internal/providers/providers.go +++ b/internal/providers/providers.go @@ -1,5 +1,6 @@ // Package providers builds from the config the parts pixad serves images // and logins with, and provides them to the fx graph: +// - the keys derived from the signing key // - the image cache, with its default limit // - the fetcher // - the processor @@ -27,6 +28,7 @@ import ( "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/logger" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -35,6 +37,7 @@ import ( // the cache registers its counters on. func Constructors() []any { return []any{ + newKeys, newCache, newFetcherConfig, httpfetcher.New, @@ -45,6 +48,12 @@ func Constructors() []any { } } +// newKeys derives the keys of seal.Keys from the signing key. fx calls it +// once, at startup. +func newKeys(cfg *config.Config) (seal.Keys, error) { + return seal.DeriveKeys(cfg.SigningKey) +} + // cacheParams defines dependencies for newCache. type cacheParams struct { fx.In @@ -229,22 +238,23 @@ func newImageService( type sessionManagerParams struct { fx.In - Config *config.Config + Keys seal.Keys Database *database.Database } -// newSessionManager builds the session manager. Session cookies are -// always Secure, HttpOnly and SameSite=Strict. The random value that each -// logout replaces is saved in the database, so it outlasts a restart. +// newSessionManager builds the session manager with its two keys. Session +// cookies are always Secure, HttpOnly and SameSite=Strict. The random value +// that each logout replaces is saved in the database, so it outlasts a +// restart. func newSessionManager( _ fx.Lifecycle, params sessionManagerParams, ) (*session.Manager, error) { - return session.NewManagerWithDatabase( - context.Background(), params.Config.SigningKey, params.Database.DB()) + return session.NewManagerWithDatabase(context.Background(), + params.Keys.SessionHashKey, params.Keys.SessionBlockKey, + params.Database.DB()) } -// newEncryptedURLGenerator builds the encrypted URL generator, with a key -// derived from the signing key. -func newEncryptedURLGenerator(cfg *config.Config) (*encurl.Generator, error) { - return encurl.NewGenerator(cfg.SigningKey) +// newEncryptedURLGenerator builds the encrypted URL generator with its key. +func newEncryptedURLGenerator(keys seal.Keys) *encurl.Generator { + return encurl.NewGenerator(keys.EncryptedURLKey) } diff --git a/internal/seal/keys.go b/internal/seal/keys.go new file mode 100644 index 0000000..b84b416 --- /dev/null +++ b/internal/seal/keys.go @@ -0,0 +1,57 @@ +package seal + +// The salt of each key that DeriveKeys derives. Each key has a salt of its +// own, so no two uses share a key. +const ( + sessionHashKeySalt = "pixa-session-hash-v1" + sessionBlockKeySalt = "pixa-session-block-v1" + encryptedURLKeySalt = "pixa-urlenc-v1" + csrfKeySalt = "pixa-csrf-v1" +) + +// Keys holds the keys derived from the signing key, one for each use. The +// signing key itself is used for two things only: it is the HMAC key of signed +// URLs and the password of the login page. +type Keys struct { + // SessionHashKey signs the session cookie. + SessionHashKey [KeySize]byte + // SessionBlockKey encrypts the session cookie. + SessionBlockKey [KeySize]byte + // EncryptedURLKey encrypts the token of an encrypted URL. + EncryptedURLKey [KeySize]byte + // CSRFKey signs the CSRF token cookie of the login and generator pages. + CSRFKey [KeySize]byte +} + +// DeriveKeys derives each key of Keys from signingKey, with DeriveKey and the +// salt of that key. +func DeriveKeys(signingKey string) (Keys, error) { + masterKey := []byte(signingKey) + + sessionHashKey, err := DeriveKey(masterKey, sessionHashKeySalt) + if err != nil { + return Keys{}, err + } + + sessionBlockKey, err := DeriveKey(masterKey, sessionBlockKeySalt) + if err != nil { + return Keys{}, err + } + + encryptedURLKey, err := DeriveKey(masterKey, encryptedURLKeySalt) + if err != nil { + return Keys{}, err + } + + csrfKey, err := DeriveKey(masterKey, csrfKeySalt) + if err != nil { + return Keys{}, err + } + + return Keys{ + SessionHashKey: sessionHashKey, + SessionBlockKey: sessionBlockKey, + EncryptedURLKey: encryptedURLKey, + CSRFKey: csrfKey, + }, nil +} diff --git a/internal/seal/keys_test.go b/internal/seal/keys_test.go new file mode 100644 index 0000000..33a6183 --- /dev/null +++ b/internal/seal/keys_test.go @@ -0,0 +1,42 @@ +package seal_test + +import ( + "bytes" + "testing" + + "sneak.berlin/go/pixa/internal/seal" +) + +// TestDeriveKeys_KeysDiffer verifies that the keys DeriveKeys derives differ +// from each other and from the signing key. +func TestDeriveKeys_KeysDiffer(t *testing.T) { + t.Parallel() + + // 32 bytes, as long as a derived key, so that no key differs from it by + // its length alone + const signingKey = "0123456789abcdef0123456789abcdef" + + keys, err := seal.DeriveKeys(signingKey) + if err != nil { + t.Fatalf("DeriveKeys() error = %v", err) + } + + named := []struct { + name string + key []byte + }{ + {"the signing key", []byte(signingKey)}, + {"SessionHashKey", keys.SessionHashKey[:]}, + {"SessionBlockKey", keys.SessionBlockKey[:]}, + {"EncryptedURLKey", keys.EncryptedURLKey[:]}, + {"CSRFKey", keys.CSRFKey[:]}, + } + + for i, a := range named { + for _, b := range named[i+1:] { + if bytes.Equal(a.key, b.key) { + t.Errorf("%s equals %s", a.name, b.name) + } + } + } +} diff --git a/internal/server/computed_side_internal_test.go b/internal/server/computed_side_internal_test.go index 95fb696..157f0c3 100644 --- a/internal/server/computed_side_internal_test.go +++ b/internal/server/computed_side_internal_test.go @@ -11,6 +11,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" ) // TestImageProxyRefusesComputedSideAboveMaximum requests sizes of a 3000x600 @@ -48,11 +49,13 @@ func TestImageProxyRefusesComputedSideAboveMaximum(t *testing.T) { s, _, _ := startImageProxy(t, upstream) - generator, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator: %v", err) + t.Fatalf("seal.DeriveKeys: %v", err) } + generator := encurl.NewGenerator(keys.EncryptedURLKey) + tests := []struct { path string // size is the size of the /v1/image/ URL, and width and height are diff --git a/internal/server/fit_outside_internal_test.go b/internal/server/fit_outside_internal_test.go index 1c095fd..55e3acd 100644 --- a/internal/server/fit_outside_internal_test.go +++ b/internal/server/fit_outside_internal_test.go @@ -9,6 +9,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" ) // TestImageProxyRefusesFitOutsideAboveMaximum requests sizes with fit=outside @@ -42,11 +43,13 @@ func TestImageProxyRefusesFitOutsideAboveMaximum(t *testing.T) { s, _, _ := startImageProxy(t, upstream) - generator, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator: %v", err) + t.Fatalf("seal.DeriveKeys: %v", err) } + generator := encurl.NewGenerator(keys.EncryptedURLKey) + tests := []struct { path string width, height int diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index 68676d8..f897262 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -16,6 +16,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/signature" ) @@ -540,11 +541,13 @@ func signedImageURL(t *testing.T, sourceHost string) string { func encryptedImageURL(t *testing.T, sourceHost string) string { t.Helper() - generator, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator: %v", err) + t.Fatalf("seal.DeriveKeys: %v", err) } + generator := encurl.NewGenerator(keys.EncryptedURLKey) + token, err := generator.Generate(&imagerequest.Request{ Source: imagerequest.Source{Host: sourceHost, Path: ipv6SourcePath}, Transform: imagerequest.Transform{Format: imagerequest.FormatPNG}, diff --git a/internal/server/svg_as_avif_internal_test.go b/internal/server/svg_as_avif_internal_test.go index 0993c16..b1ce81d 100644 --- a/internal/server/svg_as_avif_internal_test.go +++ b/internal/server/svg_as_avif_internal_test.go @@ -10,6 +10,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/seal" ) // svgServedAsAVIF is an SVG whose first bytes pass the first-bytes check for @@ -34,11 +35,13 @@ func TestImageProxyRefusesSVGServedAsAVIF(t *testing.T) { s, _, _ := startImageProxy(t, upstream) - generator, err := encurl.NewGenerator(testSigningKey) + keys, err := seal.DeriveKeys(testSigningKey) if err != nil { - t.Fatalf("encurl.NewGenerator: %v", err) + t.Fatalf("seal.DeriveKeys: %v", err) } + generator := encurl.NewGenerator(keys.EncryptedURLKey) + encryptedURL := func(format imagerequest.Format) string { token, err := generator.Generate(&imagerequest.Request{ Source: imagerequest.Source{Host: upstreamHost, Path: "/drawing.avif"}, diff --git a/internal/session/session.go b/internal/session/session.go index 788c114..ad1359e 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -22,10 +22,6 @@ import ( const ( CookieName = "pixa_session" SessionTTL = 30 * 24 * time.Hour // 30 days - - // HKDF salts for key derivation - hashKeySalt = "pixa-session-hash-v1" - blockKeySalt = "pixa-session-block-v1" ) // Errors returned by session operations. @@ -66,35 +62,23 @@ type Manager struct { logoutValue string } -// NewManager creates a session manager with keys derived from the signing key. -// It makes a new random value and keeps it in memory only, so a restart ends -// every session; NewManagerWithDatabase keeps the value in the database. +// NewManager creates a session manager whose session cookies hashKey signs +// and blockKey encrypts. It makes a new random value and keeps it in memory +// only, so a restart ends every session; NewManagerWithDatabase keeps the +// value in the database. // // Session cookies always carry the Secure, HttpOnly, and SameSite=Strict // attributes; this cannot be configured. Browsers treat http://localhost as a // trustworthy origin and accept Secure cookies there, so local development // keeps working. -func NewManager(signingKey string) (*Manager, error) { - masterKey := []byte(signingKey) - - // Derive separate keys for HMAC (hash) and encryption (block) - hashKey, err := seal.DeriveKey(masterKey, hashKeySalt) - if err != nil { - return nil, err - } - - blockKey, err := seal.DeriveKey(masterKey, blockKeySalt) - if err != nil { - return nil, err - } - +func NewManager(hashKey, blockKey [seal.KeySize]byte) *Manager { sc := securecookie.New(hashKey[:], blockKey[:]) sc.MaxAge(int(SessionTTL.Seconds())) return &Manager{ sc: sc, logoutValue: rand.Text(), - }, nil + } } // NewManagerWithDatabase creates a session manager like NewManager that keeps @@ -102,14 +86,11 @@ func NewManager(signingKey string) (*Manager, error) { // NewManager made when db has none, and otherwise starts from the one saved // there. func NewManagerWithDatabase( - ctx context.Context, signingKey string, db *sql.DB, + ctx context.Context, hashKey, blockKey [seal.KeySize]byte, db *sql.DB, ) (*Manager, error) { - m, err := NewManager(signingKey) - if err != nil { - return nil, err - } + m := NewManager(hashKey, blockKey) - _, err = db.ExecContext(ctx, + _, err := db.ExecContext(ctx, "INSERT OR IGNORE INTO logout (id, logout_value) VALUES (1, ?)", m.logoutValue) if err != nil { diff --git a/internal/session/session_cookie_attributes_test.go b/internal/session/session_cookie_attributes_test.go index 58a17a6..f016034 100644 --- a/internal/session/session_cookie_attributes_test.go +++ b/internal/session/session_cookie_attributes_test.go @@ -5,6 +5,7 @@ import ( "net/http/httptest" "testing" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -20,11 +21,13 @@ import ( func TestSessionCookieAttributesAlwaysSecure(t *testing.T) { t.Parallel() - mgr, err := session.NewManager("test-signing-key-12345") + keys, err := seal.DeriveKeys("test-signing-key-12345") if err != nil { - t.Fatalf("NewManager() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) + writePaths := []struct { name string setCookie func(t *testing.T, w http.ResponseWriter) diff --git a/internal/session/session_logout_test.go b/internal/session/session_logout_test.go index b86dbb7..604cb15 100644 --- a/internal/session/session_logout_test.go +++ b/internal/session/session_logout_test.go @@ -8,6 +8,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) @@ -80,8 +81,13 @@ func TestLogout_EndsEverySessionMadeBeforeIt(t *testing.T) { db := newTestDatabase(t) + keys, err := seal.DeriveKeys(logoutTestSigningKey) + if err != nil { + t.Fatalf("seal.DeriveKeys() error = %v", err) + } + mgr, err := session.NewManagerWithDatabase( - t.Context(), logoutTestSigningKey, db) + t.Context(), keys.SessionHashKey, keys.SessionBlockKey, db) if err != nil { t.Fatalf("NewManagerWithDatabase() error = %v", err) } @@ -109,7 +115,7 @@ func TestLogout_EndsEverySessionMadeBeforeIt(t *testing.T) { } restarted, err := session.NewManagerWithDatabase( - t.Context(), logoutTestSigningKey, db) + t.Context(), keys.SessionHashKey, keys.SessionBlockKey, db) if err != nil { t.Fatalf("NewManagerWithDatabase() after restart error = %v", err) } @@ -129,8 +135,13 @@ func TestLogout_EndsEverySessionMadeBeforeIt(t *testing.T) { func TestLogout_WithoutSession_EndsNoSession(t *testing.T) { t.Parallel() - mgr, err := session.NewManagerWithDatabase( - t.Context(), logoutTestSigningKey, newTestDatabase(t)) + keys, err := seal.DeriveKeys(logoutTestSigningKey) + if err != nil { + t.Fatalf("seal.DeriveKeys() error = %v", err) + } + + mgr, err := session.NewManagerWithDatabase(t.Context(), + keys.SessionHashKey, keys.SessionBlockKey, newTestDatabase(t)) if err != nil { t.Fatalf("NewManagerWithDatabase() error = %v", err) } diff --git a/internal/session/session_test.go b/internal/session/session_test.go index c0a2822..b994496 100644 --- a/internal/session/session_test.go +++ b/internal/session/session_test.go @@ -7,17 +7,20 @@ import ( "testing" "time" + "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" ) func TestManager_CreateAndValidate(t *testing.T) { t.Parallel() - mgr, err := session.NewManager("test-signing-key-12345") + keys, err := seal.DeriveKeys("test-signing-key-12345") if err != nil { - t.Fatalf("NewManager() error = %v", err) + t.Fatalf("seal.DeriveKeys() error = %v", err) } + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) + // Create a session w := httptest.NewRecorder() @@ -69,7 +72,8 @@ func TestManager_CreateAndValidate(t *testing.T) { func TestManager_ValidateSession_NoCookie(t *testing.T) { t.Parallel() - mgr, _ := session.NewManager("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) @@ -86,7 +90,8 @@ func TestManager_ValidateSession_NoCookie(t *testing.T) { func TestManager_ValidateSession_TamperedCookie(t *testing.T) { t.Parallel() - mgr, _ := session.NewManager("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) req.AddCookie(&http.Cookie{ @@ -110,8 +115,10 @@ func TestManager_ValidateSession_TamperedCookie(t *testing.T) { func TestManager_ValidateSession_WrongKey(t *testing.T) { t.Parallel() - mgr1, _ := session.NewManager("signing-key-1") - mgr2, _ := session.NewManager("signing-key-2") + keys1, _ := seal.DeriveKeys("signing-key-1") + keys2, _ := seal.DeriveKeys("signing-key-2") + mgr1 := session.NewManager(keys1.SessionHashKey, keys1.SessionBlockKey) + mgr2 := session.NewManager(keys2.SessionHashKey, keys2.SessionBlockKey) // Create session with mgr1 w := httptest.NewRecorder() @@ -142,7 +149,8 @@ func TestManager_ValidateSession_WrongKey(t *testing.T) { func TestManager_ClearSession(t *testing.T) { t.Parallel() - mgr, _ := session.NewManager("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) w := httptest.NewRecorder() mgr.ClearSession(w) @@ -172,7 +180,8 @@ func TestManager_ClearSession(t *testing.T) { func TestManager_IsAuthenticated(t *testing.T) { t.Parallel() - mgr, _ := session.NewManager("test-signing-key-12345") + keys, _ := seal.DeriveKeys("test-signing-key-12345") + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) // No session - should return false req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil) @@ -208,7 +217,8 @@ func TestManager_IsAuthenticated(t *testing.T) { func TestManager_CookieAttributes(t *testing.T) { t.Parallel() - mgr, _ := session.NewManager("test-key") + keys, _ := seal.DeriveKeys("test-key") + mgr := session.NewManager(keys.SessionHashKey, keys.SessionBlockKey) w := httptest.NewRecorder() _ = mgr.CreateSession(w) From 87cb1ab5f6d471f92788434c0d96d41c54a7ff51 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 15:24:24 +0100 Subject: [PATCH 106/131] Serve both image routes with one function and one error mapping (closes #316) HandleImage parses the path and verifies the signature, and HandleImageEnc decrypts the token. Each then calls serveImage, which chooses the format for auto, gets the image and serves it. respondImageError answers the image service's errors on both routes, and handleImageError is gone. /v1/image/ now answers 504 when the upstream sends no response headers within upstream_fetch_timeout, as README.md documented for /v1/e/. respondImageError logs a 500 at error level and the other answers at warn. The image service alone logs a source refused for its first bytes. A failed write is logged once, at warn, with the request and the bytes served. README.md lists each image route's answers as bullets. Tests send the same errors through both routes. Model: opus-5-5 --- README.md | 76 +++++--- TODO.md | 30 +++ .../handlers/avif_too_large_internal_test.go | 36 ++-- internal/handlers/image.go | 173 +++++++++--------- .../handlers/image_errors_internal_test.go | 170 +++++++++++++++++ internal/handlers/imageenc.go | 123 +------------ .../handlers/server_busy_internal_test.go | 72 +++----- .../unsupported_source_internal_test.go | 37 ++-- 8 files changed, 401 insertions(+), 316 deletions(-) diff --git a/README.md b/README.md index 787d958..0f3271a 100644 --- a/README.md +++ b/README.md @@ -248,32 +248,58 @@ path under `/v1/` answers 200, in maintenance mode too. - `GET` or `HEAD` `/v1/image///.`, or `/v1/image///` with no format — an image, fetched, resized and converted (below). Needs: a signature, unless the host is allowlisted (see - Source Hosts). Answers: 200; 304 when `If-None-Match` matches the image's - `ETag`; 400 for a URL or parameter that is not valid, or for the format `auto` - an `Accept` header that is not valid; 406 for the format `auto` when `Accept` - allows none of the formats it chooses from; 401 for a missing or wrong - signature, a missing `exp` or an `exp` in the past; 403 when the request's - `Referer` names a host in `referer_blocklist`, checked before the signature, - the cache and the upstream fetch; 403 when the upstream host, or a host it - redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an - address in a blocked network (see `blocked_networks`); 429 over - `image_requests_per_minute` (below); 415 when the upstream answers with a - `Content-Type` that is not one of the source formats (below), with a body - whose first bytes are not those of the format its `Content-Type` names, or - with an image whose first bytes libvips reads as another format; 502 when the - upstream answered with an error status, and for 5 minutes after that, or after - a body whose first bytes are not those of its `Content-Type`, for an image of - the same source URL that pixa can neither serve from its cache nor make from - the source image still in its cache; 503 when pixa is busy or in maintenance - mode; 500 for any other failure. + Source Hosts). Answers: + - 200 with the image. + - 304 when `If-None-Match` matches the image's `ETag`. + - 400 for a URL or parameter that is not valid. + - 400 when the format is `auto` and the `Accept` header is not valid. + - 406 when the format is `auto` and `Accept` allows none of the formats it + chooses from. + - 401 for a missing or wrong signature. + - 401 for a missing `exp`. + - 401 for an `exp` in the past. + - 403 when the request's `Referer` names a host in `referer_blocklist`, + checked before the signature, the cache and the upstream fetch. + - 403 when the upstream host, or a host it redirects to, is `localhost` or + ends in `.localhost` or `.local`. + - 403 when the upstream host, or a host it redirects to, has an address in a + blocked network (see `blocked_networks`). + - 429 over `image_requests_per_minute` (below). + - 415 when the upstream answers with a `Content-Type` that is not one of the + source formats (below). + - 415 when the upstream sends a body whose first bytes are not those of the + format its `Content-Type` names. + - 415 when the upstream sends an image whose first bytes libvips reads as + another format. + - 502 when the upstream answers with an error status. + - 502 for 5 minutes after that for the same source URL, when pixa has + neither the image nor its source image in its cache. + - 502 likewise for 5 minutes after a body whose first bytes are not those of + its `Content-Type`. + - 503 when pixa is busy. + - 503 when pixa is in maintenance mode. + - 504 when the upstream has not sent its response headers within + `upstream_fetch_timeout`. + - 500 when that time runs out while the image itself is still arriving. + - 500 for any other failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL - (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when - `If-None-Match` matches the image's `ETag`; 400 for a token that does not - decrypt, or that asks for a size or fit that is not valid; 410 once it has - expired; 504 when the upstream has not sent its response headers within - `upstream_fetch_timeout`, but 500 when that time runs out while the image - itself is still arriving; 400 for an `Accept` header that is not valid, and - 406, 403, 415, 429, 502, 503 and 500, as for `/v1/image/`. + (see Encrypted URLs). Needs: nothing but the URL. Answers: + - 200 with the image. + - 304 when `If-None-Match` matches the image's `ETag`. + - 400 for a token that does not decrypt. + - 400 for a token that asks for a size or fit that is not valid. + - 410 once the token has expired. + - 400 when the token's format is `auto` and the `Accept` header is not + valid. + - These answers, for the same causes as on `/v1/image/`: + - 406 + - 403 + - 415 + - 429 + - 502 + - 503 + - 504 + - 500 - `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: nothing. Answers: 200. - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, diff --git a/TODO.md b/TODO.md index 85667d1..bb3ec91 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,36 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 both image routes serve an image with one function, `serveImage`, + and answer its errors with one, `respondImageError` (closes #316). + `HandleImage` parses the path and verifies the signature, and `HandleImageEnc` + decrypts the token. Each then calls `serveImage`. + - `handleImageError`, the encrypted route's own mapping, is gone. + - `/v1/image/` answers 504 when the upstream has not sent its response + headers within `upstream_fetch_timeout`, as `/v1/e/` did. It answered 500. + - `respondImageError` logs the image service's errors as "failed to get + image". Each answer gets the same level on `/v1/image/` and on `/v1/e/`: + - 500: error, as pixa's own failure + - 400: warn + - 403: warn + - 415: warn + - 502: warn + - 503: warn + - 504: warn + - Before, `/v1/image/` logged all of these answers at error level. `/v1/e/` + logged only its 500, as "image request failed", without the host and the + path. + - `respondImageError` does not log the 415 for a body refused for its first + bytes, on either route. The image service logs it as "source refused", at + warn level. + - The "image served" line carries the host, the path and `fetched_bytes` on + both routes. The debug line "encrypted image request" is gone. + - The debug line "failed to decrypt URL" is now "invalid encrypted URL". A + token that decrypts to a payload that does not decode also gets it. + - A failed write is logged once, as "failed to write response" at warn + level, since most often the client has gone away. It carries the + `request_id`, the host, the path, `served_bytes` and the error, and is not + also logged as "image served". - 2026-10-26 every key derived from the signing key is derived in one place (closes #315): `seal.DeriveKeys` returns them as one `seal.Keys`, with their salts declared beside it. `internal/providers` calls it once at startup. Each diff --git a/internal/handlers/avif_too_large_internal_test.go b/internal/handlers/avif_too_large_internal_test.go index 1efad83..2febf57 100644 --- a/internal/handlers/avif_too_large_internal_test.go +++ b/internal/handlers/avif_too_large_internal_test.go @@ -12,11 +12,11 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// TestAVIFOutputTooLargeAnswers400 checks that respondImageError and -// handleImageError, which answer the errors of the two image routes, answer -// 400 naming the AVIF pixel limit when the image processor refuses an AVIF -// output above it, wrapped as the service wraps it. The request's source only -// goes into a log line, so it is left empty. +// TestAVIFOutputTooLargeAnswers400 checks that respondImageError, which +// answers the errors of both image routes, answers 400 naming the AVIF pixel +// limit when the image processor refuses an AVIF output above it, wrapped as +// the service wraps it. The request's source only goes into a log line, so it +// is left empty. func TestAVIFOutputTooLargeAnswers400(t *testing.T) { t.Parallel() @@ -25,25 +25,15 @@ func TestAVIFOutputTooLargeAnswers400(t *testing.T) { imageprocessor.ErrAVIFOutputTooLarge) wantBody := fmt.Sprintf("above %d pixels", imageprocessor.MaxAVIFOutputPixels) - plain := httptest.NewRecorder() - h.respondImageError(plain, &imagerequest.Request{}, err) + rec := httptest.NewRecorder() + h.respondImageError(rec, &imagerequest.Request{}, err) + t.Logf("status = %d, body = %q", rec.Code, rec.Body.String()) - encrypted := httptest.NewRecorder() - h.handleImageError(encrypted, err) + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest) + } - for name, rec := range map[string]*httptest.ResponseRecorder{ - "respondImageError": plain, "handleImageError": encrypted, - } { - t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String()) - - if rec.Code != http.StatusBadRequest { - t.Errorf("%s: status = %d, want %d", - name, rec.Code, http.StatusBadRequest) - } - - if !strings.Contains(rec.Body.String(), wantBody) { - t.Errorf("%s: body = %q, want it to say %q", - name, rec.Body.String(), wantBody) - } + if !strings.Contains(rec.Body.String(), wantBody) { + t.Errorf("body = %q, want it to say %q", rec.Body.String(), wantBody) } } diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 12665e1..d349d5b 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -46,30 +46,7 @@ func (s *Handlers) HandleImage() http.HandlerFunc { return } - // The signature covers the format auto, not the format chosen - format, ok := s.chooseAutoFormat(w, r, req.Transform.Format) - if !ok { - return - } - - req.Transform.Format = format - - // Get cache key for logging - cacheKey := imagerequest.CacheKey(req) - - // Get the image (from cache or fetch/process) - startTime := time.Now() - - resp, err := s.imgSvc.Get(r.Context(), req) - if err != nil { - s.respondImageError(w, req, err) - - return - } - - defer func() { _ = resp.Content.Close() }() - - s.writeImageResponse(w, r, req, expires, resp, cacheKey, startTime) + s.serveImage(w, r, req, expires) } } @@ -213,8 +190,8 @@ func (s *Handlers) validateSignature( } // parseExpires reads the exp query parameter, a Unix time in seconds. An exp -// missing from the URL gives the zero time, which the signature check takes -// as no expiration. An exp in the URL that is not a whole number, an empty +// missing from the URL gives the zero time, which the signature check refuses +// as a missing exp. An exp in the URL that is not a whole number, an empty // one included, is an error naming exp and the value. func parseExpires(query url.Values) (time.Time, error) { if !query.Has("exp") { @@ -242,7 +219,11 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc { } } -// respondImageError maps image retrieval errors to HTTP responses. +// respondImageError answers err, the image service's error for req, on both +// image routes. It logs a 500 at error level, as pixa's own failure. It logs +// the other answers at warn level, since the client, the upstream or a full +// limit caused them. The 415 for a source refused for its first bytes is not +// logged here, because the image service logs it. func (s *Handlers) respondImageError( w http.ResponseWriter, req *imagerequest.Request, err error, ) { @@ -258,59 +239,53 @@ func (s *Handlers) respondImageError( return } - s.log.Error("failed to get image", + var ( + status int + message string + ) + + switch { + case errors.Is(err, httpfetcher.ErrSSRFBlocked): + status, message = http.StatusForbidden, "forbidden" + case errors.Is(err, httpfetcher.ErrUpstreamError): + status, message = http.StatusBadGateway, "upstream error" + case errors.Is(err, httpfetcher.ErrUpstreamTimeout): + status, message = http.StatusGatewayTimeout, "upstream timeout" + case errors.Is(err, httpfetcher.ErrTooManyConnections), + errors.Is(err, imageprocessor.ErrTooManyImages), + errors.Is(err, imgcache.ErrProcessingTimedOut): + status = http.StatusServiceUnavailable + message = "server busy, try again later" + case errors.Is(err, httpfetcher.ErrInvalidContentType), + errors.Is(err, imageprocessor.ErrUnsupportedInputFormat): + status = http.StatusUnsupportedMediaType + message = "unsupported source format" + case errors.Is(err, imageprocessor.ErrOutputTooLarge): + status = http.StatusBadRequest + message = fmt.Sprintf("size too large for the source: the side "+ + "computed from its proportions is above %d", + imageprocessor.MaxOutputDimension) + case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge): + status = http.StatusBadRequest + message = fmt.Sprintf("size too large for AVIF: the output "+ + "would be above %d pixels", imageprocessor.MaxAVIFOutputPixels) + default: + s.log.Error("failed to get image", + "host", req.Source.Host, + "path", req.Source.Path, + "error", err, + ) + s.respondError(w, "internal error", http.StatusInternalServerError) + + return + } + + s.log.Warn("failed to get image", "host", req.Source.Host, "path", req.Source.Path, "error", err, ) - - // Check for specific error types - if errors.Is(err, httpfetcher.ErrSSRFBlocked) { - s.respondError(w, "forbidden", http.StatusForbidden) - - return - } - - if errors.Is(err, httpfetcher.ErrUpstreamError) { - s.respondError(w, "upstream error", http.StatusBadGateway) - - return - } - - if errors.Is(err, httpfetcher.ErrTooManyConnections) || - errors.Is(err, imageprocessor.ErrTooManyImages) || - errors.Is(err, imgcache.ErrProcessingTimedOut) { - s.respondError(w, "server busy, try again later", - http.StatusServiceUnavailable) - - return - } - - if errors.Is(err, httpfetcher.ErrInvalidContentType) || - errors.Is(err, imageprocessor.ErrUnsupportedInputFormat) { - s.respondError(w, "unsupported source format", - http.StatusUnsupportedMediaType) - - return - } - - if errors.Is(err, imageprocessor.ErrOutputTooLarge) { - s.respondError(w, fmt.Sprintf("size too large for the source: the side "+ - "computed from its proportions is above %d", - imageprocessor.MaxOutputDimension), http.StatusBadRequest) - - return - } - - if errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge) { - s.respondError(w, fmt.Sprintf("size too large for AVIF: the output "+ - "would be above %d pixels", imageprocessor.MaxAVIFOutputPixels), - http.StatusBadRequest) - - return - } - - s.respondError(w, "internal error", http.StatusInternalServerError) + s.respondError(w, message, status) } // cacheControl returns the Cache-Control header for an image served through a @@ -367,14 +342,34 @@ func notModified(w http.ResponseWriter, r *http.Request, etag string) bool { return true } -// writeImageResponse writes headers and streams the image content, -// handling conditional and HEAD requests. expires is the URL's expiry, which -// ends the Cache-Control max-age. -func (s *Handlers) writeImageResponse( +// serveImage serves the image that req asks for. Both image routes call it +// once they have the request and its URL's expiry, which ends the +// Cache-Control max-age. It chooses the format for the format auto, answers +// If-None-Match with 304 and HEAD with the headers only, and answers an error +// of the image service with respondImageError. +func (s *Handlers) serveImage( w http.ResponseWriter, r *http.Request, - req *imagerequest.Request, expires time.Time, resp *imgcache.ImageResponse, - cacheKey imagerequest.VariantKey, startTime time.Time, + req *imagerequest.Request, expires time.Time, ) { + // The signature or the token covers the format auto, not the format chosen + format, ok := s.chooseAutoFormat(w, r, req.Transform.Format) + if !ok { + return + } + + req.Transform.Format = format + + startTime := time.Now() + + resp, err := s.imgSvc.Get(r.Context(), req) + if err != nil { + s.respondImageError(w, req, err) + + return + } + + defer func() { _ = resp.Content.Close() }() + // Set response headers w.Header().Set("Content-Type", resp.ContentType) @@ -400,18 +395,28 @@ func (s *Handlers) writeImageResponse( // Stream the response w.WriteHeader(http.StatusOK) + // A failed write is most often a client that has gone away, not pixa's + // failure, so it is logged at warn level servedBytes, err := io.Copy(w, resp.Content) if err != nil { - s.log.Error("failed to write response", + s.log.Warn("failed to write response", + "request_id", middleware.GetReqID(r.Context()), + "host", req.Source.Host, + "path", req.Source.Path, + "served_bytes", servedBytes, "error", err, ) + + return } // Log cache status and timing after serving duration := time.Since(startTime) s.log.Info("image served", "request_id", middleware.GetReqID(r.Context()), - "cache_key", cacheKey, + "cache_key", imagerequest.CacheKey(req), + "host", req.Source.Host, + "path", req.Source.Path, "cache_status", resp.CacheStatus, "duration_ms", duration.Milliseconds(), "format", req.Transform.Format, diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index 0591044..a7952cb 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -1,18 +1,23 @@ package handlers import ( + "bytes" + "context" "encoding/json" "fmt" "image/color" + "io" "log/slog" "net/http" "net/http/httptest" + "slices" "strings" "testing" "testing/fstest" "time" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" @@ -414,3 +419,168 @@ func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) { }) } } + +// failingFetcher answers every fetch with its error. +type failingFetcher struct { + err error +} + +// Fetch returns f's error. +func (f failingFetcher) Fetch( + context.Context, imagerequest.Source, +) (*httpfetcher.FetchResult, error) { + return nil, f.err +} + +// photoURLsByRoute returns, for each image route, the function that makes the +// URL of the JPEG at photoPath on signedHost on that route. +func photoURLsByRoute() map[string]func(*testing.T, *Handlers) string { + return map[string]func(*testing.T, *Handlers) string{ + "/v1/image/": signedPhotoURL, + "/v1/e/": encPhotoURL, + } +} + +// TestImageRoutes_SameErrorAnswers checks that both image routes answer each +// upstream fetch error below with the same status. They log it as "failed to +// get image" at the same level: error for the 500 and warn for the others. +// Each request gets routes of its own, so no request finds another's failed +// fetch in the negative cache. +func TestImageRoutes_SameErrorAnswers(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + wantStatus int + wantLevel slog.Level + }{ + {"blocked upstream address", httpfetcher.ErrSSRFBlocked, + http.StatusForbidden, slog.LevelWarn}, + {"upstream error status", + fmt.Errorf("%w: status 404", httpfetcher.ErrUpstreamError), + http.StatusBadGateway, slog.LevelWarn}, + {"no response headers within the fetch timeout", + httpfetcher.ErrUpstreamTimeout, http.StatusGatewayTimeout, + slog.LevelWarn}, + {"no free upstream connection", httpfetcher.ErrTooManyConnections, + http.StatusServiceUnavailable, slog.LevelWarn}, + {"Content-Type not a source format", + fmt.Errorf("%w: image/svg+xml", httpfetcher.ErrInvalidContentType), + http.StatusUnsupportedMediaType, slog.LevelWarn}, + {"any other failure", io.ErrUnexpectedEOF, + http.StatusInternalServerError, slog.LevelError}, + } + + for _, tt := range tests { + for route, imageURL := range photoURLsByRoute() { + t.Run(route+" "+tt.name, func(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + h, routes := newRoutesFetchingThrough(t, + failingFetcher{err: tt.err}, + slog.New(slog.NewJSONHandler(&logs, nil))) + + rec := sendGet(t, routes, imageURL(t, h)) + if rec.Code != tt.wantStatus { + t.Errorf("status = %d, want %d: %s", + rec.Code, tt.wantStatus, rec.Body) + } + + levels := loggedLevels(t, &logs) + if !slices.Contains(levels[tt.wantLevel.String()], + "failed to get image") { + t.Errorf(`no "failed to get image" line logged at level %s`, + tt.wantLevel) + } + + if tt.wantLevel != slog.LevelError && + len(levels[slog.LevelError.String()]) != 0 { + t.Errorf("logged at error level: %q", + levels[slog.LevelError.String()]) + } + }) + } + } +} + +// failingResponseWriter is a response writer whose Write always fails, as when +// the client has gone away. +type failingResponseWriter struct { + *httptest.ResponseRecorder +} + +// Write writes nothing and returns io.ErrClosedPipe. +func (failingResponseWriter) Write([]byte) (int, error) { + return 0, io.ErrClosedPipe +} + +// TestImageRoutes_FailedWriteLoggedOnce checks that both image routes log a +// failed write of the image once, at warn level, with the request's ID, the +// source host and path, the bytes served and the error. Neither may also log +// the image as served. +func TestImageRoutes_FailedWriteLoggedOnce(t *testing.T) { + t.Parallel() + + const requestID = "test-request-id" + + for route, imageURL := range photoURLsByRoute() { + t.Run(route, func(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + h, srv := newSignedHostServer(t, + slog.New(slog.NewJSONHandler(&logs, nil))) + + ctx := context.WithValue(t.Context(), + middleware.RequestIDKey, requestID) + srv.ServeHTTP(failingResponseWriter{httptest.NewRecorder()}, + httptest.NewRequestWithContext( + ctx, http.MethodGet, imageURL(t, h), nil)) + + t.Logf("logged:\n%s", logs.String()) + + var failedWrites []map[string]any + + dec := json.NewDecoder(&logs) + for dec.More() { + var line map[string]any + + err := dec.Decode(&line) + if err != nil { + t.Fatalf("decoding log line: %v", err) + } + + switch line["msg"] { + case "image served": + t.Error(`"image served" logged after a failed write`) + case "failed to write response": + failedWrites = append(failedWrites, line) + } + } + + if len(failedWrites) != 1 { + t.Fatalf(`"failed to write response" logged %d times, want 1`, + len(failedWrites)) + } + + want := map[string]any{ + "level": slog.LevelWarn.String(), + "request_id": requestID, + "host": signedHost, + "path": photoPath, + "served_bytes": float64(0), + "error": io.ErrClosedPipe.Error(), + } + + for key, wantValue := range want { + if got := failedWrites[0][key]; got != wantValue { + t.Errorf("%s = %v, want %v", key, got, wantValue) + } + } + }) + } +} diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go index e823498..07c2772 100644 --- a/internal/handlers/imageenc.go +++ b/internal/handlers/imageenc.go @@ -2,21 +2,13 @@ package handlers import ( "errors" - "fmt" - "io" "net/http" - "strconv" "time" "github.com/go-chi/chi/v5" - "github.com/go-chi/chi/v5/middleware" "sneak.berlin/go/pixa/internal/encurl" - "sneak.berlin/go/pixa/internal/httpfetcher" - "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" - "sneak.berlin/go/pixa/internal/imgcache" - "sneak.berlin/go/pixa/internal/magic" ) // HandleImageEnc handles requests to /v1/e/{token}/* for encrypted @@ -28,83 +20,12 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc { return } - ctx := r.Context() - start := time.Now() - req, expires, ok := s.parseImageEncRequest(w, r) if !ok { return } - format, ok := s.chooseAutoFormat(w, r, req.Transform.Format) - if !ok { - return - } - - req.Transform.Format = format - - // Log the request - s.log.Debug("encrypted image request", - "host", req.Source.Host, - "path", req.Source.Path, - "dimensions", fmt.Sprintf("%dx%d", - req.Transform.Size.Width, req.Transform.Size.Height), - "format", req.Transform.Format, - ) - - // Fetch and process the image (no signature validation - // needed - encrypted URL is trusted) - resp, err := s.imgSvc.Get(ctx, req) - if err != nil { - s.handleImageError(w, err) - - return - } - - defer func() { _ = resp.Content.Close() }() - - // Set response headers - w.Header().Set("Content-Type", resp.ContentType) - - if resp.ContentLength > 0 { - w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10)) - } - - // Cache headers: max-age ends at the URL's expiry - w.Header().Set("Cache-Control", cacheControl(expires)) - w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) - - if notModified(w, r, resp.ETag) { - return - } - - // A HEAD request gets the headers only - if r.Method == http.MethodHead { - w.WriteHeader(http.StatusOK) - - return - } - - // Stream the response - written, err := io.Copy(w, resp.Content) - if err != nil { - s.log.Error("failed to write response", "error", err) - - return - } - - // Log completion - duration := time.Since(start) - s.log.Info("image served", - "request_id", middleware.GetReqID(ctx), - "cache_key", imagerequest.CacheKey(req), - "host", req.Source.Host, - "path", req.Source.Path, - "format", req.Transform.Format, - "cache_status", resp.CacheStatus, - "served_bytes", written, - "duration_ms", duration.Milliseconds(), - ) + s.serveImage(w, r, req, expires) } } @@ -133,7 +54,7 @@ func (s *Handlers) parseImageEncRequest( return nil, time.Time{}, false } - s.log.Debug("failed to decrypt URL", "error", err) + s.log.Debug("invalid encrypted URL", "error", err) s.respondError(w, "invalid encrypted URL", http.StatusBadRequest) return nil, time.Time{}, false @@ -141,8 +62,9 @@ func (s *Handlers) parseImageEncRequest( // Apply the same dimension and fit-mode bounds as the plain image // route: a sealed payload is trusted for its origin, not for staying - // within limits, so an over-limit size or unknown fit mode is a 400 - // here rather than an out-of-memory or a 500 from the processor. + // within limits. This refuses an over-limit size before the source is + // fetched. An unknown fit mode is a 400 here rather than a 500 from the + // processor. err = imagerequest.ValidateImageRequest(req) if err != nil { s.log.Debug("encrypted URL failed validation", "error", err) @@ -154,38 +76,3 @@ func (s *Handlers) parseImageEncRequest( return req, expires, true } - -// handleImageError converts image service errors to HTTP responses. -func (s *Handlers) handleImageError(w http.ResponseWriter, err error) { - switch { - case errors.Is(err, httpfetcher.ErrSSRFBlocked): - s.respondError(w, "forbidden", http.StatusForbidden) - case errors.Is(err, httpfetcher.ErrUpstreamError): - s.respondError(w, "upstream error", http.StatusBadGateway) - case errors.Is(err, httpfetcher.ErrUpstreamTimeout): - s.respondError(w, "upstream timeout", http.StatusGatewayTimeout) - case errors.Is(err, httpfetcher.ErrTooManyConnections), - errors.Is(err, imageprocessor.ErrTooManyImages), - errors.Is(err, imgcache.ErrProcessingTimedOut): - s.respondError(w, "server busy, try again later", - http.StatusServiceUnavailable) - case errors.Is(err, httpfetcher.ErrInvalidContentType), - errors.Is(err, imageprocessor.ErrUnsupportedInputFormat), - errors.Is(err, magic.ErrMagicByteMismatch), - errors.Is(err, magic.ErrUnknownFormat), - errors.Is(err, magic.ErrNotEnoughData): - s.respondError(w, "unsupported source format", - http.StatusUnsupportedMediaType) - case errors.Is(err, imageprocessor.ErrOutputTooLarge): - s.respondError(w, fmt.Sprintf("size too large for the source: the side "+ - "computed from its proportions is above %d", - imageprocessor.MaxOutputDimension), http.StatusBadRequest) - case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge): - s.respondError(w, fmt.Sprintf("size too large for AVIF: the output "+ - "would be above %d pixels", imageprocessor.MaxAVIFOutputPixels), - http.StatusBadRequest) - default: - s.log.Error("image request failed", "error", err) - s.respondError(w, "internal error", http.StatusInternalServerError) - } -} diff --git a/internal/handlers/server_busy_internal_test.go b/internal/handlers/server_busy_internal_test.go index a7cf030..6743959 100644 --- a/internal/handlers/server_busy_internal_test.go +++ b/internal/handlers/server_busy_internal_test.go @@ -15,9 +15,10 @@ import ( "sneak.berlin/go/pixa/internal/imgcache" ) -// TestServerBusyAnswers503 checks that both image routes answer 503 with -// a clear error when the image service gives up waiting for a free upstream -// connection or processing slot, wrapped as the service wraps them. +// TestServerBusyAnswers503 checks that respondImageError, which answers the +// errors of both image routes, answers 503 with a clear error when the image +// service gives up waiting for a free upstream connection or processing slot, +// wrapped as the service wraps them. func TestServerBusyAnswers503(t *testing.T) { t.Parallel() @@ -30,35 +31,28 @@ func TestServerBusyAnswers503(t *testing.T) { fmt.Errorf("upstream fetch failed: %w", httpfetcher.ErrTooManyConnections), fmt.Errorf("image processing failed: %w", imageprocessor.ErrTooManyImages), } { - plain := httptest.NewRecorder() - h.respondImageError(plain, req, err) + rec := httptest.NewRecorder() + h.respondImageError(rec, req, err) - encrypted := httptest.NewRecorder() - h.handleImageError(encrypted, err) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("for %v: status = %d, want %d", + err, rec.Code, http.StatusServiceUnavailable) + } - for route, rec := range map[string]*httptest.ResponseRecorder{ - "/v1/image/": plain, "/v1/e/": encrypted, - } { - if rec.Code != http.StatusServiceUnavailable { - t.Errorf("%s for %v: status = %d, want %d", - route, err, rec.Code, http.StatusServiceUnavailable) - } - - if !strings.Contains(rec.Body.String(), "server busy, try again later") { - t.Errorf("%s for %v: body = %q, want the server busy error", - route, err, rec.Body.String()) - } + if !strings.Contains(rec.Body.String(), "server busy, try again later") { + t.Errorf("for %v: body = %q, want the server busy error", + err, rec.Body.String()) } } } -// TestProcessingTimedOutAnswers503 checks that respondImageError and -// handleImageError, which answer the errors of the two image routes, answer -// 503 with the server busy error when the image service reports that -// processing stopped at the deadline of the request doing it, wrapped as the -// service wraps it. A request that waited for that processing gets this error -// while its own deadline may have time left. The request's source only goes -// into a log line, so it is left empty. +// TestProcessingTimedOutAnswers503 checks that respondImageError, which +// answers the errors of both image routes, answers 503 with the server busy +// error when the image service reports that processing stopped at the +// deadline of the request doing it, wrapped as the service wraps it. A request +// that waited for that processing gets this error while its own deadline may +// have time left. The request's source only goes into a log line, so it is +// left empty. func TestProcessingTimedOutAnswers503(t *testing.T) { t.Parallel() @@ -66,25 +60,15 @@ func TestProcessingTimedOutAnswers503(t *testing.T) { err := fmt.Errorf("%w: failed to encode: %w", imgcache.ErrProcessingTimedOut, context.DeadlineExceeded) - plain := httptest.NewRecorder() - h.respondImageError(plain, &imagerequest.Request{}, err) + rec := httptest.NewRecorder() + h.respondImageError(rec, &imagerequest.Request{}, err) + t.Logf("status = %d, body = %q", rec.Code, rec.Body.String()) - encrypted := httptest.NewRecorder() - h.handleImageError(encrypted, err) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable) + } - for name, rec := range map[string]*httptest.ResponseRecorder{ - "respondImageError": plain, "handleImageError": encrypted, - } { - t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String()) - - if rec.Code != http.StatusServiceUnavailable { - t.Errorf("%s: status = %d, want %d", - name, rec.Code, http.StatusServiceUnavailable) - } - - if !strings.Contains(rec.Body.String(), "server busy, try again later") { - t.Errorf("%s: body = %q, want the server busy error", - name, rec.Body.String()) - } + if !strings.Contains(rec.Body.String(), "server busy, try again later") { + t.Errorf("body = %q, want the server busy error", rec.Body.String()) } } diff --git a/internal/handlers/unsupported_source_internal_test.go b/internal/handlers/unsupported_source_internal_test.go index c319700..030274f 100644 --- a/internal/handlers/unsupported_source_internal_test.go +++ b/internal/handlers/unsupported_source_internal_test.go @@ -13,10 +13,11 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// TestUnsupportedSourceFormatAnswers415 checks that both image routes answer -// 415 with a clear error when the upstream answers with a content type pixa -// does not accept, such as SVG, and when orig is asked of a source in a -// format the processor cannot name, wrapped as the service wraps them. +// TestUnsupportedSourceFormatAnswers415 checks that respondImageError, which +// answers the errors of both image routes, answers 415 with a clear error when +// the upstream answers with a content type pixa does not accept, such as SVG, +// and when orig is asked of a source in a format the processor cannot name, +// wrapped as the service wraps them. func TestUnsupportedSourceFormatAnswers415(t *testing.T) { t.Parallel() @@ -31,26 +32,18 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) { fmt.Errorf("image processing failed: %w: unknown", imageprocessor.ErrUnsupportedInputFormat), } { - plain := httptest.NewRecorder() - h.respondImageError(plain, req, err) + rec := httptest.NewRecorder() + h.respondImageError(rec, req, err) + t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String()) - encrypted := httptest.NewRecorder() - h.handleImageError(encrypted, err) + if rec.Code != http.StatusUnsupportedMediaType { + t.Errorf("for %v: status = %d, want %d", + err, rec.Code, http.StatusUnsupportedMediaType) + } - for route, rec := range map[string]*httptest.ResponseRecorder{ - "HandleImage": plain, "HandleImageEnc": encrypted, - } { - t.Logf("%s for %v: %d %s", route, err, rec.Code, rec.Body.String()) - - if rec.Code != http.StatusUnsupportedMediaType { - t.Errorf("%s for %v: status = %d, want %d", - route, err, rec.Code, http.StatusUnsupportedMediaType) - } - - if !strings.Contains(rec.Body.String(), "unsupported source format") { - t.Errorf("%s for %v: body = %q, want the unsupported source format error", - route, err, rec.Body.String()) - } + if !strings.Contains(rec.Body.String(), "unsupported source format") { + t.Errorf("for %v: body = %q, want the unsupported source format error", + err, rec.Body.String()) } } } From 17376d885be9367de3345db0e7b07f80535147e1 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 20:00:25 +0100 Subject: [PATCH 107/131] Split internal/imgcache into the image proxy and the cache (closes #317) The new package internal/imageproxy holds Proxy, which was imgcache.Service, with its Config, ImageResponse, CacheStatus and errors. imageproxy.New makes and registers the counters of upstream fetches and transforms. internal/imgcache holds the cache only, with its hit and miss counters, and imgcache.CacheConfig is imgcache.Config. The cache reads and writes each kind it keeps with one pair of methods: Variant and PutVariant, Source and PutSource, Failure and PutFailure. Source replaces LookupSource followed by GetSourceContent. imgcache.go and module.go are gone. The proxy's tests moved with it. The cache's tests check that the reconciliation pass removes an empty or cut file and its row. The proxy's tests remove them by hand, then check that Get fetches or converts again. Model: opus-5-5 --- TODO.md | 31 ++ internal/handlers/cache_hit_internal_test.go | 17 +- .../handlers/first_bytes_internal_test.go | 9 +- internal/handlers/handlers.go | 6 +- internal/handlers/handlers_internal_test.go | 9 +- internal/handlers/image.go | 4 +- .../image_cache_control_internal_test.go | 9 +- .../handlers/image_errors_internal_test.go | 9 +- .../referer_blocklist_internal_test.go | 9 +- .../handlers/server_busy_internal_test.go | 4 +- .../imageproxy/cache_hit_internal_test.go | 100 +++++ .../concurrent_misses_internal_test.go | 76 ++-- internal/imageproxy/counters_internal_test.go | 77 ++++ .../divzero_internal_test.go | 2 +- .../service.go => imageproxy/imageproxy.go} | 232 ++++++++---- .../imageproxy_internal_test.go} | 78 ++-- ...max_concurrent_processing_internal_test.go | 22 +- .../negative_cache_internal_test.go | 96 +++++ .../partial_cache_files_internal_test.go | 149 ++++++++ .../pendingaccesstimes_internal_test.go | 78 ++++ .../stats_internal_test.go | 42 ++- internal/imageproxy/testutil_internal_test.go | 355 ++++++++++++++++++ internal/imgcache/cache.go | 277 +++++++------- internal/imgcache/cache_hit_internal_test.go | 121 +----- internal/imgcache/cache_internal_test.go | 4 +- internal/imgcache/counters.go | 16 +- internal/imgcache/counters_internal_test.go | 29 +- internal/imgcache/eviction.go | 4 +- internal/imgcache/eviction_internal_test.go | 38 +- internal/imgcache/imgcache.go | 29 -- internal/imgcache/module.go | 8 - .../imgcache/negative_cache_internal_test.go | 103 +---- .../partial_cache_files_internal_test.go | 223 +++-------- .../pendingaccesstimes_internal_test.go | 74 +--- .../imgcache/source_content_internal_test.go | 30 +- internal/imgcache/testutil_internal_test.go | 186 +-------- internal/providers/providers.go | 22 +- internal/server/source_url_internal_test.go | 8 +- 38 files changed, 1473 insertions(+), 1113 deletions(-) create mode 100644 internal/imageproxy/cache_hit_internal_test.go rename internal/{imgcache => imageproxy}/concurrent_misses_internal_test.go (85%) create mode 100644 internal/imageproxy/counters_internal_test.go rename internal/{imgcache => imageproxy}/divzero_internal_test.go (98%) rename internal/{imgcache/service.go => imageproxy/imageproxy.go} (68%) rename internal/{imgcache/service_internal_test.go => imageproxy/imageproxy_internal_test.go} (86%) rename internal/{imgcache => imageproxy}/max_concurrent_processing_internal_test.go (86%) create mode 100644 internal/imageproxy/negative_cache_internal_test.go create mode 100644 internal/imageproxy/partial_cache_files_internal_test.go create mode 100644 internal/imageproxy/pendingaccesstimes_internal_test.go rename internal/{imgcache => imageproxy}/stats_internal_test.go (83%) create mode 100644 internal/imageproxy/testutil_internal_test.go delete mode 100644 internal/imgcache/imgcache.go delete mode 100644 internal/imgcache/module.go diff --git a/TODO.md b/TODO.md index bb3ec91..0618377 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,37 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 `internal/imgcache` holds the cache only (closes #317). The new + package `internal/imageproxy` fetches, transforms, caches and serves images. + It imports `imgcache`, and `imgcache` does not import it. + - `imgcache.Service` is `imageproxy.Proxy`, built by `imageproxy.New` from + an `imageproxy.Config`. + - `ImageResponse`, `CacheStatus`, `ErrNegativeCached` and + `ErrProcessingTimedOut` moved with it, under their names. + - `imgcache.CacheConfig` is `imgcache.Config`. + - The cache has one read and one write for each kind it keeps. Each takes + the place of an earlier method: + - `Variant`, of `GetVariant` + - `PutVariant`, of `StoreVariant` + - `Source`, of `LookupSource` followed by `GetSourceContent` + - `PutSource`, of `StoreSource` + - `Failure`, of `checkNegativeCache` + - `PutFailure`, of `StoreNegative` + - `Variant` and `Source` each open the file and hand its access time to the + background writer. A cache hit still makes no database call. + - `imgcache.go` and `module.go` are gone. + - `imageproxy.New` makes the counters of upstream fetches and transforms, + and registers them on the registry it is given. The cache keeps only the + hit and miss counters. + - A cached source whose row cannot be read is logged as "failed to load + cached source, fetching". It was logged as "source lookup failed". + - The proxy's tests moved to `internal/imageproxy`. They reach the cache + through its database, its state directory and its metrics registry. + - The tests of an empty or cut cached file are split in two: + - `internal/imgcache` checks that the reconciliation pass removes the + file and its row, with a warning naming the file. + - `internal/imageproxy` removes the file and its row as the pass does, + and checks that `Get` fetches or converts again. - 2026-10-26 both image routes serve an image with one function, `serveImage`, and answer its errors with one, `respondImageError` (closes #316). `HandleImage` parses the path and verifies the signature, and `HandleImageEnc` diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go index f62a2b0..6295146 100644 --- a/internal/handlers/cache_hit_internal_test.go +++ b/internal/handlers/cache_hit_internal_test.go @@ -23,6 +23,7 @@ import ( "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/seal" @@ -122,7 +123,7 @@ func newCountingDatabaseServer( tb.Fatalf("encoding the photo: %v", err) } - cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ + cache, err := imgcache.NewCache(db, imgcache.Config{ StateDir: tb.TempDir(), NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -131,7 +132,7 @@ func newCountingDatabaseServer( tb.Fatalf("imgcache.NewCache() error = %v", err) } - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + svc, err := imageproxy.New(&imageproxy.Config{ Cache: cache, Fetcher: newMockFetcher(fstest.MapFS{ countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()}, @@ -142,9 +143,9 @@ func newCountingDatabaseServer( }), MaxResponseSize: 50 << 20, Logger: slog.New(slog.DiscardHandler), - }) + }, prometheus.NewRegistry()) if err != nil { - tb.Fatalf("imgcache.NewService() error = %v", err) + tb.Fatalf("imageproxy.New() error = %v", err) } keys, err := seal.DeriveKeys(testSigningKey) @@ -228,9 +229,9 @@ func TestImageRoutes_CacheHitsMakeNoDatabaseCall(t *testing.T) { rec := serveImage(t, srv, target) cacheStatus := rec.Header().Get("X-Pixa-Cache") - if cacheStatus != string(imgcache.CacheHit) { + if cacheStatus != string(imageproxy.CacheHit) { t.Errorf("GET %s: X-Pixa-Cache = %q, want %s", - target, cacheStatus, imgcache.CacheHit) + target, cacheStatus, imageproxy.CacheHit) } } @@ -260,9 +261,9 @@ func BenchmarkHandleImage_CacheHits(b *testing.B) { rec := serveImage(b, srv, target) cacheStatus := rec.Header().Get("X-Pixa-Cache") - if cacheStatus != string(imgcache.CacheHit) { + if cacheStatus != string(imageproxy.CacheHit) { b.Errorf("GET %s: X-Pixa-Cache = %q, want %s", - target, cacheStatus, imgcache.CacheHit) + target, cacheStatus, imageproxy.CacheHit) return } diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go index 3a25b79..3125edc 100644 --- a/internal/handlers/first_bytes_internal_test.go +++ b/internal/handlers/first_bytes_internal_test.go @@ -17,6 +17,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/seal" @@ -38,7 +39,7 @@ func newRoutesFetchingThrough( ) (*Handlers, http.Handler) { t.Helper() - cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -47,7 +48,7 @@ func newRoutesFetchingThrough( t.Fatalf("imgcache.NewCache() error = %v", err) } - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + svc, err := imageproxy.New(&imageproxy.Config{ Cache: cache, Fetcher: fetcher, Processor: imageprocessor.New(imageprocessor.Params{ @@ -56,9 +57,9 @@ func newRoutesFetchingThrough( }), MaxResponseSize: 50 << 20, Logger: log, - }) + }, prometheus.NewRegistry()) if err != nil { - t.Fatalf("imgcache.NewService() error = %v", err) + t.Fatalf("imageproxy.New() error = %v", err) } keys, err := seal.DeriveKeys(testSigningKey) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 50c4f1e..0f4e134 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -13,7 +13,7 @@ import ( "sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/healthcheck" - "sneak.berlin/go/pixa/internal/imgcache" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" @@ -29,7 +29,7 @@ type Params struct { Config *config.Config Keys seal.Keys - ImageService *imgcache.Service + ImageService *imageproxy.Proxy SessionManager *session.Manager EncryptedURLGenerator *encurl.Generator } @@ -39,7 +39,7 @@ type Handlers struct { log *slog.Logger hc *healthcheck.Healthcheck config *config.Config - imgSvc *imgcache.Service + imgSvc *imageproxy.Proxy sessMgr *session.Manager encGen *encurl.Generator csrfProtect func(http.Handler) http.Handler diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index 9568849..de83bdf 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -23,6 +23,7 @@ import ( "sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/signature" @@ -49,7 +50,7 @@ func setupTestHandler(t *testing.T) *testFixtures { tmpDir := t.TempDir() db := setupTestDB(t) - cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ + cache, err := imgcache.NewCache(db, imgcache.Config{ StateDir: tmpDir, NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -58,7 +59,7 @@ func setupTestHandler(t *testing.T) *testFixtures { t.Fatalf("failed to create cache: %v", err) } - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + svc, err := imageproxy.New(&imageproxy.Config{ Cache: cache, Fetcher: newMockFetcher(mockFS), Processor: imageprocessor.New(imageprocessor.Params{ @@ -66,9 +67,9 @@ func setupTestHandler(t *testing.T) *testFixtures { MaxConcurrentProcessing: 2, }), MaxResponseSize: 50 << 20, - }) + }, prometheus.NewRegistry()) if err != nil { - t.Fatalf("failed to create service: %v", err) + t.Fatalf("failed to create proxy: %v", err) } h := &Handlers{ diff --git a/internal/handlers/image.go b/internal/handlers/image.go index d349d5b..68c2fee 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -14,8 +14,8 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" - "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/magic" ) @@ -253,7 +253,7 @@ func (s *Handlers) respondImageError( status, message = http.StatusGatewayTimeout, "upstream timeout" case errors.Is(err, httpfetcher.ErrTooManyConnections), errors.Is(err, imageprocessor.ErrTooManyImages), - errors.Is(err, imgcache.ErrProcessingTimedOut): + errors.Is(err, imageproxy.ErrProcessingTimedOut): status = http.StatusServiceUnavailable message = "server busy, try again later" case errors.Is(err, httpfetcher.ErrInvalidContentType), diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index a561c8d..97db3af 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -17,6 +17,7 @@ import ( "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/seal" @@ -35,7 +36,7 @@ func newSignedHostServer( ) (*Handlers, http.Handler) { t.Helper() - cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -46,7 +47,7 @@ func newSignedHostServer( jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + svc, err := imageproxy.New(&imageproxy.Config{ Cache: cache, Fetcher: newMockFetcher(fstest.MapFS{ signedHost + photoPath: &fstest.MapFile{Data: jpegData}, @@ -57,9 +58,9 @@ func newSignedHostServer( }), MaxResponseSize: 50 << 20, Logger: log, - }) + }, prometheus.NewRegistry()) if err != nil { - t.Fatalf("imgcache.NewService() error = %v", err) + t.Fatalf("imageproxy.New() error = %v", err) } keys, err := seal.DeriveKeys(testSigningKey) diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index a7952cb..f4fee3c 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -22,6 +22,7 @@ import ( "sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/signature" @@ -36,7 +37,7 @@ const allowlistedHost = "allowed.example.com" func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { t.Helper() - cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -45,7 +46,7 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { t.Fatalf("failed to create cache: %v", err) } - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + svc, err := imageproxy.New(&imageproxy.Config{ Cache: cache, Fetcher: fetcher, Processor: imageprocessor.New(imageprocessor.Params{ @@ -53,9 +54,9 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { MaxConcurrentProcessing: 2, }), MaxResponseSize: 50 << 20, - }) + }, prometheus.NewRegistry()) if err != nil { - t.Fatalf("failed to create service: %v", err) + t.Fatalf("failed to create proxy: %v", err) } h := &Handlers{ diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index 51fa8ec..98691c4 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -15,6 +15,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/seal" @@ -52,7 +53,7 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost), } - cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{ + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ StateDir: t.TempDir(), NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -61,7 +62,7 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) t.Fatalf("imgcache.NewCache() error = %v", err) } - svc, err := imgcache.NewService(&imgcache.ServiceConfig{ + svc, err := imageproxy.New(&imageproxy.Config{ Cache: cache, Fetcher: fetcher, Processor: imageprocessor.New(imageprocessor.Params{ @@ -69,9 +70,9 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) MaxConcurrentProcessing: 2, }), MaxResponseSize: 50 << 20, - }) + }, prometheus.NewRegistry()) if err != nil { - t.Fatalf("imgcache.NewService() error = %v", err) + t.Fatalf("imageproxy.New() error = %v", err) } keys, err := seal.DeriveKeys(testSigningKey) diff --git a/internal/handlers/server_busy_internal_test.go b/internal/handlers/server_busy_internal_test.go index 6743959..fa5297c 100644 --- a/internal/handlers/server_busy_internal_test.go +++ b/internal/handlers/server_busy_internal_test.go @@ -11,8 +11,8 @@ import ( "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" - "sneak.berlin/go/pixa/internal/imgcache" ) // TestServerBusyAnswers503 checks that respondImageError, which answers the @@ -58,7 +58,7 @@ func TestProcessingTimedOutAnswers503(t *testing.T) { h := &Handlers{log: slog.New(slog.DiscardHandler)} err := fmt.Errorf("%w: failed to encode: %w", - imgcache.ErrProcessingTimedOut, context.DeadlineExceeded) + imageproxy.ErrProcessingTimedOut, context.DeadlineExceeded) rec := httptest.NewRecorder() h.respondImageError(rec, &imagerequest.Request{}, err) diff --git a/internal/imageproxy/cache_hit_internal_test.go b/internal/imageproxy/cache_hit_internal_test.go new file mode 100644 index 0000000..470ef8f --- /dev/null +++ b/internal/imageproxy/cache_hit_internal_test.go @@ -0,0 +1,100 @@ +package imageproxy + +import ( + "bytes" + "os" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestProxy_Get_MakesAgainVariantOfNoOutputFormat replaces a cached +// variant's file with text, and checks that the next request is a miss +// answered with the variant made again, and the one after a hit. +func TestProxy_Get_MakesAgainVariantOfNoOutputFormat(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + req := photoVariant(fixtures, 85, imagerequest.FitCover) + + want, _ := getImage(t, svc, req) + + path := variantFilePath(svc, imagerequest.CacheKey(req)) + + err := os.WriteFile(path, []byte("this is text, not an image"), + imgcache.StorageFilePerm) + if err != nil { + t.Fatalf("replacing the variant file: %v", err) + } + + got, status := getImage(t, svc, req) + t.Logf("after the replacement: %d bytes, %s", len(got), status) + + if status != CacheMiss || !bytes.Equal(got, want) { + t.Errorf("after the replacement: Get() = %d bytes, %s; want the %d bytes "+ + "made again, %s", len(got), status, len(want), CacheMiss) + } + + got, status = getImage(t, svc, req) + + if status != CacheHit || !bytes.Equal(got, want) { + t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s", + len(got), status, len(want), CacheHit) + } +} + +// TestProxy_Get_HitWaitsForNoDatabaseWrite holds the database's one +// connection while a cached variant is requested. The hit must be answered +// while the connection is still held, and its access time must reach the +// database once it is free. +func TestProxy_Get_HitWaitsForNoDatabaseWrite(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + req := photoVariant(fixtures, 85, imagerequest.FitCover) + key := imagerequest.CacheKey(req) + + getImage(t, svc, req) + + // Each connection to an in-memory SQLite database opens a new, empty one, + // so the goroutine that writes the access times must share the + // connection that holds the schema. + svc.db.SetMaxOpenConns(1) + svc.cache.StartPendingAccessTimeWrites() + + defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }() + + anHourAgo := time.Now().Add(-time.Hour) + setVariantLastAccessed(t, svc.db, key, anHourAgo) + + releaseDatabase := holdDatabase(t, svc.db) + defer releaseDatabase() + + results := startGet(t.Context(), svc, req) + + select { + case got := <-results: + if got.err != nil { + t.Fatalf("Get() of the cached variant error = %v", got.err) + } + case <-time.After(pendingAccessTimesWait): + t.Fatal("the cache hit waited for the database") + } + + releaseDatabase() + + deadline := time.Now().Add(pendingAccessTimesWait) + + for countRows(t, svc.db, ` + SELECT COUNT(*) FROM variant_content + WHERE cache_key = ? AND last_accessed_at > ? + `, string(key), anHourAgo.UTC().Format(sqliteTimestampFormat)) == 0 { + if time.Now().After(deadline) { + t.Fatal("the hit's access time was not written") + } + + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/imgcache/concurrent_misses_internal_test.go b/internal/imageproxy/concurrent_misses_internal_test.go similarity index 85% rename from internal/imgcache/concurrent_misses_internal_test.go rename to internal/imageproxy/concurrent_misses_internal_test.go index 594e4fb..4c7409f 100644 --- a/internal/imgcache/concurrent_misses_internal_test.go +++ b/internal/imageproxy/concurrent_misses_internal_test.go @@ -1,4 +1,4 @@ -package imgcache +package imageproxy import ( "bytes" @@ -56,15 +56,15 @@ func (f *heldFetcher) releaseFetches() { f.releaseOnce.Do(func() { close(f.release) }) } -// setupHeldFetchService returns a test service whose fetches go through a +// setupHeldFetchProxy returns a test proxy whose fetches go through a // heldFetcher. Its database is limited to one connection: each connection to // an in-memory SQLite database opens a new, empty one, so requests running at // once must share the connection that holds the schema. -func setupHeldFetchService(t *testing.T) (*Service, *TestFixtures, *heldFetcher) { +func setupHeldFetchProxy(t *testing.T) (*testProxy, *TestFixtures, *heldFetcher) { t.Helper() - svc, fixtures := SetupTestService(t) - svc.cache.db.SetMaxOpenConns(1) + svc, fixtures := setupTestProxy(t) + svc.db.SetMaxOpenConns(1) fetcher := &heldFetcher{ upstream: svc.fetcher, @@ -107,7 +107,7 @@ type getResult struct { // startGet calls Get in a goroutine of its own and delivers what it returned // on the channel. func startGet( - ctx context.Context, svc *Service, req *imagerequest.Request, + ctx context.Context, svc *testProxy, req *imagerequest.Request, ) <-chan getResult { results := make(chan getResult, 1) @@ -139,14 +139,14 @@ func jpegSize(data []byte) (int, int) { return config.Width, config.Height } -// TestService_Get_ConcurrentMissesShareOneFetch starts several requests for +// TestProxy_Get_ConcurrentMissesShareOneFetch starts several requests for // one uncached variant while the first one's fetch is held. Between them they // must fetch the source once and transcode it once, every one must be answered // with the same 50x25 JPEG, and each must count one miss. -func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { +func TestProxy_Get_ConcurrentMissesShareOneFetch(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) const requests = 8 @@ -183,7 +183,7 @@ func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { t.Errorf("%d requests made %d upstream fetches, want 1", requests, fetches) } - // NewTestFS builds the same files the test service's fetcher serves. + // NewTestFS builds the same files the test proxy's fetcher serves. testFS, _ := NewTestFS(t) photo, err := fs.ReadFile(testFS, fixtures.GoodHostJPEG) @@ -193,18 +193,18 @@ func TestService_Get_ConcurrentMissesShareOneFetch(t *testing.T) { want := counterValues{0, requests, 1, int64(len(photo)), 1} - if got := readCounters(svc.cache); got != want { + if got := readCounters(t, svc); got != want { t.Errorf("counters = %+v, want %+v", got, want) } } -// TestService_Get_ConcurrentVariantsStayApart requests three variants of the +// TestProxy_Get_ConcurrentVariantsStayApart requests three variants of the // test photo at once that differ only in quality or fit. Each must be made by // a fetch and a transcode of its own, and each answer must be its own variant. -func TestService_Get_ConcurrentVariantsStayApart(t *testing.T) { +func TestProxy_Get_ConcurrentVariantsStayApart(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) cover := startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover)) lowQuality := startGet(t.Context(), svc, @@ -253,14 +253,14 @@ func TestService_Get_ConcurrentVariantsStayApart(t *testing.T) { } } -// TestService_Get_WaiterStopsWhenItsContextEnds has a second request for a +// TestProxy_Get_WaiterStopsWhenItsContextEnds has a second request for a // variant join the first one's held fetch, then ends the second request's // context. The second request must return at once with the context's error, // while the fetch is still held, and the first must still be answered. -func TestService_Get_WaiterStopsWhenItsContextEnds(t *testing.T) { +func TestProxy_Get_WaiterStopsWhenItsContextEnds(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) first := startGet(t.Context(), svc, photoVariant(fixtures, 85, imagerequest.FitCover)) @@ -298,15 +298,15 @@ func TestService_Get_WaiterStopsWhenItsContextEnds(t *testing.T) { } } -// TestService_Get_FirstRequestLeavingKeepsTheWork ends the context of the +// TestProxy_Get_FirstRequestLeavingKeepsTheWork ends the context of the // request whose fetch is held, after a second request has joined it. The fetch // and transcode must go on and answer the second request. The first request // waits for its own work, as every request did before misses were shared, and // is answered too. -func TestService_Get_FirstRequestLeavingKeepsTheWork(t *testing.T) { +func TestProxy_Get_FirstRequestLeavingKeepsTheWork(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) firstCtx, cancelFirst := context.WithCancel(t.Context()) first := startGet(firstCtx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) @@ -337,11 +337,11 @@ func TestService_Get_FirstRequestLeavingKeepsTheWork(t *testing.T) { } } -// TestService_Get_ConcurrentMissesShareAFailure has several requests for an +// TestProxy_Get_ConcurrentMissesShareAFailure has several requests for an // image that cannot be served arrive while its fetch is held: the one fetch // answers all of them with its error. The request after them is answered from // the negative cache, where the failure is kept. -func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { +func TestProxy_Get_ConcurrentMissesShareAFailure(t *testing.T) { t.Parallel() tests := []struct { @@ -362,7 +362,7 @@ func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) request := func() *imagerequest.Request { req := photoVariant(fixtures, 85, imagerequest.FitCover) @@ -398,12 +398,12 @@ func TestService_Get_ConcurrentMissesShareAFailure(t *testing.T) { } } -// TestService_Get_EndedRequestFetchesNothing checks that a request whose +// TestProxy_Get_EndedRequestFetchesNothing checks that a request whose // context has already ended when it misses the cache starts no fetch. -func TestService_Get_EndedRequestFetchesNothing(t *testing.T) { +func TestProxy_Get_EndedRequestFetchesNothing(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) ctx, cancel := context.WithCancel(t.Context()) cancel() @@ -418,14 +418,14 @@ func TestService_Get_EndedRequestFetchesNothing(t *testing.T) { } } -// TestService_Get_ReturnsByItsDeadline gives a request a deadline and holds +// TestProxy_Get_ReturnsByItsDeadline gives a request a deadline and holds // its fetch until the fetch's context ends, as the fetcher does while it waits // for a free connection to the host. The request must return by its deadline // with the deadline's error. -func TestService_Get_ReturnsByItsDeadline(t *testing.T) { +func TestProxy_Get_ReturnsByItsDeadline(t *testing.T) { t.Parallel() - svc, fixtures, _ := setupHeldFetchService(t) + svc, fixtures, _ := setupHeldFetchProxy(t) const timeout = 200 * time.Millisecond @@ -444,16 +444,16 @@ func TestService_Get_ReturnsByItsDeadline(t *testing.T) { } } -// TestService_Get_WaiterGetsProcessingTimedOut has a request with no deadline +// TestProxy_Get_WaiterGetsProcessingTimedOut has a request with no deadline // join the held fetch of a first request with a short one. With the only // processing slot held, the first request's transcode waits for it until the // first request's deadline ends, and stops then. The waiting request, whose // own context has not ended, must get ErrProcessingTimedOut, which the image // routes answer 503 as when pixa is busy. -func TestService_Get_WaiterGetsProcessingTimedOut(t *testing.T) { +func TestProxy_Get_WaiterGetsProcessingTimedOut(t *testing.T) { t.Parallel() - svc, fixtures, fetcher := setupHeldFetchService(t) + svc, fixtures, fetcher := setupHeldFetchProxy(t) svc.processor = imageprocessor.New( imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}, ) @@ -494,13 +494,13 @@ func (panickingFetcher) Fetch( panic("upstream fetcher panicked") } -// TestService_Get_PanicBecomesAnError checks that a panic while a variant is +// TestProxy_Get_PanicBecomesAnError checks that a panic while a variant is // being made reaches its request as an error naming the panic, instead of // being raised again. -func TestService_Get_PanicBecomesAnError(t *testing.T) { +func TestProxy_Get_PanicBecomesAnError(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) svc.fetcher = panickingFetcher{} var err error @@ -520,13 +520,13 @@ func TestService_Get_PanicBecomesAnError(t *testing.T) { } } -// TestService_Get_PanicIsReportedToSentry checks that a panic while a variant +// TestProxy_Get_PanicIsReportedToSentry checks that a panic while a variant // is being made is reported through the Sentry hub on the request's context, // where the Sentry middleware puts one when sentry_dsn is set. -func TestService_Get_PanicIsReportedToSentry(t *testing.T) { +func TestProxy_Get_PanicIsReportedToSentry(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) svc.fetcher = panickingFetcher{} transport := &sentry.MockTransport{} diff --git a/internal/imageproxy/counters_internal_test.go b/internal/imageproxy/counters_internal_test.go new file mode 100644 index 0000000..5e19842 --- /dev/null +++ b/internal/imageproxy/counters_internal_test.go @@ -0,0 +1,77 @@ +package imageproxy + +import ( + "errors" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// TestNew_RegistersItsCountersOnItsRegistry checks that New registers the +// proxy's counters on the registry it is given, under the names README.md +// gives them. The cache registers its own counters on another registry. +func TestNew_RegistersItsCountersOnItsRegistry(t *testing.T) { + t.Parallel() + + cache, err := imgcache.NewCache(setupTestDB(t), + imgcache.Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, + prometheus.NewRegistry()) + if err != nil { + t.Fatalf("imgcache.NewCache() error = %v", err) + } + + registry := prometheus.NewRegistry() + + _, err = New(&Config{Cache: cache}, registry) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + names := []string{ + "pixa_upstream_fetches_total", + "pixa_upstream_fetch_bytes_total", + "pixa_transforms_total", + } + + count, err := testutil.GatherAndCount(registry, names...) + if err != nil { + t.Fatalf("testutil.GatherAndCount() error = %v", err) + } + + if count != len(names) { + t.Errorf("%d of the counters %v are on the registry, want %d", + count, names, len(names)) + } +} + +// TestNew_FailsWhenItsCountersCannotBeRegistered makes a second proxy on +// the registry of a first. Its counters have the same names, so the +// registry refuses them, and New must return that error. +func TestNew_FailsWhenItsCountersCannotBeRegistered(t *testing.T) { + t.Parallel() + + cache, err := imgcache.NewCache(setupTestDB(t), + imgcache.Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, + prometheus.NewRegistry()) + if err != nil { + t.Fatalf("imgcache.NewCache() error = %v", err) + } + + registry := prometheus.NewRegistry() + + _, err = New(&Config{Cache: cache}, registry) + if err != nil { + t.Fatalf("New() of the first proxy error = %v", err) + } + + _, err = New(&Config{Cache: cache}, registry) + t.Logf("New() of the second proxy error = %v", err) + + var alreadyRegistered prometheus.AlreadyRegisteredError + if !errors.As(err, &alreadyRegistered) { + t.Errorf("New() of the second proxy error = %v, want %T", + err, alreadyRegistered) + } +} diff --git a/internal/imgcache/divzero_internal_test.go b/internal/imageproxy/divzero_internal_test.go similarity index 98% rename from internal/imgcache/divzero_internal_test.go rename to internal/imageproxy/divzero_internal_test.go index b4117be..66a643c 100644 --- a/internal/imgcache/divzero_internal_test.go +++ b/internal/imageproxy/divzero_internal_test.go @@ -1,4 +1,4 @@ -package imgcache +package imageproxy import ( "fmt" diff --git a/internal/imgcache/service.go b/internal/imageproxy/imageproxy.go similarity index 68% rename from internal/imgcache/service.go rename to internal/imageproxy/imageproxy.go index 35d762a..7df0381 100644 --- a/internal/imgcache/service.go +++ b/internal/imageproxy/imageproxy.go @@ -1,4 +1,7 @@ -package imgcache +// Package imageproxy fetches, transforms, caches and serves images. Its +// Proxy serves an image from the cache when it can, and otherwise makes it +// from the cached or fetched source image. +package imageproxy import ( "bytes" @@ -13,16 +16,18 @@ import ( "github.com/dustin/go-humanize" "github.com/getsentry/sentry-go" "github.com/go-chi/chi/v5/middleware" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/singleflight" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/magic" ) -// Service orchestrates the cache, fetcher, and processor. -type Service struct { - cache *Cache +// Proxy orchestrates the cache, fetcher, and processor. +type Proxy struct { + cache *imgcache.Cache fetcher httpfetcher.Fetcher processor *imageprocessor.ImageProcessor log *slog.Logger @@ -30,12 +35,18 @@ type Service struct { // variantsInProgress lets the requests that miss the same variant at the // same time share one fetch and one transcode. variantsInProgress singleflight.Group + + // The counts of upstream fetches and transforms, made and registered by + // New. They are kept in memory, so adding to one waits for nothing. + upstreamFetches prometheus.Counter + upstreamFetchBytes prometheus.Counter + transforms prometheus.Counter } -// ServiceConfig holds configuration for the image service. -type ServiceConfig struct { +// Config holds the proxy's configuration. +type Config struct { // Cache is the cache instance - Cache *Cache + Cache *imgcache.Cache // Fetcher fetches source images from upstream hosts Fetcher httpfetcher.Fetcher // Processor converts and resizes the images @@ -47,11 +58,16 @@ type ServiceConfig struct { Logger *slog.Logger } -// errCacheRequired is returned when the service is built without a cache. +// errCacheRequired is returned when the proxy is built without a cache. var errCacheRequired = errors.New("cache is required") -// NewService creates a new image service. -func NewService(cfg *ServiceConfig) (*Service, error) { +// New creates a new image proxy and registers its counters on +// metricsRegistry. pixad passes Prometheus' default registry, the one +// /metrics serves. It fails when metricsRegistry already holds counters of +// the same names. +func New( + cfg *Config, metricsRegistry prometheus.Registerer, +) (*Proxy, error) { if cfg.Cache == nil { return nil, errCacheRequired } @@ -68,13 +84,36 @@ func NewService(cfg *ServiceConfig) (*Service, error) { return nil, err } - return &Service{ + p := &Proxy{ cache: cfg.Cache, fetcher: cfg.Fetcher, processor: cfg.Processor, log: log, maxResponseSize: cfg.MaxResponseSize, - }, nil + upstreamFetches: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_upstream_fetches_total", + Help: "Fetches of a source image from upstream that read any bytes.", + }), + upstreamFetchBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_upstream_fetch_bytes_total", + Help: "Bytes read from upstream by fetches of source images.", + }), + transforms: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_transforms_total", + Help: "Images the image processor transformed.", + }), + } + + for _, counter := range []prometheus.Counter{ + p.upstreamFetches, p.upstreamFetchBytes, p.transforms, + } { + err = metricsRegistry.Register(counter) + if err != nil { + return nil, fmt.Errorf("failed to register the proxy's counters: %w", err) + } + } + + return p, nil } // ErrNegativeCached is returned when a URL is in the negative cache (recently failed). @@ -86,25 +125,49 @@ var ErrNegativeCached = errors.New("request is in negative cache (recently faile var ErrProcessingTimedOut = errors.New( "image processing stopped at the deadline of the request doing it") +// ImageResponse represents a processed image ready to serve +type ImageResponse struct { + // Content is the image data reader + Content io.ReadCloser + // ContentLength is the size in bytes (-1 if unknown) + ContentLength int64 + // ContentType is the MIME type of the response + ContentType string + // ETag is the entity tag for caching + ETag string + // CacheStatus indicates HIT or MISS + CacheStatus CacheStatus + // FetchedBytes is the number of bytes fetched from upstream (0 if cache hit) + FetchedBytes int64 +} + +// CacheStatus indicates whether the response was served from cache. +type CacheStatus string + +// Cache status values for response headers. +const ( + CacheHit CacheStatus = "HIT" + CacheMiss CacheStatus = "MISS" +) + // Get retrieves a processed image, fetching and processing if necessary. // While its source URL is in the negative cache, a cached variant is still // served and a variant not in the cache is still made from a cached source; // only a request that needs the source fetched is refused. // -// A cache hit opens the variant file and makes no database call: its access -// time is handed to the goroutine that StartPendingAccessTimeWrites starts, -// without waiting for it, and its hit is counted in memory. A variant file -// that cannot be served, such as one whose first bytes are not those of an -// output format, is a miss, and the variant is made again. -func (s *Service) Get( +// A cache hit opens the variant file and makes no database call: the cache +// hands its access time to the goroutine that writes access times, without +// waiting for it, and its hit is counted in memory. A variant file that +// cannot be served, such as one whose first bytes are not those of an output +// format, is a miss, and the variant is made again. +func (p *Proxy) Get( ctx context.Context, req *imagerequest.Request, ) (*ImageResponse, error) { cacheKey := imagerequest.CacheKey(req) - reader, size, contentType, err := s.cache.GetVariant(cacheKey) + reader, size, contentType, err := p.cache.Variant(cacheKey) if err == nil { - s.cache.touchVariant(cacheKey) - s.cache.IncrementStats(true, 0) + p.cache.IncrementStats(true) return &ImageResponse{ Content: reader, @@ -115,17 +178,17 @@ func (s *Service) Get( }, nil } - if !errors.Is(err, ErrNotFound) { - s.log.Warn("cached variant not served, making it again", + if !errors.Is(err, imgcache.ErrNotFound) { + p.log.Warn("cached variant not served, making it again", "cache_key", cacheKey, "error", err) } // Cache miss - get the variant, processed once for all the requests that // miss it at the same time, then count this request's miss, also when it // failed or the request context has ended meanwhile - response, err := s.processOrWait(ctx, req) + response, err := p.processOrWait(ctx, req) - s.cache.IncrementStats(false, 0) + p.cache.IncrementStats(false) if err != nil { return nil, err @@ -138,8 +201,8 @@ func (s *Service) Get( // WaitForProcessing waits until no image is being processed, or until ctx // ends, and returns how many images were still being processed then. -func (s *Service) WaitForProcessing(ctx context.Context) int { - return s.processor.WaitForProcessing(ctx) +func (p *Proxy) WaitForProcessing(ctx context.Context) int { + return p.processor.WaitForProcessing(ctx) } // errPanicked is returned when processing a variant panicked. @@ -151,7 +214,7 @@ var errPanicked = errors.New("panic while processing image") // and take no upstream connection or processing slot. The processing ignores // the first request's cancellation, so the others are still served if that // client goes away, but keeps its deadline. -func (s *Service) processOrWait( +func (p *Proxy) processOrWait( ctx context.Context, req *imagerequest.Request, ) (*ImageResponse, error) { // A request that has already ended starts no processing @@ -165,7 +228,7 @@ func (s *Service) processOrWait( // only when no other request is processing the variant processing := make(chan struct{}) - results := s.variantsInProgress.DoChan(string(cacheKey), + results := p.variantsInProgress.DoChan(string(cacheKey), func() (_ any, err error) { close(processing) @@ -176,7 +239,7 @@ func (s *Service) processOrWait( defer func() { recovered := recover() if recovered != nil { - s.log.Error("panic while processing image", + p.log.Error("panic while processing image", "host", req.Source.Host, "path", req.Source.Path, "panic", recovered, "stack", string(debug.Stack())) @@ -191,7 +254,7 @@ func (s *Service) processOrWait( processingCtx, cancel := withoutCancelKeepingDeadline(ctx) defer cancel() - return s.processFromSourceOrFetch(processingCtx, req, cacheKey) + return p.processFromSourceOrFetch(processingCtx, req, cacheKey) }) var result singleflight.Result @@ -240,25 +303,31 @@ func withoutCancelKeepingDeadline( return context.WithDeadline(context.WithoutCancel(ctx), deadline) } -// loadCachedSource opens source content from cache, without reading it, and -// returns it with its size; nil if the cached data is unavailable, empty or -// exceeds maxResponseSize. -func (s *Service) loadCachedSource( - sourceKey imagerequest.SourceKey, +// loadCachedSource opens src's cached source, without reading it, and +// returns it with its size; nil if it is not in the cache, cannot be read, +// is empty or exceeds maxResponseSize. +func (p *Proxy) loadCachedSource( + ctx context.Context, src imagerequest.Source, ) (io.ReadCloser, int64) { - reader, size, err := s.cache.GetSourceContent(sourceKey) + reader, size, err := p.cache.Source(ctx, src) + if errors.Is(err, imgcache.ErrNotFound) { + return nil, 0 + } + if err != nil { - s.log.Warn("failed to load cached source, fetching", "error", err) + p.log.Warn("failed to load cached source, fetching", "error", err) return nil, 0 } - if size > s.maxResponseSize { + p.log.Debug("using cached source", "source_key", src.Key()) + + if size > p.maxResponseSize { _ = reader.Close() - s.log.Warn("cached source exceeds max response size, discarding", - "source_key", sourceKey, - "max_bytes", s.maxResponseSize, + p.log.Warn("cached source exceeds max response size, discarding", + "source_key", src.Key(), + "max_bytes", p.maxResponseSize, ) return nil, 0 @@ -275,30 +344,16 @@ func (s *Service) loadCachedSource( // processFromSourceOrFetch processes an image, using cached source content // if available. -func (s *Service) processFromSourceOrFetch( +func (p *Proxy) processFromSourceOrFetch( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { - // Check if we have cached source content - sourceKey, err := s.cache.LookupSource(ctx, req) - if err != nil { - s.log.Warn("source lookup failed", "error", err) - } - - var ( - source io.ReadCloser - sourceSize int64 - ) - - if sourceKey != "" { - s.log.Debug("using cached source", "source_key", sourceKey) - source, sourceSize = s.loadCachedSource(sourceKey) - } + source, sourceSize := p.loadCachedSource(ctx, req.Source) // Fetch from upstream if we don't have source data or it's empty if source == nil { - return s.fetchAndProcess(ctx, req, cacheKey) + return p.fetchAndProcess(ctx, req, cacheKey) } defer func() { _ = source.Close() }() @@ -306,26 +361,26 @@ func (s *Service) processFromSourceOrFetch( // Process using cached source; nothing was fetched from upstream. The // image processor reads the source only once it has a processing slot, // so a request waiting for one holds none of it in memory. - return s.processAndStore(ctx, req, cacheKey, source, sourceSize) + return p.processAndStore(ctx, req, cacheKey, source, sourceSize) } // fetchAndProcess fetches from upstream, processes, and caches the result. // It counts the fetch with the bytes read from upstream, including when // reading the response or a later step fails. It fetches nothing while the // source URL is in the negative cache. -func (s *Service) fetchAndProcess( +func (p *Proxy) fetchAndProcess( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { // Check negative cache - skip fetching for recently-failed URLs - negHit, err := s.cache.checkNegativeCache(ctx, req) + negHit, err := p.cache.Failure(ctx, req.Source) if err != nil { - s.log.Warn("negative cache check failed", "error", err) + p.log.Warn("negative cache check failed", "error", err) } if negHit { - s.log.Debug("negative cache hit", + p.log.Debug("negative cache hit", "host", req.Source.Host, "path", req.Source.Path, ) @@ -334,14 +389,14 @@ func (s *Service) fetchAndProcess( } // Fetch from upstream - s.log.Debug("fetching from upstream", "host", req.Source.Host, + p.log.Debug("fetching from upstream", "host", req.Source.Host, "path", req.Source.Path, "query", req.Source.Query) - fetchResult, err := s.fetcher.Fetch(ctx, req.Source) + fetchResult, err := p.fetcher.Fetch(ctx, req.Source) if err != nil { // Store negative cache for certain errors if isNegativeCacheable(err) { - s.storeNegative(ctx, req) + p.storeNegative(ctx, req) } return nil, fmt.Errorf("upstream fetch failed: %w", err) @@ -358,7 +413,7 @@ func (s *Service) fetchAndProcess( fetchBytes := int64(len(sourceData)) // Counted also when the request context has ended meanwhile - s.cache.IncrementUpstreamFetch(fetchBytes) + p.incrementUpstreamFetch(fetchBytes) if err != nil { return nil, fmt.Errorf("failed to read upstream response: %w", err) @@ -374,7 +429,7 @@ func (s *Service) fetchAndProcess( } // Log upstream fetch details - s.log.Info("upstream fetched", + p.log.Info("upstream fetched", "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, "path", req.Source.Path, @@ -392,33 +447,44 @@ func (s *Service) fetchAndProcess( // error status does. err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) if err != nil { - s.log.Warn("source refused", "request_id", middleware.GetReqID(ctx), + p.log.Warn("source refused", "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, "path", req.Source.Path, "content_type", fetchResult.ContentType, "error", err) - s.storeNegative(ctx, req) + p.storeNegative(ctx, req) return nil, fmt.Errorf("content validation failed: %w", err) } // Store source content - _, err = s.cache.StoreSource(ctx, req, bytes.NewReader(sourceData)) + _, err = p.cache.PutSource(ctx, req.Source, bytes.NewReader(sourceData)) if err != nil { - s.log.Warn("failed to store source content", "error", err) + p.log.Warn("failed to store source content", "error", err) // Continue even if caching fails } - return s.processAndStore( + return p.processAndStore( ctx, req, cacheKey, bytes.NewReader(sourceData), fetchBytes, ) } +// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes. +// A fetch that read no bytes is not counted. +func (p *Proxy) incrementUpstreamFetch(fetchBytes int64) { + if fetchBytes <= 0 { + return + } + + p.upstreamFetches.Inc() + p.upstreamFetchBytes.Add(float64(fetchBytes)) +} + // storeNegative stores a negative cache entry for req's source, whose fetch // failed, so the source is not fetched again until the entry expires. A // failure to store the entry is only logged. -func (s *Service) storeNegative(ctx context.Context, req *imagerequest.Request) { - err := s.cache.StoreNegative(ctx, req) +func (p *Proxy) storeNegative(ctx context.Context, req *imagerequest.Request) { + err := p.cache.PutFailure(ctx, req.Source) if err != nil { - s.log.Warn("failed to store negative cache entry", + p.log.Warn("failed to store negative cache entry", "host", req.Source.Host, "path", req.Source.Path, "error", err) } } @@ -433,7 +499,7 @@ type processedVariant struct { // processAndStore processes the image read from source and stores the // result. -func (s *Service) processAndStore( +func (p *Proxy) processAndStore( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, @@ -445,7 +511,7 @@ func (s *Service) processAndStore( // ctx has the deadline of the request doing the processing, which can // end while a request waiting for its image has time left - processResult, err := s.processor.Process(ctx, source, &req.Transform) + processResult, err := p.processor.Process(ctx, source, &req.Transform) if errors.Is(err, context.DeadlineExceeded) { return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err) } @@ -457,7 +523,7 @@ func (s *Service) processAndStore( processDuration := time.Since(processStart) // Counted also when the request context has ended meanwhile - s.cache.IncrementTransformCount() + p.transforms.Inc() // Read processed content processedData, err := io.ReadAll(processResult.Content) @@ -476,7 +542,7 @@ func (s *Service) processAndStore( sizePercent = float64(outputSize) / float64(fetchBytes) * 100.0 } - s.log.Info("image converted", + p.log.Info("image converted", "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, "path", req.Source.Path, @@ -495,9 +561,9 @@ func (s *Service) processAndStore( ) // Store variant to cache - err = s.cache.StoreVariant(ctx, cacheKey, bytes.NewReader(processedData)) + err = p.cache.PutVariant(ctx, cacheKey, bytes.NewReader(processedData)) if err != nil { - s.log.Warn("failed to store variant", "error", err) + p.log.Warn("failed to store variant", "error", err) // Continue even if caching fails } diff --git a/internal/imgcache/service_internal_test.go b/internal/imageproxy/imageproxy_internal_test.go similarity index 86% rename from internal/imgcache/service_internal_test.go rename to internal/imageproxy/imageproxy_internal_test.go index ed8ae02..5e392ea 100644 --- a/internal/imgcache/service_internal_test.go +++ b/internal/imageproxy/imageproxy_internal_test.go @@ -1,4 +1,4 @@ -package imgcache +package imageproxy import ( "context" @@ -17,10 +17,10 @@ const ( testPathUpload = "/uploads/image.jpg" ) -func TestService_Get_GoodHost(t *testing.T) { +func TestProxy_Get_GoodHost(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := &imagerequest.Request{ @@ -58,12 +58,12 @@ func TestService_Get_GoodHost(t *testing.T) { } } -// TestService_Get_OtherHost checks that the image service serves a second host -// as it serves GoodHost. -func TestService_Get_OtherHost(t *testing.T) { +// TestProxy_Get_OtherHost checks that the proxy serves a second host as it +// serves GoodHost. +func TestProxy_Get_OtherHost(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := &imagerequest.Request{ @@ -97,10 +97,10 @@ func TestService_Get_OtherHost(t *testing.T) { } } -func TestService_Get_InvalidFile(t *testing.T) { +func TestProxy_Get_InvalidFile(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := photoVariant(fixtures, 85, imagerequest.FitCover) @@ -114,10 +114,10 @@ func TestService_Get_InvalidFile(t *testing.T) { } } -func TestService_Get_NotFound(t *testing.T) { +func TestProxy_Get_NotFound(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := photoVariant(fixtures, 85, imagerequest.FitCover) @@ -130,10 +130,10 @@ func TestService_Get_NotFound(t *testing.T) { } } -func TestService_Get_FormatConversion(t *testing.T) { +func TestProxy_Get_FormatConversion(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() tests := []struct { @@ -213,10 +213,10 @@ func TestService_Get_FormatConversion(t *testing.T) { } } -func TestService_Get_Caching(t *testing.T) { +func TestProxy_Get_Caching(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := &imagerequest.Request{ @@ -272,7 +272,7 @@ func TestService_Get_Caching(t *testing.T) { } } -// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and +// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and // a variant of it and sets back two days every time pixa stored for them. // For the source: last_accessed_at in source_content, and the access and // modification times of its file. For the variant: last_accessed_at in @@ -283,10 +283,10 @@ func TestService_Get_Caching(t *testing.T) { // Left at the present: the files' status change and birth times, which a // program cannot set, and the times of the directories holding the files, // which other entries share. -func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { +func TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) variant := photoVariant(fixtures, 85, imagerequest.FitCover) resp, err := svc.Get(t.Context(), variant) @@ -297,8 +297,8 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { _ = resp.Content.Close() twoDaysAgo := time.Now().AddDate(0, 0, -2) - backdateSource(t, svc.cache, variant, twoDaysAgo) - backdateVariant(t, svc.cache, variant, twoDaysAgo) + backdateSource(t, svc, variant, twoDaysAgo) + backdateVariant(t, svc, variant, twoDaysAgo) resp, err = svc.Get(t.Context(), variant) if err != nil { @@ -320,23 +320,23 @@ func TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { _ = resp.Content.Close() - fetches := readCounters(svc.cache).upstreamFetchCount + fetches := readCounters(t, svc).upstreamFetchCount if fetches != 1 { t.Errorf("upstream fetches = %d, want 1", fetches) } } // backdateSource sets every time pixa stored for req's cached source to when, -// as listed on TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo. +// as listed on TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo. func backdateSource( - t *testing.T, cache *Cache, req *imagerequest.Request, when time.Time, + t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time, ) { t.Helper() key := req.Source.Key() - setSourceLastAccessed(t, cache, key, when) + setSourceLastAccessed(t, svc.db, key, when) - path := cache.srcContent.keyToPath(key) + path := sourceFilePath(svc, key) err := os.Chtimes(path, when, when) if err != nil { @@ -346,16 +346,16 @@ func backdateSource( // backdateVariant sets every time pixa stored for the cached variant req asks // for to when, as listed on -// TestService_Get_ServesSourceAndVariantCachedTwoDaysAgo. +// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo. func backdateVariant( - t *testing.T, cache *Cache, req *imagerequest.Request, when time.Time, + t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time, ) { t.Helper() key := imagerequest.CacheKey(req) - path := cache.variants.keyToPath(key) + path := variantFilePath(svc, key) - setVariantLastAccessed(t, cache, key, when) + setVariantLastAccessed(t, svc.db, key, when) err := os.Chtimes(path, when, when) if err != nil { @@ -363,10 +363,10 @@ func backdateVariant( } } -func TestService_Get_DifferentSizes(t *testing.T) { +func TestProxy_Get_DifferentSizes(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() // Request same image at different sizes @@ -417,10 +417,10 @@ func TestService_Get_DifferentSizes(t *testing.T) { } } -func TestService_Get_ContextCancellation(t *testing.T) { +func TestProxy_Get_ContextCancellation(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately @@ -444,10 +444,10 @@ func TestService_Get_ContextCancellation(t *testing.T) { } } -func TestService_Get_ReturnsETag(t *testing.T) { +func TestProxy_Get_ReturnsETag(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := &imagerequest.Request{ @@ -481,10 +481,10 @@ func TestService_Get_ReturnsETag(t *testing.T) { } } -func TestService_Get_ETagConsistency(t *testing.T) { +func TestProxy_Get_ETagConsistency(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() req := &imagerequest.Request{ @@ -526,10 +526,10 @@ func TestService_Get_ETagConsistency(t *testing.T) { } } -func TestService_Get_DifferentETagsForDifferentContent(t *testing.T) { +func TestProxy_Get_DifferentETagsForDifferentContent(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx := context.Background() // Request same image at different sizes - should get different ETags diff --git a/internal/imgcache/max_concurrent_processing_internal_test.go b/internal/imageproxy/max_concurrent_processing_internal_test.go similarity index 86% rename from internal/imgcache/max_concurrent_processing_internal_test.go rename to internal/imageproxy/max_concurrent_processing_internal_test.go index 1de6011..5d49d16 100644 --- a/internal/imgcache/max_concurrent_processing_internal_test.go +++ b/internal/imageproxy/max_concurrent_processing_internal_test.go @@ -1,4 +1,4 @@ -package imgcache +package imageproxy import ( "image/color" @@ -55,16 +55,16 @@ func holdProcessingSlot( return release } -// TestService_Get_WaitsForSlotBeforeReadingCachedSource checks that a +// TestProxy_Get_WaitsForSlotBeforeReadingCachedSource checks that a // request whose source is cached holds none of it while it waits for a // processing slot: it reads the cached file only once it has a slot. With // the only slot held, a request for a new width of the cached 100x100 photo // waits; the cached file is then rewritten as a 100x50 image before the slot // is freed, so the request must answer with that image scaled to 40x20. -func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { +func TestProxy_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) svc.processor = imageprocessor.New( imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}, ) @@ -77,13 +77,15 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { _ = resp.Content.Close() - sourceKey, err := svc.cache.LookupSource(t.Context(), - widthOnlyRequest(fixtures, 50)) - if err != nil || sourceKey == "" { - t.Fatalf("LookupSource() = %q, %v; want the cached source", - sourceKey, err) + cachedSource := widthOnlyRequest(fixtures, 50).Source + + reader, _, err := svc.cache.Source(t.Context(), cachedSource) + if err != nil { + t.Fatalf("Source() error = %v; want the cached source", err) } + _ = reader.Close() + release := holdProcessingSlot(t, svc.processor) var ( @@ -103,7 +105,7 @@ func TestService_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { // before waiting, it would have read it by now. time.Sleep(100 * time.Millisecond) - err = os.WriteFile(svc.cache.srcContent.keyToPath(sourceKey), + err = os.WriteFile(sourceFilePath(svc, cachedSource.Key()), generateTestJPEG(t, 100, 50, color.RGBA{0, 0, 255, 255}), 0o600) if err != nil { t.Fatalf("failed to rewrite the cached source: %v", err) diff --git a/internal/imageproxy/negative_cache_internal_test.go b/internal/imageproxy/negative_cache_internal_test.go new file mode 100644 index 0000000..23ca916 --- /dev/null +++ b/internal/imageproxy/negative_cache_internal_test.go @@ -0,0 +1,96 @@ +package imageproxy + +import ( + "errors" + "os" + "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// TestProxy_Get_ServesCachedVariantOfNegativeCachedURL caches a variant of +// the test photo, then keeps a failed fetch of the photo's URL in the negative +// cache, as a failed fetch for another variant does once the source has been +// evicted. The cached variant is still served, and a variant not in the cache +// is made from the cached source. Once the source is evicted, a variant not in +// the cache needs a fetch and is refused. +func TestProxy_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + + resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) + if err != nil { + t.Fatalf("Get() first request error = %v", err) + } + + _ = resp.Content.Close() + + err = svc.cache.PutFailure(t.Context(), + photoVariant(fixtures, 85, imagerequest.FitCover).Source) + if err != nil { + t.Fatalf("PutFailure() error = %v", err) + } + + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) + if err != nil { + t.Fatalf("Get() of the cached variant error = %v", err) + } + + _ = resp.Content.Close() + + if resp.CacheStatus != CacheHit { + t.Errorf("cached variant CacheStatus = %q, want %q", + resp.CacheStatus, CacheHit) + } + + resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover)) + t.Logf("Get() of a variant made from the cached source: error = %v", err) + + if err != nil { + t.Fatalf("Get() of a variant made from the cached source error = %v", err) + } + + _ = resp.Content.Close() + + source := photoVariant(fixtures, 85, imagerequest.FitCover).Source + + reader, _, err := svc.cache.Source(t.Context(), source) + if err != nil { + t.Fatalf("Source() error = %v, want the cached source", err) + } + + _ = reader.Close() + + // Evicts the source as the cache's eviction does: its row, then its file + _, err = svc.db.ExecContext(t.Context(), + `DELETE FROM source_content WHERE source_key = ?`, string(source.Key())) + if err != nil { + t.Fatalf("deleting the source's row: %v", err) + } + + err = os.Remove(sourceFilePath(svc, source.Key())) + if err != nil { + t.Fatalf("removing the source's file: %v", err) + } + + _, err = svc.Get(t.Context(), photoVariant(fixtures, 60, imagerequest.FitCover)) + t.Logf("Get() of a variant whose source was evicted: error = %v", err) + + if !errors.Is(err, ErrNegativeCached) { + t.Errorf("Get() of a variant whose source was evicted error = %v, want %v", + err, ErrNegativeCached) + } +} + +func TestProxy_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) { + t.Parallel() + + // This test verifies that Proxy.Get() checks the negative cache + // We can't easily test the full pipeline without vips, but we can + // verify the error type + err := ErrNegativeCached + if !errors.Is(err, ErrNegativeCached) { + t.Error("ErrNegativeCached should be identifiable with errors.Is") + } +} diff --git a/internal/imageproxy/partial_cache_files_internal_test.go b/internal/imageproxy/partial_cache_files_internal_test.go new file mode 100644 index 0000000..f1b7c4f --- /dev/null +++ b/internal/imageproxy/partial_cache_files_internal_test.go @@ -0,0 +1,149 @@ +package imageproxy + +import ( + "bytes" + "io" + "os" + "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// getImage calls Get for req and returns the image it answered with and +// its cache status. +func getImage( + t *testing.T, svc *testProxy, req *imagerequest.Request, +) ([]byte, CacheStatus) { + t.Helper() + + resp, err := svc.Get(t.Context(), req) + if err != nil { + t.Fatalf("Get() of quality %d error = %v", req.Transform.Quality, err) + } + + defer func() { _ = resp.Content.Close() }() + + body, err := io.ReadAll(resp.Content) + if err != nil { + t.Fatalf("reading the image of quality %d: %v", req.Transform.Quality, err) + } + + return body, resp.CacheStatus +} + +// fileSize returns the size of the file at path. +func fileSize(t *testing.T, path string) int64 { + t.Helper() + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + + return info.Size() +} + +// cachedSourcePath returns the path of the cached source file req uses. +func cachedSourcePath( + t *testing.T, svc *testProxy, req *imagerequest.Request, +) string { + t.Helper() + + reader, _, err := svc.cache.Source(t.Context(), req.Source) + if err != nil { + t.Fatalf("Source() error = %v; want the cached source", err) + } + + _ = reader.Close() + + return sourceFilePath(svc, req.Source.Key()) +} + +// TestProxy_Get_FetchesAgainForRemovedCachedSource removes the cached +// source's row and file, as the reconciliation pass does when the file is +// empty or cut short. internal/imgcache tests that the pass does so. The +// test then asks for two new sizes of the source. The first must fetch it +// again and cache it whole, so the second is made from the cache without a +// third fetch. +func TestProxy_Get_FetchesAgainForRemovedCachedSource(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + + getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + + source := photoVariant(fixtures, 85, imagerequest.FitCover).Source + path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + wholeSize := fileSize(t, path) + + // Removes the source as the reconciliation pass does: its row, then its file + _, err := svc.db.ExecContext(t.Context(), + `DELETE FROM source_content WHERE source_key = ?`, string(source.Key())) + if err != nil { + t.Fatalf("deleting the source's row: %v", err) + } + + err = os.Remove(path) + if err != nil { + t.Fatalf("removing the source's file: %v", err) + } + + getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) + getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) + + fetches := readCounters(t, svc).upstreamFetchCount + t.Logf("upstream fetches = %d", fetches) + + if fetches != 2 { + t.Errorf("upstream fetches = %d, want 2", fetches) + } + + size := fileSize(t, path) + if size != wholeSize { + t.Errorf("cached source is %d bytes, want %d", size, wholeSize) + } +} + +// TestProxy_Get_ConvertsAgainForRemovedCachedVariant caches a variant of +// the photo and removes its row and file, as the reconciliation pass does +// when the file is empty or cut short. internal/imgcache tests that the +// pass does so. The test then checks that Get converts the image again, and +// that the next Get serves the whole variant from the cache. +func TestProxy_Get_ConvertsAgainForRemovedCachedVariant(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + req := photoVariant(fixtures, 85, imagerequest.FitCover) + + want, _ := getImage(t, svc, req) + + key := imagerequest.CacheKey(req) + + // Removes the variant as the reconciliation pass does: its row, then its file + _, err := svc.db.ExecContext(t.Context(), + `DELETE FROM variant_content WHERE cache_key = ?`, string(key)) + if err != nil { + t.Fatalf("deleting the variant's row: %v", err) + } + + err = os.Remove(variantFilePath(svc, key)) + if err != nil { + t.Fatalf("removing the variant's file: %v", err) + } + + got, status := getImage(t, svc, req) + t.Logf("after the removal: %d bytes, %s", len(got), status) + + if status != CacheMiss || !bytes.Equal(got, want) { + t.Errorf("after the removal: Get() = %d bytes, %s; want the %d bytes "+ + "converted again, %s", len(got), status, len(want), CacheMiss) + } + + got, status = getImage(t, svc, req) + t.Logf("next request: %d bytes, %s", len(got), status) + + if status != CacheHit || !bytes.Equal(got, want) { + t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s", + len(got), status, len(want), CacheHit) + } +} diff --git a/internal/imageproxy/pendingaccesstimes_internal_test.go b/internal/imageproxy/pendingaccesstimes_internal_test.go new file mode 100644 index 0000000..3fcb1cf --- /dev/null +++ b/internal/imageproxy/pendingaccesstimes_internal_test.go @@ -0,0 +1,78 @@ +package imageproxy + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// pendingAccessTimesWait is how long a test waits for the database's +// connection, for a cache hit to be answered, for an access time to reach +// the database, or for a request to return past its deadline. +const pendingAccessTimesWait = 5 * time.Second + +// holdDatabase takes the one connection of db, so that every other query +// waits for it, and returns the func that frees it. +func holdDatabase(t *testing.T, db *sql.DB) func() { + t.Helper() + + ctx, cancel := context.WithTimeout(t.Context(), pendingAccessTimesWait) + defer cancel() + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("failed to take the database connection: %v", err) + } + + return func() { _ = conn.Close() } +} + +// TestProxy_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the +// database's one connection while a request whose fetch is held reaches its +// deadline. The request must return with the deadline's error while the +// connection is still held, as it waits for no database write past its +// deadline. +func TestProxy_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { + t.Parallel() + + svc, fixtures, fetcher := setupHeldFetchProxy(t) + + svc.cache.StartPendingAccessTimeWrites() + defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }() + + // Room for the request to reach its fetch on a busy host + const timeout = 2 * time.Second + + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + + deadline, _ := ctx.Deadline() + + results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + + // The request has made its database reads by the time it fetches. + select { + case <-fetcher.started: + case <-time.After(timeout): + t.Fatal("request did not reach its fetch by its deadline") + } + + releaseDatabase := holdDatabase(t, svc.db) + defer releaseDatabase() + + select { + case got := <-results: + t.Logf("Get() returned %v after its deadline, error = %v", + time.Since(deadline), got.err) + + if !errors.Is(got.err, context.DeadlineExceeded) { + t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded) + } + case <-time.After(time.Until(deadline) + pendingAccessTimesWait): + t.Fatal("request did not return while the database was busy") + } +} diff --git a/internal/imgcache/stats_internal_test.go b/internal/imageproxy/stats_internal_test.go similarity index 83% rename from internal/imgcache/stats_internal_test.go rename to internal/imageproxy/stats_internal_test.go index 6e4912a..14fa7cf 100644 --- a/internal/imgcache/stats_internal_test.go +++ b/internal/imageproxy/stats_internal_test.go @@ -1,4 +1,4 @@ -package imgcache +package imageproxy import ( "bytes" @@ -12,15 +12,16 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// TestService_Get_CountsStats walks Get through a miss that fetches the +// TestProxy_Get_CountsStats walks Get through a miss that fetches the // source, a hit, a miss that reuses the cached source, and two misses whose -// source cannot be used, checking every counter of the cache after each. -func TestService_Get_CountsStats(t *testing.T) { +// source cannot be used, checking every counter of the proxy and its cache +// after each. +func TestProxy_Get_CountsStats(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) - // NewTestFS builds the same files the test service's fetcher serves. + // NewTestFS builds the same files the test proxy's fetcher serves. testFS, _ := NewTestFS(t) photo, err := fs.ReadFile(testFS, fixtures.GoodHostJPEG) @@ -76,7 +77,7 @@ func TestService_Get_CountsStats(t *testing.T) { _ = resp.Content.Close() } - got := readCounters(svc.cache) + got := readCounters(t, svc) if got != step.want { t.Fatalf("after the %s: counters = %+v, want %+v", step.name, got, step.want) } @@ -114,10 +115,11 @@ func (u *fakeUpstream) Read(p []byte) (int, error) { return 0, u.err } -// TestService_Get_CountsInterruptedMisses checks every counter of the cache -// after a miss whose request context ends during or after the upstream -// fetch, and after a miss whose upstream body is over the size limit. -func TestService_Get_CountsInterruptedMisses(t *testing.T) { +// TestProxy_Get_CountsInterruptedMisses checks every counter of the proxy +// and its cache after a miss whose request context ends during or after the +// upstream fetch, and after a miss whose upstream body is over the size +// limit. +func TestProxy_Get_CountsInterruptedMisses(t *testing.T) { t.Parallel() photo := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) @@ -145,7 +147,7 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() @@ -182,7 +184,7 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { _ = resp.Content.Close() } - got := readCounters(svc.cache) + got := readCounters(t, svc) if got != tc.want { t.Errorf("counters = %+v, want %+v", got, tc.want) } @@ -190,13 +192,13 @@ func TestService_Get_CountsInterruptedMisses(t *testing.T) { } } -// TestService_Get_CountsHitAfterRequestEnds checks every counter of the -// cache after a hit served with a request context that has already ended: -// only the hit count moves. -func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { +// TestProxy_Get_CountsHitAfterRequestEnds checks every counter of the +// proxy and its cache after a hit served with a request context that has +// already ended: only the hit count moves. +func TestProxy_Get_CountsHitAfterRequestEnds(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + svc, fixtures := setupTestProxy(t) req := &imagerequest.Request{ Source: imagerequest.Source{ @@ -219,7 +221,7 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { _ = resp.Content.Close() - want := readCounters(svc.cache) + want := readCounters(t, svc) want.hitCount++ ctx, cancel := context.WithCancel(t.Context()) @@ -236,7 +238,7 @@ func TestService_Get_CountsHitAfterRequestEnds(t *testing.T) { t.Fatalf("CacheStatus = %v, want %v", resp.CacheStatus, CacheHit) } - got := readCounters(svc.cache) + got := readCounters(t, svc) if got != want { t.Errorf("counters = %+v, want %+v", got, want) } diff --git a/internal/imageproxy/testutil_internal_test.go b/internal/imageproxy/testutil_internal_test.go new file mode 100644 index 0000000..8f0229f --- /dev/null +++ b/internal/imageproxy/testutil_internal_test.go @@ -0,0 +1,355 @@ +package imageproxy + +import ( + "bytes" + "context" + "database/sql" + "image" + "image/color" + "image/gif" + "image/jpeg" + "image/png" + "io/fs" + "path/filepath" + "testing" + "testing/fstest" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/database" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/imgcache" +) + +// Shared test data literals, extracted as constants for goconst. +const ( + testContentTypeJPEG = "image/jpeg" +) + +// sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP +// produces, so injected timestamps compare correctly against ones the +// cache writes. +const sqliteTimestampFormat = "2006-01-02 15:04:05" + +// TestFixtures contains paths to test files in the mock filesystem. +type TestFixtures struct { + // Valid image files + GoodHostJPEG string // valid JPEG on GoodHost + GoodHostPNG string // valid PNG on GoodHost + GoodHostGIF string // valid GIF on GoodHost + OtherHostJPEG string // valid JPEG on OtherHost + OtherHostPNG string // valid PNG on OtherHost + + // Invalid/edge case files + InvalidFile string // file with wrong magic bytes + EmptyFile string // zero-byte file + TextFile string // text file masquerading as image + + // Hostnames + GoodHost string // a host the mock fetcher serves + OtherHost string // a second host the mock fetcher serves +} + +// DefaultFixtures returns the standard test fixture paths. +func DefaultFixtures() *TestFixtures { + return &TestFixtures{ + GoodHostJPEG: "goodhost.example.com/images/photo.jpg", + GoodHostPNG: "goodhost.example.com/images/logo.png", + GoodHostGIF: "goodhost.example.com/images/animation.gif", + OtherHostJPEG: "otherhost.example.com/uploads/image.jpg", + OtherHostPNG: "otherhost.example.com/uploads/icon.png", + InvalidFile: "goodhost.example.com/images/fake.jpg", + EmptyFile: "goodhost.example.com/images/empty.jpg", + TextFile: "goodhost.example.com/images/text.png", + GoodHost: "goodhost.example.com", + OtherHost: "otherhost.example.com", + } +} + +// NewTestFS creates a mock filesystem with test images. +func NewTestFS(t *testing.T) (fs.FS, *TestFixtures) { + t.Helper() + + fixtures := DefaultFixtures() + + // Generate test images + jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) + pngData := generateTestPNG(t, 100, 100, color.RGBA{0, 255, 0, 255}) + gifData := generateTestGIF(t, 100, 100, color.RGBA{0, 0, 255, 255}) + + // Create the mock filesystem + mockFS := fstest.MapFS{ + // Good host files + fixtures.GoodHostJPEG: &fstest.MapFile{Data: jpegData}, + fixtures.GoodHostPNG: &fstest.MapFile{Data: pngData}, + fixtures.GoodHostGIF: &fstest.MapFile{Data: gifData}, + + // Other host files + fixtures.OtherHostJPEG: &fstest.MapFile{Data: jpegData}, + fixtures.OtherHostPNG: &fstest.MapFile{Data: pngData}, + + // Invalid files + fixtures.InvalidFile: &fstest.MapFile{Data: []byte("not a real image file content")}, + fixtures.EmptyFile: &fstest.MapFile{Data: []byte{}}, + fixtures.TextFile: &fstest.MapFile{Data: []byte("This is just text, not a PNG")}, + } + + return mockFS, fixtures +} + +// generateTestJPEG creates a minimal valid JPEG image. +func generateTestJPEG(t *testing.T, width, height int, c color.Color) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + img.Set(x, y, c) + } + } + + var buf bytes.Buffer + + err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}) + if err != nil { + t.Fatalf("failed to encode test JPEG: %v", err) + } + + return buf.Bytes() +} + +// generateTestPNG creates a minimal valid PNG image. +func generateTestPNG(t *testing.T, width, height int, c color.Color) []byte { + t.Helper() + + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + img.Set(x, y, c) + } + } + + var buf bytes.Buffer + + err := png.Encode(&buf, img) + if err != nil { + t.Fatalf("failed to encode test PNG: %v", err) + } + + return buf.Bytes() +} + +// generateTestGIF creates a minimal valid GIF image. +func generateTestGIF(t *testing.T, width, height int, c color.Color) []byte { + t.Helper() + + img := image.NewPaletted( + image.Rect(0, 0, width, height), + []color.Color{c, color.White}, + ) + for y := range height { + for x := range width { + img.SetColorIndex(x, y, 0) + } + } + + var buf bytes.Buffer + + err := gif.Encode(&buf, img, nil) + if err != nil { + t.Fatalf("failed to encode test GIF: %v", err) + } + + return buf.Bytes() +} + +// testProxy is a Proxy with the database and the state directory of its +// cache, through which a test reads and changes the cache behind the proxy. +// registry is the metrics registry the proxy and its cache share. +type testProxy struct { + *Proxy + + db *sql.DB + stateDir string + registry *prometheus.Registry +} + +// setupTestProxy creates a Proxy with mock fetcher for testing. +func setupTestProxy(t *testing.T) (*testProxy, *TestFixtures) { + t.Helper() + + mockFS, fixtures := NewTestFS(t) + + // Create temp directory for cache + tmpDir := t.TempDir() + + // Create in-memory database + db := setupTestDB(t) + + registry := prometheus.NewRegistry() + + cache, err := imgcache.NewCache(db, imgcache.Config{ + StateDir: tmpDir, + NegativeTTL: 5 * time.Minute, + MaxBytes: 1 << 30, + }, registry) + if err != nil { + t.Fatalf("failed to create cache: %v", err) + } + + proxy, err := New(&Config{ + Cache: cache, + Fetcher: httpfetcher.NewMock(mockFS), + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + MaxResponseSize: 50 << 20, + }, registry) + if err != nil { + t.Fatalf("failed to create proxy: %v", err) + } + + return &testProxy{ + Proxy: proxy, + db: db, + stateDir: tmpDir, + registry: registry, + }, fixtures +} + +// counterValues holds the values of the counters of a proxy and its cache. +type counterValues struct { + hitCount int64 + missCount int64 + upstreamFetchCount int64 + upstreamFetchBytes int64 + transformCount int64 +} + +// readCounters reads the counters of svc and its cache from their metrics +// registry. +func readCounters(t *testing.T, svc *testProxy) counterValues { + t.Helper() + + families, err := svc.registry.Gather() + if err != nil { + t.Fatalf("failed to gather the counters: %v", err) + } + + values := make(map[string]int64, len(families)) + + for _, family := range families { + values[family.GetName()] = int64(family.GetMetric()[0].GetCounter().GetValue()) + } + + return counterValues{ + hitCount: values["pixa_cache_hits_total"], + missCount: values["pixa_cache_misses_total"], + upstreamFetchCount: values["pixa_upstream_fetches_total"], + upstreamFetchBytes: values["pixa_upstream_fetch_bytes_total"], + transformCount: values["pixa_transforms_total"], + } +} + +// sourceFilePath returns the path of the file of the source key in svc's +// cache, as README.md gives it: /cache/sources///. +func sourceFilePath(svc *testProxy, key imagerequest.SourceKey) string { + name := string(key) + + return filepath.Join(svc.stateDir, "cache", "sources", name[0:2], name[2:4], name) +} + +// variantFilePath returns the path of the file of the variant key in svc's +// cache, as README.md gives it: /cache/variants///. +func variantFilePath(svc *testProxy, key imagerequest.VariantKey) string { + name := string(key) + + return filepath.Join(svc.stateDir, "cache", "variants", name[0:2], name[2:4], name) +} + +// setVariantLastAccessed backdates the last access time of a tracked +// variant. +func setVariantLastAccessed( + t *testing.T, db *sql.DB, key imagerequest.VariantKey, when time.Time, +) { + t.Helper() + + res, err := db.ExecContext(t.Context(), + `UPDATE variant_content SET last_accessed_at = ? WHERE cache_key = ?`, + when.UTC().Format(sqliteTimestampFormat), string(key), + ) + if err != nil { + t.Fatalf("failed to set variant last_accessed_at: %v", err) + } + + affected, err := res.RowsAffected() + if err != nil { + t.Fatalf("failed to read affected rows: %v", err) + } + + if affected != 1 { + t.Fatalf("variant %s has no accounting row (affected=%d); "+ + "stores must track variants in the database", key, affected) + } +} + +// setSourceLastAccessed backdates the last access time of a tracked +// source content blob. +func setSourceLastAccessed( + t *testing.T, db *sql.DB, key imagerequest.SourceKey, when time.Time, +) { + t.Helper() + + res, err := db.ExecContext(t.Context(), + `UPDATE source_content SET last_accessed_at = ? WHERE source_key = ?`, + when.UTC().Format(sqliteTimestampFormat), string(key), + ) + if err != nil { + t.Fatalf("failed to set source last_accessed_at: %v", err) + } + + affected, err := res.RowsAffected() + if err != nil { + t.Fatalf("failed to read affected rows: %v", err) + } + + if affected != 1 { + t.Fatalf("source %s has no accounting row (affected=%d)", key, affected) + } +} + +// countRows returns the number of rows the given query yields. +func countRows(t *testing.T, db *sql.DB, query string, args ...any) int { + t.Helper() + + var n int + + err := db.QueryRowContext(t.Context(), query, args...).Scan(&n) + if err != nil { + t.Fatalf("count query %q failed: %v", query, err) + } + + return n +} + +// setupTestDB creates an in-memory SQLite database for testing +// using the production schema. +func setupTestDB(t *testing.T) *sql.DB { + t.Helper() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("failed to open test db: %v", err) + } + + // Use the real production schema via migrations + err = database.ApplyMigrations(context.Background(), db, nil) + if err != nil { + t.Fatalf("failed to apply migrations: %v", err) + } + + return db +} diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index e1b22a7..ab5cbcf 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -1,3 +1,10 @@ +// Package imgcache is the image cache. It keeps: +// - the source images fetched from upstream +// - the variants made from them +// - the negative cache: the source URLs whose fetch failed recently +// +// Each image is a file with a row in the database. Eviction removes the +// least recently used images to keep the cache under its size limit. package imgcache import ( @@ -26,8 +33,12 @@ const maxPendingAccessTimes = 10000 // have expired. const cleanExpiredBatchSize = 1000 -// CacheConfig holds cache configuration. -type CacheConfig struct { +// DefaultNegativeTTL is how long pixad keeps a failed fetch in the negative +// cache. +const DefaultNegativeTTL = 5 * time.Minute + +// Config holds cache configuration. +type Config struct { StateDir string NegativeTTL time.Duration @@ -48,7 +59,7 @@ type Cache struct { db *sql.DB srcContent *ContentStorage // source images by source key variants *VariantStorage // processed variants by cache key - config CacheConfig + config Config log *slog.Logger // Eviction machinery. The channels are created in NewCache so @@ -73,13 +84,10 @@ type Cache struct { pendingAccessTimesDone chan struct{} pendingAccessTimesCancel context.CancelFunc - // The counts of what the cache does, made and registered by + // The counts of cache hits and misses, made and registered by // registerCounters (see counters.go) - cacheHits prometheus.Counter - cacheMisses prometheus.Counter - upstreamFetches prometheus.Counter - upstreamFetchBytes prometheus.Counter - transforms prometheus.Counter + cacheHits prometheus.Counter + cacheMisses prometheus.Counter // reconciliationPageSize is the most rows one read of a content table // returns in the reconciliation pass. NewCache sets it to @@ -97,7 +105,7 @@ type Cache struct { // metricsRegistry. pixad passes Prometheus' default registry, the one // /metrics serves. It makes no database call. func NewCache( - db *sql.DB, config CacheConfig, metricsRegistry prometheus.Registerer, + db *sql.DB, config Config, metricsRegistry prometheus.Registerer, ) (*Cache, error) { log := config.Logger if log == nil { @@ -147,75 +155,35 @@ func NewCache( return c, nil } -// GetVariant opens a cached variant and returns a reader, its size and its +// Variant opens a cached variant and returns a reader, its size and its // content type, read from the variant file's first bytes. It opens that one -// file and makes no database call. A variant not in the cache, or any -// variant of a disabled cache, is ErrNotFound; a variant file whose first -// bytes are not those of an output format is another error. -func (c *Cache) GetVariant( +// file and makes no database call. A variant it opens has its access time +// handed to the background writer without waiting (see touchVariant). A +// variant not in the cache, or any variant of a disabled cache, is +// ErrNotFound; a variant file whose first bytes are not those of an output +// format is another error. +func (c *Cache) Variant( cacheKey imagerequest.VariantKey, ) (io.ReadCloser, int64, string, error) { if c.config.MaxBytes == 0 { return nil, 0, "", ErrNotFound } - return c.variants.LoadWithContentType(cacheKey) + reader, size, contentType, err := c.variants.LoadWithContentType(cacheKey) + if err != nil { + return nil, 0, "", err + } + + c.touchVariant(cacheKey) + + return reader, size, contentType, nil } -// StoreSource stores fetched source content: it writes the file named by -// the source's key, then the source's row. When the row cannot be written -// the file is removed and the store fails, as a file with no row is never -// served or evicted; if that removal fails too, the next reconciliation -// pass removes the file. On a disabled cache it is a no-op returning an -// empty key. -func (c *Cache) StoreSource( - ctx context.Context, - req *imagerequest.Request, - content io.Reader, -) (imagerequest.SourceKey, error) { - if c.config.MaxBytes == 0 { - return "", nil - } - - data, err := io.ReadAll(content) - if err != nil { - return "", fmt.Errorf("failed to read source content: %w", err) - } - - key := req.Source.Key() - - size, err := c.srcContent.Store(key, data) - if err != nil { - return "", fmt.Errorf("failed to store source content: %w", err) - } - - _, err = c.db.ExecContext(ctx, ` - INSERT INTO source_content (source_key, size_bytes) - VALUES (?, ?) - ON CONFLICT(source_key) DO UPDATE SET - size_bytes = excluded.size_bytes, - last_accessed_at = CURRENT_TIMESTAMP - `, string(key), size) - if err != nil { - deleteErr := c.srcContent.Delete(key) - if deleteErr != nil { - c.log.Warn("failed to remove source file whose row was not written", - "source_key", key, "error", deleteErr) - } - - return "", fmt.Errorf("failed to insert source content: %w", err) - } - - c.notifyWritePressure() - - return key, nil -} - -// StoreVariant stores a processed variant by its cache key and records +// PutVariant stores a processed variant by its cache key and records // it in the size accounting. On a disabled cache it is a no-op. The // accounting insert is best-effort (the startup reconciliation pass // adopts any variant file that misses its accounting row). -func (c *Cache) StoreVariant( +func (c *Cache) PutVariant( ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader, ) error { if c.config.MaxBytes == 0 { @@ -244,19 +212,18 @@ func (c *Cache) StoreVariant( return nil } -// LookupSource checks if we have cached source content for a request. -// Returns the source key if found, or an empty key if not; a row whose -// file is missing is not found. A source found has its access time -// handed to the background writer without waiting (see -// touchSourceContent); a disabled cache always reports no cached source. -func (c *Cache) LookupSource( - ctx context.Context, req *imagerequest.Request, -) (imagerequest.SourceKey, error) { +// Source opens the cached source of src and returns a reader and its size +// in bytes. A source whose row or file is missing is ErrNotFound, as is any +// source of a disabled cache. A source it opens has its access time handed +// to the background writer without waiting (see touchSourceContent). +func (c *Cache) Source( + ctx context.Context, src imagerequest.Source, +) (io.ReadCloser, int64, error) { if c.config.MaxBytes == 0 { - return "", nil + return nil, 0, ErrNotFound } - key := req.Source.Key() + key := src.Key() var rowExists bool @@ -264,25 +231,100 @@ func (c *Cache) LookupSource( SELECT EXISTS(SELECT 1 FROM source_content WHERE source_key = ?) `, string(key)).Scan(&rowExists) if err != nil { - return "", fmt.Errorf("failed to lookup source: %w", err) + return nil, 0, fmt.Errorf("failed to lookup source: %w", err) } if !rowExists { - return "", nil + return nil, 0, ErrNotFound } - // Verify the content file exists - if !c.srcContent.Exists(key) { - return "", nil + reader, size, err := c.srcContent.LoadWithSize(key) + if err != nil { + return nil, 0, err } c.touchSourceContent(key) + return reader, size, nil +} + +// PutSource stores fetched source content: it writes the file named by +// the source's key, then the source's row. When the row cannot be written +// the file is removed and the store fails, as a file with no row is never +// served or evicted; if that removal fails too, the next reconciliation +// pass removes the file. On a disabled cache it is a no-op returning an +// empty key. +func (c *Cache) PutSource( + ctx context.Context, + src imagerequest.Source, + content io.Reader, +) (imagerequest.SourceKey, error) { + if c.config.MaxBytes == 0 { + return "", nil + } + + data, err := io.ReadAll(content) + if err != nil { + return "", fmt.Errorf("failed to read source content: %w", err) + } + + key := src.Key() + + size, err := c.srcContent.Store(key, data) + if err != nil { + return "", fmt.Errorf("failed to store source content: %w", err) + } + + _, err = c.db.ExecContext(ctx, ` + INSERT INTO source_content (source_key, size_bytes) + VALUES (?, ?) + ON CONFLICT(source_key) DO UPDATE SET + size_bytes = excluded.size_bytes, + last_accessed_at = CURRENT_TIMESTAMP + `, string(key), size) + if err != nil { + deleteErr := c.srcContent.Delete(key) + if deleteErr != nil { + c.log.Warn("failed to remove source file whose row was not written", + "source_key", key, "error", deleteErr) + } + + return "", fmt.Errorf("failed to insert source content: %w", err) + } + + c.notifyWritePressure() + return key, nil } -// StoreNegative stores a negative cache entry for a failed fetch. -func (c *Cache) StoreNegative(ctx context.Context, req *imagerequest.Request) error { +// Failure reports whether src is in the negative cache. It only reads: a +// row past its expiry is not a hit, and the eviction goroutine deletes it +// with the other expired rows (see CleanExpired). +func (c *Cache) Failure( + ctx context.Context, src imagerequest.Source, +) (bool, error) { + var expiresAt time.Time + + err := c.db.QueryRowContext(ctx, ` + SELECT expires_at FROM negative_cache + WHERE source_host = ? AND source_path = ? AND source_query = ? + `, src.Host, src.Path, src.Query).Scan(&expiresAt) + + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("failed to check negative cache: %w", err) + } + + expired := time.Now().After(expiresAt) + + return !expired, nil +} + +// PutFailure stores a negative cache entry for src, whose fetch failed. +func (c *Cache) PutFailure(ctx context.Context, src imagerequest.Source) error { expiresAt := time.Now().UTC().Add(c.config.NegativeTTL) _, err := c.db.ExecContext(ctx, ` @@ -291,7 +333,7 @@ func (c *Cache) StoreNegative(ctx context.Context, req *imagerequest.Request) er VALUES (?, ?, ?, ?) ON CONFLICT(source_host, source_path, source_query) DO UPDATE SET expires_at = excluded.expires_at - `, req.Source.Host, req.Source.Path, req.Source.Query, expiresAt) + `, src.Host, src.Path, src.Query, expiresAt) if err != nil { return fmt.Errorf("failed to insert negative cache: %w", err) } @@ -299,18 +341,6 @@ func (c *Cache) StoreNegative(ctx context.Context, req *imagerequest.Request) er return nil } -// GetSourceContent returns a reader for cached source content by its key, -// and the content's size in bytes. -func (c *Cache) GetSourceContent( - key imagerequest.SourceKey, -) (io.ReadCloser, int64, error) { - if c.config.MaxBytes == 0 { - return nil, 0, ErrNotFound - } - - return c.srcContent.LoadWithSize(key) -} - // CleanExpired removes the negative cache rows past their expiry, one // batch per statement, until none is left. Once ctx ends it starts no // further batch and returns ctx's error. The eviction goroutine calls it @@ -334,34 +364,15 @@ func (c *Cache) CleanExpired(ctx context.Context) error { } } -// IncrementStats counts a cache hit or miss, and an upstream fetch that read -// fetchBytes bytes, as IncrementUpstreamFetch does. Like the other Increment -// methods, it only adds to the cache's Prometheus counters in memory (see -// counters.go), so it never waits for the database or fails. -func (c *Cache) IncrementStats(hit bool, fetchBytes int64) { +// IncrementStats counts a cache hit or miss. It only adds to the cache's +// Prometheus counters in memory (see counters.go), so it never waits for the +// database or fails. +func (c *Cache) IncrementStats(hit bool) { if hit { c.cacheHits.Inc() } else { c.cacheMisses.Inc() } - - c.IncrementUpstreamFetch(fetchBytes) -} - -// IncrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes. -// A fetch that read no bytes is not counted. -func (c *Cache) IncrementUpstreamFetch(fetchBytes int64) { - if fetchBytes <= 0 { - return - } - - c.upstreamFetches.Inc() - c.upstreamFetchBytes.Add(float64(fetchBytes)) -} - -// IncrementTransformCount counts one image transcoded by the image processor. -func (c *Cache) IncrementTransformCount() { - c.transforms.Inc() } // touchVariant hands the access time of a variant served from the cache to @@ -388,32 +399,6 @@ func (c *Cache) touchSourceContent(key imagerequest.SourceKey) { } } -// checkNegativeCache checks if a request is in the negative cache. It only -// reads: a row past its expiry is not a hit, and the eviction goroutine -// deletes it with the other expired rows (see CleanExpired). -func (c *Cache) checkNegativeCache( - ctx context.Context, req *imagerequest.Request, -) (bool, error) { - var expiresAt time.Time - - err := c.db.QueryRowContext(ctx, ` - SELECT expires_at FROM negative_cache - WHERE source_host = ? AND source_path = ? AND source_query = ? - `, req.Source.Host, req.Source.Path, req.Source.Query).Scan(&expiresAt) - - if errors.Is(err, sql.ErrNoRows) { - return false, nil - } - - if err != nil { - return false, fmt.Errorf("failed to check negative cache: %w", err) - } - - expired := time.Now().After(expiresAt) - - return !expired, nil -} - // cleanExpiredBatch deletes up to cleanExpiredBatchSize negative cache // rows past their expiry in one statement, and returns how many it // deleted. DELETE ... LIMIT needs a SQLite build option, so the subquery diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index d4a3c76..068bce0 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -6,7 +6,6 @@ import ( "fmt" "image/color" "io" - "os" "testing" "time" @@ -15,10 +14,10 @@ import ( "sneak.berlin/go/pixa/internal/magic" ) -// TestGetVariant_ContentTypeFromFirstBytes stores an image in each output -// format as a variant and checks that GetVariant serves it whole with the +// TestVariant_ContentTypeFromFirstBytes stores an image in each output +// format as a variant and checks that Variant serves it whole with the // content type of its format, read from the file. -func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { +func TestVariant_ContentTypeFromFirstBytes(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) @@ -50,14 +49,14 @@ func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { key := imagerequest.VariantKey("aabbccdd10" + string(format)) - err = cache.StoreVariant(t.Context(), key, bytes.NewReader(image)) + err = cache.PutVariant(t.Context(), key, bytes.NewReader(image)) if err != nil { - t.Fatalf("StoreVariant(%s) error = %v", format, err) + t.Fatalf("PutVariant(%s) error = %v", format, err) } - reader, size, contentType, err := cache.GetVariant(key) + reader, size, contentType, err := cache.Variant(key) if err != nil { - t.Fatalf("GetVariant(%s) error = %v", format, err) + t.Fatalf("Variant(%s) error = %v", format, err) } served, err := io.ReadAll(reader) @@ -80,11 +79,11 @@ func TestGetVariant_ContentTypeFromFirstBytes(t *testing.T) { } } -// TestGetVariant_RefusesFileOfNoOutputFormat checks that GetVariant refuses +// TestVariant_RefusesFileOfNoOutputFormat checks that Variant refuses // a variant file whose first bytes are not those of an output format, with // an error other than ErrNotFound, and that a variant with no file, or any // variant of a disabled cache, is ErrNotFound. -func TestGetVariant_RefusesFileOfNoOutputFormat(t *testing.T) { +func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) @@ -110,118 +109,30 @@ func TestGetVariant_RefusesFileOfNoOutputFormat(t *testing.T) { t.Fatalf("%s: storing the variant file: %v", tt.name, err) } - reader, _, contentType, err := cache.GetVariant(key) - t.Logf("%s: GetVariant() error = %v", tt.name, err) + reader, _, contentType, err := cache.Variant(key) + t.Logf("%s: Variant() error = %v", tt.name, err) if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) { - t.Errorf("%s: GetVariant() = reader %v, content type %q, error %v; "+ + t.Errorf("%s: Variant() = reader %v, content type %q, error %v; "+ "want no reader and %v", tt.name, reader, contentType, err, tt.wantErr) } } - reader, _, _, err := cache.GetVariant(testVariantKeyFour) + reader, _, _, err := cache.Variant(testVariantKeyFour) if reader != nil || !errors.Is(err, ErrNotFound) { - t.Errorf("GetVariant() of a variant with no file = reader %v, error %v; "+ + t.Errorf("Variant() of a variant with no file = reader %v, error %v; "+ "want no reader and %v", reader, err, ErrNotFound) } disabled, _ := newEvictionTestCache(t, 0) - reader, _, _, err = disabled.GetVariant(testVariantKeyOne) + reader, _, _, err = disabled.Variant(testVariantKeyOne) if reader != nil || !errors.Is(err, ErrNotFound) { - t.Errorf("GetVariant() on a disabled cache = reader %v, error %v; "+ + t.Errorf("Variant() on a disabled cache = reader %v, error %v; "+ "want no reader and %v", reader, err, ErrNotFound) } } -// TestService_Get_MakesAgainVariantOfNoOutputFormat replaces a cached -// variant's file with text, and checks that the next request is a miss -// answered with the variant made again, and the one after a hit. -func TestService_Get_MakesAgainVariantOfNoOutputFormat(t *testing.T) { - t.Parallel() - - svc, fixtures := SetupTestService(t) - req := photoVariant(fixtures, 85, imagerequest.FitCover) - - want, _ := getImage(t, svc, req) - - path := svc.cache.variants.keyToPath(imagerequest.CacheKey(req)) - - err := os.WriteFile(path, []byte("this is text, not an image"), StorageFilePerm) - if err != nil { - t.Fatalf("replacing the variant file: %v", err) - } - - got, status := getImage(t, svc, req) - t.Logf("after the replacement: %d bytes, %s", len(got), status) - - if status != CacheMiss || !bytes.Equal(got, want) { - t.Errorf("after the replacement: Get() = %d bytes, %s; want the %d bytes "+ - "made again, %s", len(got), status, len(want), CacheMiss) - } - - got, status = getImage(t, svc, req) - - if status != CacheHit || !bytes.Equal(got, want) { - t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s", - len(got), status, len(want), CacheHit) - } -} - -// TestService_Get_HitWaitsForNoDatabaseWrite holds the database's one -// connection while a cached variant is requested. The hit must be answered -// while the connection is still held, and its access time must reach the -// database once it is free. -func TestService_Get_HitWaitsForNoDatabaseWrite(t *testing.T) { - t.Parallel() - - svc, fixtures := SetupTestService(t) - req := photoVariant(fixtures, 85, imagerequest.FitCover) - key := imagerequest.CacheKey(req) - - getImage(t, svc, req) - - // Each connection to an in-memory SQLite database opens a new, empty one, - // so the goroutine that writes the access times must share the - // connection that holds the schema. - svc.cache.db.SetMaxOpenConns(1) - svc.cache.StartPendingAccessTimeWrites() - - defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }() - - anHourAgo := time.Now().Add(-time.Hour) - setVariantLastAccessed(t, svc.cache, key, anHourAgo) - - releaseDatabase := holdDatabase(t, svc.cache) - defer releaseDatabase() - - results := startGet(t.Context(), svc, req) - - select { - case got := <-results: - if got.err != nil { - t.Fatalf("Get() of the cached variant error = %v", got.err) - } - case <-time.After(pendingAccessTimesWait): - t.Fatal("the cache hit waited for the database") - } - - releaseDatabase() - - deadline := time.Now().Add(pendingAccessTimesWait) - - for countRows(t, svc.cache, ` - SELECT COUNT(*) FROM variant_content - WHERE cache_key = ? AND last_accessed_at > ? - `, string(key), anHourAgo.UTC().Format(sqliteTimestampFormat)) == 0 { - if time.Now().After(deadline) { - t.Fatal("the hit's access time was not written") - } - - time.Sleep(10 * time.Millisecond) - } -} - // TestTouchVariantAndTouchSourceContent_DropAccessTimesPastTheLimit hands one // more access time of a variant, and one more of a source, than may wait to // be written. touchVariant and touchSourceContent must return each time diff --git a/internal/imgcache/cache_internal_test.go b/internal/imgcache/cache_internal_test.go index b37af29..0a1cf9b 100644 --- a/internal/imgcache/cache_internal_test.go +++ b/internal/imgcache/cache_internal_test.go @@ -50,7 +50,7 @@ func TestCache_CleanExpired(t *testing.T) { tmpDir := t.TempDir() db := setupTestDB(t) - cache, _ := NewCache(db, CacheConfig{ + cache, _ := NewCache(db, Config{ StateDir: tmpDir, NegativeTTL: 1 * time.Millisecond, MaxBytes: 1 << 30, @@ -103,7 +103,7 @@ func TestCache_StorageDirectoriesCreated(t *testing.T) { tmpDir := t.TempDir() db := setupTestDB(t) - _, err := NewCache(db, CacheConfig{ + _, err := NewCache(db, Config{ StateDir: tmpDir, MaxBytes: 1 << 30, }, prometheus.NewRegistry()) diff --git a/internal/imgcache/counters.go b/internal/imgcache/counters.go index 8e50780..edb14ac 100644 --- a/internal/imgcache/counters.go +++ b/internal/imgcache/counters.go @@ -19,22 +19,8 @@ func (c *Cache) registerCounters(metricsRegistry prometheus.Registerer) error { Name: "pixa_cache_misses_total", Help: "Image requests not served from the cache.", }) - c.upstreamFetches = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "pixa_upstream_fetches_total", - Help: "Fetches of a source image from upstream that read any bytes.", - }) - c.upstreamFetchBytes = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "pixa_upstream_fetch_bytes_total", - Help: "Bytes read from upstream by fetches of source images.", - }) - c.transforms = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "pixa_transforms_total", - Help: "Images the image processor transformed.", - }) - for _, counter := range []prometheus.Counter{ - c.cacheHits, c.cacheMisses, c.upstreamFetches, c.upstreamFetchBytes, c.transforms, - } { + for _, counter := range []prometheus.Counter{c.cacheHits, c.cacheMisses} { err := metricsRegistry.Register(counter) if err != nil { return fmt.Errorf("failed to register the cache's counters: %w", err) diff --git a/internal/imgcache/counters_internal_test.go b/internal/imgcache/counters_internal_test.go index cb11aa3..0fd62f3 100644 --- a/internal/imgcache/counters_internal_test.go +++ b/internal/imgcache/counters_internal_test.go @@ -8,26 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" ) -// counterValues holds the values of a cache's counters. -type counterValues struct { - hitCount int64 - missCount int64 - upstreamFetchCount int64 - upstreamFetchBytes int64 - transformCount int64 -} - -// readCounters reads cache's counters. -func readCounters(cache *Cache) counterValues { - return counterValues{ - hitCount: int64(testutil.ToFloat64(cache.cacheHits)), - missCount: int64(testutil.ToFloat64(cache.cacheMisses)), - upstreamFetchCount: int64(testutil.ToFloat64(cache.upstreamFetches)), - upstreamFetchBytes: int64(testutil.ToFloat64(cache.upstreamFetchBytes)), - transformCount: int64(testutil.ToFloat64(cache.transforms)), - } -} - // TestNewCache_RegistersItsCountersOnItsRegistry checks that NewCache // registers the cache's counters on the registry it is given, under the // names README.md gives them. @@ -37,7 +17,7 @@ func TestNewCache_RegistersItsCountersOnItsRegistry(t *testing.T) { registry := prometheus.NewRegistry() _, err := NewCache(evictionTestDB(t), - CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) if err != nil { t.Fatalf("NewCache() error = %v", err) } @@ -45,9 +25,6 @@ func TestNewCache_RegistersItsCountersOnItsRegistry(t *testing.T) { names := []string{ "pixa_cache_hits_total", "pixa_cache_misses_total", - "pixa_upstream_fetches_total", - "pixa_upstream_fetch_bytes_total", - "pixa_transforms_total", } count, err := testutil.GatherAndCount(registry, names...) @@ -70,13 +47,13 @@ func TestNewCache_FailsWhenItsCountersCannotBeRegistered(t *testing.T) { registry := prometheus.NewRegistry() _, err := NewCache(evictionTestDB(t), - CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) if err != nil { t.Fatalf("NewCache() of the first cache error = %v", err) } _, err = NewCache(evictionTestDB(t), - CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) t.Logf("NewCache() of the second cache error = %v", err) var alreadyRegistered prometheus.AlreadyRegisteredError diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 2b1feb0..8b576ad 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -317,7 +317,7 @@ func (c *Cache) evictVariant( // evictSourceBlob removes one source: its row first, then its file, as // evictVariant does for a variant. It holds no lock: a store of the same // source in between can leave the source's row with no file, which -// LookupSource treats as a miss. +// Source treats as a miss. func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error { _, err := c.db.ExecContext(ctx, `DELETE FROM source_content WHERE source_key = ?`, string(key)) @@ -479,7 +479,7 @@ func (c *Cache) runCleanExpired(ctx context.Context) { // then on the first periodic eviction tick that comes at least ten times // as long after the last pass ended as that pass took, off the request // hot path: it adopts variant files that predate the accounting table -// (or whose accounting insert failed, e.g. StoreVariant's best-effort +// (or whose accounting insert failed, e.g. PutVariant's best-effort // insert under transient DB contention), drops variant accounting rows // whose files are missing, removes source blob files the database does // not know, removes source and variant files that are empty or not the diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index fec51d9..898d990 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -67,7 +67,7 @@ func newEvictionTestCache(t *testing.T, maxBytes int64) (*Cache, string) { db := evictionTestDB(t) // maxBytes zero turns the disk cache off, as cache_max_bytes: 0 does. - cache, err := NewCache(db, CacheConfig{ + cache, err := NewCache(db, Config{ StateDir: tmpDir, NegativeTTL: 5 * time.Minute, MaxBytes: maxBytes, @@ -98,11 +98,11 @@ func storeEvictionTestSource( }, } - key, err := cache.StoreSource( - context.Background(), req, bytes.NewReader(content), + key, err := cache.PutSource( + context.Background(), req.Source, bytes.NewReader(content), ) if err != nil { - t.Fatalf("StoreSource(%s%s) failed: %v", host, path, err) + t.Fatalf("PutSource(%s%s) failed: %v", host, path, err) } return key @@ -115,9 +115,9 @@ func storeEvictionTestVariant( ) { t.Helper() - err := cache.StoreVariant(t.Context(), key, bytes.NewReader(content)) + err := cache.PutVariant(t.Context(), key, bytes.NewReader(content)) if err != nil { - t.Fatalf("StoreVariant(%s) failed: %v", key, err) + t.Fatalf("PutVariant(%s) failed: %v", key, err) } } @@ -423,20 +423,20 @@ func assertDisabledCacheWritesAreNoOps( ctx := t.Context() - err := cache.StoreVariant( + err := cache.PutVariant( ctx, imagerequest.CacheKey(req), bytes.NewReader([]byte("data")), ) if err != nil { - t.Fatalf("StoreVariant on disabled cache must be a no-op, got error: %v", err) + t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err) } - key, err := cache.StoreSource(ctx, req, bytes.NewReader([]byte("data"))) + key, err := cache.PutSource(ctx, req.Source, bytes.NewReader([]byte("data"))) if err != nil { - t.Fatalf("StoreSource on disabled cache must be a no-op, got error: %v", err) + t.Fatalf("PutSource on disabled cache must be a no-op, got error: %v", err) } if key != "" { - t.Errorf("StoreSource on disabled cache returned key %q, want empty", key) + t.Errorf("PutSource on disabled cache returned key %q, want empty", key) } } @@ -449,13 +449,13 @@ func assertDisabledCacheReadsAlwaysMiss( ctx := t.Context() - key, err := cache.LookupSource(ctx, req) - if err != nil { - t.Fatalf("LookupSource on disabled cache failed: %v", err) + reader, _, err := cache.Source(ctx, req.Source) + if !errors.Is(err, ErrNotFound) { + t.Fatalf("Source on disabled cache error = %v, want %v", err, ErrNotFound) } - if key != "" { - t.Errorf("LookupSource on disabled cache = %q, want empty", key) + if reader != nil { + t.Errorf("Source on disabled cache = reader %v, want none", reader) } } @@ -544,7 +544,7 @@ func TestEvictionRunsUnderWritePressure(t *testing.T) { } // TestEvictionRunsOnPeriodicSchedule writes three variant files straight -// to disk, bypassing StoreVariant, so they have no accounting rows and no +// to disk, bypassing PutVariant, so they have no accounting rows and no // write-pressure notification fires. Only a periodic reconciliation pass // can then adopt them, and only the eviction pass that follows it can // evict them. @@ -697,7 +697,7 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { // TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves // reconciliation is not a one-shot startup-only pass: it must also run // on the periodic ticker, so a variant file that lands on disk with no -// accounting row well after startup (e.g. because StoreVariant's +// accounting row well after startup (e.g. because PutVariant's // best-effort accounting insert failed under transient contention, or // any other cause of an untracked file appearing during steady-state // operation) is still adopted into accounting eventually, rather than @@ -733,7 +733,7 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { // Simulate a variant whose accounting insert failed after the // process was already running and serving requests: the content - // file is written directly, bypassing StoreVariant's (and thus its + // file is written directly, bypassing PutVariant's (and thus its // accounting insert) entirely, exactly as would happen if that // insert had failed and only the file write had succeeded. untracked := bytes.Repeat([]byte{0x41}, 900) diff --git a/internal/imgcache/imgcache.go b/internal/imgcache/imgcache.go deleted file mode 100644 index 83dd0d8..0000000 --- a/internal/imgcache/imgcache.go +++ /dev/null @@ -1,29 +0,0 @@ -// Package imgcache provides types for the image caching proxy. -package imgcache - -import "io" - -// ImageResponse represents a processed image ready to serve -type ImageResponse struct { - // Content is the image data reader - Content io.ReadCloser - // ContentLength is the size in bytes (-1 if unknown) - ContentLength int64 - // ContentType is the MIME type of the response - ContentType string - // ETag is the entity tag for caching - ETag string - // CacheStatus indicates HIT or MISS - CacheStatus CacheStatus - // FetchedBytes is the number of bytes fetched from upstream (0 if cache hit) - FetchedBytes int64 -} - -// CacheStatus indicates whether the response was served from cache. -type CacheStatus string - -// Cache status values for response headers. -const ( - CacheHit CacheStatus = "HIT" - CacheMiss CacheStatus = "MISS" -) diff --git a/internal/imgcache/module.go b/internal/imgcache/module.go deleted file mode 100644 index 81d329a..0000000 --- a/internal/imgcache/module.go +++ /dev/null @@ -1,8 +0,0 @@ -package imgcache - -import "time" - -// CacheConfig defaults. -const ( - DefaultNegativeTTL = 5 * time.Minute -) diff --git a/internal/imgcache/negative_cache_internal_test.go b/internal/imgcache/negative_cache_internal_test.go index 709f090..0c980c0 100644 --- a/internal/imgcache/negative_cache_internal_test.go +++ b/internal/imgcache/negative_cache_internal_test.go @@ -17,7 +17,7 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { db := setupTestDB(t) dir := t.TempDir() - cache, err := NewCache(db, CacheConfig{ + cache, err := NewCache(db, Config{ StateDir: dir, NegativeTTL: 5 * time.Minute, MaxBytes: 1 << 30, @@ -35,7 +35,7 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { } // Initially should not be in negative cache - hit, err := cache.checkNegativeCache(ctx, req) + hit, err := cache.Failure(ctx, req.Source) if err != nil { t.Fatal(err) } @@ -45,13 +45,13 @@ func TestNegativeCache_StoreAndCheck(t *testing.T) { } // Store a negative entry - err = cache.StoreNegative(ctx, req) + err = cache.PutFailure(ctx, req.Source) if err != nil { t.Fatal(err) } // Now should be in negative cache - hit, err = cache.checkNegativeCache(ctx, req) + hit, err = cache.Failure(ctx, req.Source) if err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func TestNegativeCache_Expired(t *testing.T) { db := setupTestDB(t) dir := t.TempDir() - cache, err := NewCache(db, CacheConfig{ + cache, err := NewCache(db, Config{ StateDir: dir, NegativeTTL: 1 * time.Millisecond, // very short TTL MaxBytes: 1 << 30, @@ -85,7 +85,7 @@ func TestNegativeCache_Expired(t *testing.T) { } // Store a negative entry with very short TTL - err = cache.StoreNegative(ctx, req) + err = cache.PutFailure(ctx, req.Source) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func TestNegativeCache_Expired(t *testing.T) { time.Sleep(10 * time.Millisecond) // Should no longer be in negative cache - hit, err := cache.checkNegativeCache(ctx, req) + hit, err := cache.Failure(ctx, req.Source) if err != nil { t.Fatal(err) } @@ -104,71 +104,6 @@ func TestNegativeCache_Expired(t *testing.T) { } } -// TestService_Get_ServesCachedVariantOfNegativeCachedURL caches a variant of -// the test photo, then keeps a failed fetch of the photo's URL in the negative -// cache, as a failed fetch for another variant does once the source has been -// evicted. The cached variant is still served, and a variant not in the cache -// is made from the cached source. Once the source is evicted, a variant not in -// the cache needs a fetch and is refused. -func TestService_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { - t.Parallel() - - svc, fixtures := SetupTestService(t) - - resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) - if err != nil { - t.Fatalf("Get() first request error = %v", err) - } - - _ = resp.Content.Close() - - err = svc.cache.StoreNegative(t.Context(), - photoVariant(fixtures, 85, imagerequest.FitCover)) - if err != nil { - t.Fatalf("StoreNegative() error = %v", err) - } - - resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover)) - if err != nil { - t.Fatalf("Get() of the cached variant error = %v", err) - } - - _ = resp.Content.Close() - - if resp.CacheStatus != CacheHit { - t.Errorf("cached variant CacheStatus = %q, want %q", - resp.CacheStatus, CacheHit) - } - - resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover)) - t.Logf("Get() of a variant made from the cached source: error = %v", err) - - if err != nil { - t.Fatalf("Get() of a variant made from the cached source error = %v", err) - } - - _ = resp.Content.Close() - - sourceKey, err := svc.cache.LookupSource(t.Context(), - photoVariant(fixtures, 85, imagerequest.FitCover)) - if err != nil || sourceKey == "" { - t.Fatalf("LookupSource() = %q, %v, want the cached source", sourceKey, err) - } - - err = svc.cache.evictSourceBlob(t.Context(), sourceKey) - if err != nil { - t.Fatalf("evictSourceBlob() error = %v", err) - } - - _, err = svc.Get(t.Context(), photoVariant(fixtures, 60, imagerequest.FitCover)) - t.Logf("Get() of a variant whose source was evicted: error = %v", err) - - if !errors.Is(err, ErrNegativeCached) { - t.Errorf("Get() of a variant whose source was evicted error = %v, want %v", - err, ErrNegativeCached) - } -} - // TestEvictionLoopRemovesExpiredNegativeCacheRows stores a failed fetch // that expires a second later and starts the eviction loop with a short // interval: a later tick must remove its row although its URL is never @@ -190,7 +125,7 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - cache, err := NewCache(evictionTestDB(t), CacheConfig{ + cache, err := NewCache(evictionTestDB(t), Config{ StateDir: t.TempDir(), NegativeTTL: time.Second, MaxBytes: tt.maxBytes, @@ -203,9 +138,9 @@ func TestEvictionLoopRemovesExpiredNegativeCacheRows(t *testing.T) { Source: imagerequest.Source{Host: testHostExample, Path: "/gone.jpg"}, } - err = cache.StoreNegative(t.Context(), expired) + err = cache.PutFailure(t.Context(), expired.Source) if err != nil { - t.Fatalf("StoreNegative failed: %v", err) + t.Fatalf("PutFailure failed: %v", err) } const notExpiredPath = "/later.jpg" @@ -269,7 +204,7 @@ func TestCleanExpiredDeletesInBatches(t *testing.T) { t.Parallel() cache, err := NewCache(evictionTestDB(t), - CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) @@ -325,7 +260,7 @@ func TestCleanExpiredRemovesMoreThanOneBatch(t *testing.T) { t.Parallel() cache, err := NewCache(evictionTestDB(t), - CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) @@ -357,7 +292,7 @@ func TestCleanExpiredStopsOnceContextEnds(t *testing.T) { t.Parallel() cache, err := NewCache(evictionTestDB(t), - CacheConfig{StateDir: t.TempDir(), MaxBytes: 1 << 30}, + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, prometheus.NewRegistry()) if err != nil { t.Fatalf("failed to create cache: %v", err) @@ -410,15 +345,3 @@ func negativeCacheTableRowCount(tb testing.TB, cache *Cache) int { return count } - -func TestService_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) { - t.Parallel() - - // This test verifies that Service.Get() checks the negative cache - // We can't easily test the full pipeline without vips, but we can - // verify the error type - err := ErrNegativeCached - if !errors.Is(err, ErrNegativeCached) { - t.Error("ErrNegativeCached should be identifiable with errors.Is") - } -} diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index 2179396..b89b720 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -12,67 +12,6 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// getImage calls Get for req and returns the image it answered with and -// its cache status. -func getImage( - t *testing.T, svc *Service, req *imagerequest.Request, -) ([]byte, CacheStatus) { - t.Helper() - - resp, err := svc.Get(t.Context(), req) - if err != nil { - t.Fatalf("Get() of quality %d error = %v", req.Transform.Quality, err) - } - - defer func() { _ = resp.Content.Close() }() - - body, err := io.ReadAll(resp.Content) - if err != nil { - t.Fatalf("reading the image of quality %d: %v", req.Transform.Quality, err) - } - - return body, resp.CacheStatus -} - -// fileSize returns the size of the file at path. -func fileSize(t *testing.T, path string) int64 { - t.Helper() - - info, err := os.Stat(path) - if err != nil { - t.Fatalf("stat %s: %v", path, err) - } - - return info.Size() -} - -// truncateFile cuts the file at path down to size bytes. -func truncateFile(t *testing.T, path string, size int64) { - t.Helper() - - t.Logf("cutting %s from %d to %d bytes", - filepath.Base(path), fileSize(t, path), size) - - err := os.Truncate(path, size) - if err != nil { - t.Fatalf("truncating %s: %v", path, err) - } -} - -// cachedSourcePath returns the path of the cached source file req uses. -func cachedSourcePath( - t *testing.T, svc *Service, req *imagerequest.Request, -) string { - t.Helper() - - sourceKey, err := svc.cache.LookupSource(t.Context(), req) - if err != nil || sourceKey == "" { - t.Fatalf("LookupSource() = %q, %v; want the cached source", sourceKey, err) - } - - return svc.cache.srcContent.keyToPath(sourceKey) -} - // reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does // when it starts, and returns what the pass logged. func reconcileAsAtStartup(t *testing.T, cache *Cache) string { @@ -107,127 +46,65 @@ func assertWarningNames(t *testing.T, logs, path string) { t.Errorf("reconciliation logged no warning naming %s", path) } -// TestService_Get_FetchesAgainForEmptyCachedSource empties the cached source -// file, as a crash or a restored snapshot could, and runs the reconciliation -// pass, which must remove it with a warning naming it. It then asks for two -// new sizes of the source: the first must fetch it again and cache it whole, -// so that the second is made from the cache without a third fetch. -func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) { +// TestReconciliationRemovesDamagedFiles empties a cached source file and a +// cached variant file, or cuts them to half their size, as a crash or a +// restored snapshot could. It checks that the reconciliation pass removes +// each file and its row, with a warning naming the file. +func TestReconciliationRemovesDamagedFiles(t *testing.T) { t.Parallel() - svc, fixtures := SetupTestService(t) + content := []byte("the whole cached image") - getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + for name, size := range map[string]int64{ + "empty": 0, + "partial": int64(len(content) / 2), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() - path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - wholeSize := fileSize(t, path) + cache, _ := newEvictionTestCache(t, 1<<30) - truncateFile(t, path, 0) - assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) + sourceKey := storeEvictionTestSource(t, cache, testHostCDN, testPathCat, content) + storeEvictionTestVariant(t, cache, testVariantKeyOne, content) - getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) - getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) + sourcePath := cache.srcContent.keyToPath(sourceKey) + variantPath := cache.variants.keyToPath(testVariantKeyOne) - fetches := readCounters(svc.cache).upstreamFetchCount - t.Logf("upstream fetches = %d", fetches) + for _, path := range []string{sourcePath, variantPath} { + err := os.Truncate(path, size) + if err != nil { + t.Fatalf("cutting %s to %d bytes: %v", path, size, err) + } + } - if fetches != 2 { - t.Errorf("upstream fetches = %d, want 2: the empty cached source "+ - "was kept and fetched again on every new size", fetches) + logs := reconcileAsAtStartup(t, cache) + + assertWarningNames(t, logs, sourcePath) + assertWarningNames(t, logs, variantPath) + + if cache.srcContent.Exists(sourceKey) { + t.Error("the damaged source file is still on disk") + } + + if cache.variants.Exists(testVariantKeyOne) { + t.Error("the damaged variant file is still on disk") + } + + rows := countRows(t, cache, + `SELECT COUNT(*) FROM source_content WHERE source_key = ?`, + string(sourceKey)) + if rows != 0 { + t.Errorf("the damaged source file kept %d rows, want 0", rows) + } + + rows = countRows(t, cache, + `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`, + string(testVariantKeyOne)) + if rows != 0 { + t.Errorf("the damaged variant file kept %d rows, want 0", rows) + } + }) } - - size := fileSize(t, path) - if size != wholeSize { - t.Errorf("cached source is %d bytes, want %d", size, wholeSize) - } -} - -// TestService_Get_FetchesAgainForPartialCachedSource cuts the cached source -// file to half its size, as a crash or a restored snapshot could, and runs -// the reconciliation pass, which must remove it with a warning naming it. -// It then asks for two new sizes of the source: the first must fetch it -// again rather than fail to decode the partial file, and the second is -// made from the cache. -func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) { - t.Parallel() - - svc, fixtures := SetupTestService(t) - - getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - - path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - wholeSize := fileSize(t, path) - - truncateFile(t, path, wholeSize/2) - assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) - - getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) - getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover)) - - fetches := readCounters(svc.cache).upstreamFetchCount - t.Logf("upstream fetches = %d", fetches) - - if fetches != 2 { - t.Errorf("upstream fetches = %d, want 2", fetches) - } - - size := fileSize(t, path) - if size != wholeSize { - t.Errorf("cached source is %d bytes, want %d", size, wholeSize) - } -} - -// assertConvertedAgain caches a variant of the photo, cuts its file to size -// bytes and runs the reconciliation pass, which must remove it with a -// warning naming it. It then checks that Get converts the image again -// instead of serving the file, and that the next Get serves the whole -// variant from the cache. -func assertConvertedAgain(t *testing.T, size int64) { - t.Helper() - - svc, fixtures := SetupTestService(t) - req := photoVariant(fixtures, 85, imagerequest.FitCover) - - want, _ := getImage(t, svc, req) - - path := svc.cache.variants.keyToPath(imagerequest.CacheKey(req)) - - truncateFile(t, path, size) - assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path) - - got, status := getImage(t, svc, req) - t.Logf("after the cut: %d bytes, %s", len(got), status) - - if status != CacheMiss || !bytes.Equal(got, want) { - t.Errorf("after the cut: Get() = %d bytes, %s; want the %d bytes "+ - "converted again, %s", len(got), status, len(want), CacheMiss) - } - - got, status = getImage(t, svc, req) - t.Logf("next request: %d bytes, %s", len(got), status) - - if status != CacheHit || !bytes.Equal(got, want) { - t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s", - len(got), status, len(want), CacheHit) - } -} - -// TestService_Get_ConvertsAgainForEmptyCachedVariant empties a cached -// variant file, as a crash or a restored snapshot could, and checks that it -// is converted again rather than served empty. -func TestService_Get_ConvertsAgainForEmptyCachedVariant(t *testing.T) { - t.Parallel() - - assertConvertedAgain(t, 0) -} - -// TestService_Get_ConvertsAgainForPartialCachedVariant cuts a cached -// variant file to ten bytes, as a crash or a restored snapshot could, and -// checks that it is converted again rather than served cut short. -func TestService_Get_ConvertsAgainForPartialCachedVariant(t *testing.T) { - t.Parallel() - - assertConvertedAgain(t, 10) } // TestReconciliationRemovesEmptyVariantFileWithoutRow checks that the diff --git a/internal/imgcache/pendingaccesstimes_internal_test.go b/internal/imgcache/pendingaccesstimes_internal_test.go index aa8a100..b21ca78 100644 --- a/internal/imgcache/pendingaccesstimes_internal_test.go +++ b/internal/imgcache/pendingaccesstimes_internal_test.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "errors" + "io" "log/slog" "strings" "testing" @@ -15,8 +16,7 @@ import ( // pendingAccessTimesWait is how long a test waits for the database's // connection, for an access time to reach the database, for a write to start -// waiting for the connection, for a request to return past its deadline, or -// for StopPendingAccessTimeWrites to return. +// waiting for the connection, or for StopPendingAccessTimeWrites to return. const pendingAccessTimesWait = 5 * time.Second // otherWritesWait is how long a test gives other writes, once one write waits @@ -56,52 +56,6 @@ func waitForConnectionWaits(t *testing.T, db *sql.DB, waits int64) { } } -// TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the -// database's one connection while a request whose fetch is held reaches its -// deadline. The request must return with the deadline's error while the -// connection is still held, as it waits for no database write past its -// deadline. -func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) { - t.Parallel() - - svc, fixtures, fetcher := setupHeldFetchService(t) - - svc.cache.StartPendingAccessTimeWrites() - defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }() - - // Room for the request to reach its fetch on a busy host - const timeout = 2 * time.Second - - ctx, cancel := context.WithTimeout(t.Context(), timeout) - defer cancel() - - deadline, _ := ctx.Deadline() - - results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) - - // The request has made its database reads by the time it fetches. - select { - case <-fetcher.started: - case <-time.After(timeout): - t.Fatal("request did not reach its fetch by its deadline") - } - - releaseDatabase := holdDatabase(t, svc.cache) - defer releaseDatabase() - - select { - case got := <-results: - t.Logf("Get() returned %v after its deadline, error = %v", - time.Since(deadline), got.err) - - if !errors.Is(got.err, context.DeadlineExceeded) { - t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded) - } - case <-time.After(time.Until(deadline) + pendingAccessTimesWait): - t.Fatal("request did not return while the database was busy") - } -} - // TestStopPendingAccessTimeWrites_LetsTheWriteUnderWayFinish holds the // database's one connection while the goroutine that writes the pending // access times waits for it, hands over another access time, then stops that @@ -242,11 +196,11 @@ func TestStopPendingAccessTimeWrites_ReturnsWhenItsContextEnds(t *testing.T) { } } -// TestLookupSource_HandsAccessTimeToTheBackgroundWriter finds a cached source -// whose access time is an hour old. LookupSource must not write the access +// TestSource_HandsAccessTimeToTheBackgroundWriter finds a cached source +// whose access time is an hour old. Source must not write the access // time itself but hand it over to be written, and StopPendingAccessTimeWrites // must then write it. -func TestLookupSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { +func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) @@ -261,15 +215,21 @@ func TestLookupSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { ` anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat) - found, err := cache.LookupSource(t.Context(), &imagerequest.Request{ - Source: imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}, - }) - if err != nil || found != key { - t.Fatalf("LookupSource() = %q, %v; want %q", found, err, key) + reader, _, err := cache.Source(t.Context(), + imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}) + if err != nil { + t.Fatalf("Source() error = %v; want the source of %q", err, key) + } + + found, err := io.ReadAll(reader) + _ = reader.Close() + + if err != nil || string(found) != "source bytes" { + t.Fatalf("Source() reads %q, %v; want the source of %q", found, err, key) } if countRows(t, cache, touchedQuery, string(key), anHourAgoInDatabase) != 0 { - t.Error("LookupSource wrote the source's access time itself") + t.Error("Source wrote the source's access time itself") } if waiting := len(cache.pendingSourceAccessTimes); waiting != 1 { diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go index 595b7c6..3d12835 100644 --- a/internal/imgcache/source_content_internal_test.go +++ b/internal/imgcache/source_content_internal_test.go @@ -10,11 +10,11 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// TestStoreSource_StoresIdenticalBytesOfTwoURLsTwice stores the same bytes +// TestPutSource_StoresIdenticalBytesOfTwoURLsTwice stores the same bytes // for two source URLs and checks that each URL has a source_content row of // its own and a file of its own, named by its source key under // cache/sources///. -func TestStoreSource_StoresIdenticalBytesOfTwoURLsTwice(t *testing.T) { +func TestPutSource_StoresIdenticalBytesOfTwoURLsTwice(t *testing.T) { t.Parallel() cache, stateDir := newEvictionTestCache(t, 1<<30) @@ -84,18 +84,12 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { t.Errorf("the evicted source still has %d source_content rows", n) } - kept, err := cache.LookupSource(t.Context(), &imagerequest.Request{ - Source: imagerequest.Source{Host: testHostCDN, Path: "/kept.jpg"}, - }) - t.Logf("LookupSource() of the kept source = %q, error %v", kept, err) + reader, _, err := cache.Source(t.Context(), + imagerequest.Source{Host: testHostCDN, Path: "/kept.jpg"}) + t.Logf("Source() of the kept source: error %v", err) - if err != nil || kept == "" { - t.Fatalf("LookupSource() of the kept source = %q, %v; want it cached", kept, err) - } - - reader, _, err := cache.GetSourceContent(kept) if err != nil { - t.Fatalf("GetSourceContent() of the kept source error = %v", err) + t.Fatalf("Source() of the kept source error = %v; want it cached", err) } defer func() { _ = reader.Close() }() @@ -111,10 +105,10 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { } } -// TestStoreSource_FailedRowWriteLeavesNoFile drops source_content, so that -// StoreSource cannot write the source's row, and checks that the store fails +// TestPutSource_FailedRowWriteLeavesNoFile drops source_content, so that +// PutSource cannot write the source's row, and checks that the store fails // and leaves no file under the source's key. -func TestStoreSource_FailedRowWriteLeavesNoFile(t *testing.T) { +func TestPutSource_FailedRowWriteLeavesNoFile(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) @@ -127,12 +121,12 @@ func TestStoreSource_FailedRowWriteLeavesNoFile(t *testing.T) { req := &imagerequest.Request{ Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, } - key, err := cache.StoreSource(t.Context(), req, + key, err := cache.PutSource(t.Context(), req.Source, bytes.NewReader([]byte("source bytes"))) - t.Logf("StoreSource() = %q, error = %v", key, err) + t.Logf("PutSource() = %q, error = %v", key, err) if err == nil { - t.Error("StoreSource() error = nil, want the failed row write") + t.Error("PutSource() error = nil, want the failed row write") } if cache.srcContent.Exists(req.Source.Key()) { diff --git a/internal/imgcache/testutil_internal_test.go b/internal/imgcache/testutil_internal_test.go index ec44fb3..8d2ebf7 100644 --- a/internal/imgcache/testutil_internal_test.go +++ b/internal/imgcache/testutil_internal_test.go @@ -2,98 +2,19 @@ package imgcache import ( "bytes" - "context" - "database/sql" "image" "image/color" - "image/gif" "image/jpeg" - "image/png" - "io/fs" "testing" - "testing/fstest" - "time" - - "github.com/prometheus/client_golang/prometheus" - "sneak.berlin/go/pixa/internal/database" - "sneak.berlin/go/pixa/internal/httpfetcher" - "sneak.berlin/go/pixa/internal/imageprocessor" ) // Shared test data literals, extracted as constants for goconst. const ( - testHostCDN = "cdn.example.com" - testHostExample = "example.com" - testPathCat = "/photos/cat.jpg" - testContentTypeJPEG = "image/jpeg" + testHostCDN = "cdn.example.com" + testHostExample = "example.com" + testPathCat = "/photos/cat.jpg" ) -// TestFixtures contains paths to test files in the mock filesystem. -type TestFixtures struct { - // Valid image files - GoodHostJPEG string // valid JPEG on GoodHost - GoodHostPNG string // valid PNG on GoodHost - GoodHostGIF string // valid GIF on GoodHost - OtherHostJPEG string // valid JPEG on OtherHost - OtherHostPNG string // valid PNG on OtherHost - - // Invalid/edge case files - InvalidFile string // file with wrong magic bytes - EmptyFile string // zero-byte file - TextFile string // text file masquerading as image - - // Hostnames - GoodHost string // a host the mock fetcher serves - OtherHost string // a second host the mock fetcher serves -} - -// DefaultFixtures returns the standard test fixture paths. -func DefaultFixtures() *TestFixtures { - return &TestFixtures{ - GoodHostJPEG: "goodhost.example.com/images/photo.jpg", - GoodHostPNG: "goodhost.example.com/images/logo.png", - GoodHostGIF: "goodhost.example.com/images/animation.gif", - OtherHostJPEG: "otherhost.example.com/uploads/image.jpg", - OtherHostPNG: "otherhost.example.com/uploads/icon.png", - InvalidFile: "goodhost.example.com/images/fake.jpg", - EmptyFile: "goodhost.example.com/images/empty.jpg", - TextFile: "goodhost.example.com/images/text.png", - GoodHost: "goodhost.example.com", - OtherHost: "otherhost.example.com", - } -} - -// NewTestFS creates a mock filesystem with test images. -func NewTestFS(t *testing.T) (fs.FS, *TestFixtures) { - t.Helper() - - fixtures := DefaultFixtures() - - // Generate test images - jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) - pngData := generateTestPNG(t, 100, 100, color.RGBA{0, 255, 0, 255}) - gifData := generateTestGIF(t, 100, 100, color.RGBA{0, 0, 255, 255}) - - // Create the mock filesystem - mockFS := fstest.MapFS{ - // Good host files - fixtures.GoodHostJPEG: &fstest.MapFile{Data: jpegData}, - fixtures.GoodHostPNG: &fstest.MapFile{Data: pngData}, - fixtures.GoodHostGIF: &fstest.MapFile{Data: gifData}, - - // Other host files - fixtures.OtherHostJPEG: &fstest.MapFile{Data: jpegData}, - fixtures.OtherHostPNG: &fstest.MapFile{Data: pngData}, - - // Invalid files - fixtures.InvalidFile: &fstest.MapFile{Data: []byte("not a real image file content")}, - fixtures.EmptyFile: &fstest.MapFile{Data: []byte{}}, - fixtures.TextFile: &fstest.MapFile{Data: []byte("This is just text, not a PNG")}, - } - - return mockFS, fixtures -} - // generateTestJPEG creates a minimal valid JPEG image. func generateTestJPEG(t *testing.T, width, height int, c color.Color) []byte { t.Helper() @@ -114,104 +35,3 @@ func generateTestJPEG(t *testing.T, width, height int, c color.Color) []byte { return buf.Bytes() } - -// generateTestPNG creates a minimal valid PNG image. -func generateTestPNG(t *testing.T, width, height int, c color.Color) []byte { - t.Helper() - - img := image.NewRGBA(image.Rect(0, 0, width, height)) - for y := range height { - for x := range width { - img.Set(x, y, c) - } - } - - var buf bytes.Buffer - - err := png.Encode(&buf, img) - if err != nil { - t.Fatalf("failed to encode test PNG: %v", err) - } - - return buf.Bytes() -} - -// generateTestGIF creates a minimal valid GIF image. -func generateTestGIF(t *testing.T, width, height int, c color.Color) []byte { - t.Helper() - - img := image.NewPaletted( - image.Rect(0, 0, width, height), - []color.Color{c, color.White}, - ) - for y := range height { - for x := range width { - img.SetColorIndex(x, y, 0) - } - } - - var buf bytes.Buffer - - err := gif.Encode(&buf, img, nil) - if err != nil { - t.Fatalf("failed to encode test GIF: %v", err) - } - - return buf.Bytes() -} - -// SetupTestService creates a Service with mock fetcher for testing. -func SetupTestService(t *testing.T) (*Service, *TestFixtures) { - t.Helper() - - mockFS, fixtures := NewTestFS(t) - - // Create temp directory for cache - tmpDir := t.TempDir() - - // Create in-memory database - db := setupServiceTestDB(t) - - cache, err := NewCache(db, CacheConfig{ - StateDir: tmpDir, - NegativeTTL: 5 * time.Minute, - MaxBytes: 1 << 30, - }, prometheus.NewRegistry()) - if err != nil { - t.Fatalf("failed to create cache: %v", err) - } - - svc, err := NewService(&ServiceConfig{ - Cache: cache, - Fetcher: httpfetcher.NewMock(mockFS), - Processor: imageprocessor.New(imageprocessor.Params{ - MaxInputBytes: 50 << 20, - MaxConcurrentProcessing: 2, - }), - MaxResponseSize: 50 << 20, - }) - if err != nil { - t.Fatalf("failed to create service: %v", err) - } - - return svc, fixtures -} - -// setupServiceTestDB creates an in-memory SQLite database for testing -// using the production schema. -func setupServiceTestDB(t *testing.T) *sql.DB { - t.Helper() - - db, err := sql.Open("sqlite", ":memory:") - if err != nil { - t.Fatalf("failed to open test db: %v", err) - } - - // Use the real production schema via migrations - err = database.ApplyMigrations(context.Background(), db, nil) - if err != nil { - t.Fatalf("failed to apply migrations: %v", err) - } - - return db -} diff --git a/internal/providers/providers.go b/internal/providers/providers.go index 2a11e61..dda53ca 100644 --- a/internal/providers/providers.go +++ b/internal/providers/providers.go @@ -26,6 +26,7 @@ import ( "sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/seal" @@ -34,7 +35,7 @@ import ( // Constructors returns the constructors of the parts this package builds, // for fx.Provide. The fx graph must also provide the prometheus.Registerer -// the cache registers its counters on. +// the cache and the image service register their counters on. func Constructors() []any { return []any{ newKeys, @@ -79,7 +80,7 @@ func newCache(lc fx.Lifecycle, params cacheParams) (*imgcache.Cache, error) { return nil, err } - cache, err := imgcache.NewCache(db, imgcache.CacheConfig{ + cache, err := imgcache.NewCache(db, imgcache.Config{ StateDir: cfg.StateDir, NegativeTTL: imgcache.DefaultNegativeTTL, MaxBytes: maxBytes, @@ -213,25 +214,26 @@ func newProcessor(cfg *config.Config) *imageprocessor.ImageProcessor { type imageServiceParams struct { fx.In - Config *config.Config - Cache *imgcache.Cache - Fetcher *httpfetcher.HTTPFetcher - Processor *imageprocessor.ImageProcessor - Logger *logger.Logger + Config *config.Config + Cache *imgcache.Cache + Fetcher *httpfetcher.HTTPFetcher + Processor *imageprocessor.ImageProcessor + Logger *logger.Logger + MetricsRegistry prometheus.Registerer } // newImageService builds the image service from the cache, the fetcher // and the processor, with the largest upstream response the config allows. func newImageService( _ fx.Lifecycle, params imageServiceParams, -) (*imgcache.Service, error) { - return imgcache.NewService(&imgcache.ServiceConfig{ +) (*imageproxy.Proxy, error) { + return imageproxy.New(&imageproxy.Config{ Cache: params.Cache, Fetcher: params.Fetcher, Processor: params.Processor, MaxResponseSize: params.Config.UpstreamMaxResponseSize, Logger: params.Logger.Get(), - }) + }, params.MetricsRegistry) } // sessionManagerParams defines dependencies for newSessionManager. diff --git a/internal/server/source_url_internal_test.go b/internal/server/source_url_internal_test.go index f897262..bb37d2d 100644 --- a/internal/server/source_url_internal_test.go +++ b/internal/server/source_url_internal_test.go @@ -14,8 +14,8 @@ import ( "time" "sneak.berlin/go/pixa/internal/encurl" + "sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imagerequest" - "sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/signature" @@ -227,7 +227,7 @@ func TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart(t *testing.T) { for _, imageURL := range imageURLs { cacheStatus := getImage(t, s, imageURL).Header().Get("X-Pixa-Cache") - if cacheStatus != string(imgcache.CacheHit) { + if cacheStatus != string(imageproxy.CacheHit) { t.Errorf("second GET %s: X-Pixa-Cache = %q, want HIT", imageURL, cacheStatus) } @@ -256,7 +256,7 @@ func TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest(t *testing.T) { second := getImage(t, s, "/v1/image/"+upstreamHost+"/photo.png%3Fname=my%20cat/orig.png") - if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) { + if got := second.Header().Get("X-Pixa-Cache"); got != string(imageproxy.CacheHit) { t.Errorf("second X-Pixa-Cache = %q, want HIT", got) } @@ -365,7 +365,7 @@ func TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL(t *testing.T) { getImage(t, s, imageURLs[0]) second := getImage(t, s, imageURLs[1]) - if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) { + if got := second.Header().Get("X-Pixa-Cache"); got != string(imageproxy.CacheHit) { t.Errorf("second X-Pixa-Cache = %q, want HIT", got) } From ac850823f4af31640503864cda21e94d8b20e7ed Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 20:24:33 +0100 Subject: [PATCH 108/131] Keep cached sources and variants in one FileStore type (closes #318) imgcache.FileStore is a directory of files named by a hex key, at //. It writes a file under a temp name, syncs it and renames it into place. It opens, reports the size of and deletes the file for a key. The cache keeps its sources in one FileStore and its variants in another, so ContentStorage and VariantStorage are gone. FileStore has no Exists: the reconciliation pass checks a variant's file with Size, and the tests look for the file on disk. Cache.Variant reads the content type from the file's first bytes itself. storage.go is now filestore.go, and the unused testdata/red.avif is gone. Model: opus-5-5 --- TODO.md | 17 + internal/imgcache/cache.go | 66 +++- internal/imgcache/cache_hit_internal_test.go | 2 +- internal/imgcache/eviction.go | 11 +- internal/imgcache/eviction_internal_test.go | 37 +- internal/imgcache/filestore.go | 185 ++++++++++ internal/imgcache/filestore_internal_test.go | 184 ++++++++++ .../partial_cache_files_internal_test.go | 41 ++- .../imgcache/source_content_internal_test.go | 6 +- internal/imgcache/storage.go | 326 ------------------ internal/imgcache/storage_internal_test.go | 156 --------- internal/imgcache/testdata/red.avif | Bin 281 -> 0 bytes 12 files changed, 496 insertions(+), 535 deletions(-) create mode 100644 internal/imgcache/filestore.go create mode 100644 internal/imgcache/filestore_internal_test.go delete mode 100644 internal/imgcache/storage.go delete mode 100644 internal/imgcache/storage_internal_test.go delete mode 100644 internal/imgcache/testdata/red.avif diff --git a/TODO.md b/TODO.md index 0618377..e76626d 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,23 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 the cache keeps its source images in one `imgcache.FileStore` and + its variants in another (closes #318). A `FileStore` is a directory of files, + each named by a hex key, at `//`. + - `ContentStorage` and `VariantStorage` are gone. + - `FileStore` has these methods: + - `Write`, which writes a file under a temp name, syncs it and renames + it into place + - `Open`, which returns the file and its size + - `Size` + - `Delete` + - `FileStore` has no `Exists`. The reconciliation pass checks that a + variant's file is there with `Size`. The tests look for the file on disk. + - `Cache.Variant` reads the content type from the variant file's first + bytes, as `VariantStorage.LoadWithContentType` did. A cache hit still + makes no database call. + - `storage.go` is now `filestore.go`. `internal/imgcache/testdata/red.avif`, + which no test used, is gone. - 2026-10-26 `internal/imgcache` holds the cache only (closes #317). The new package `internal/imageproxy` fetches, transforms, caches and serves images. It imports `imgcache`, and `imgcache` does not import it. diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index ab5cbcf..d1376b2 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/magic" ) // maxPendingAccessTimes is how many access times of variants served from @@ -57,8 +58,8 @@ type Config struct { // Cache implements the caching layer for the image proxy. type Cache struct { db *sql.DB - srcContent *ContentStorage // source images by source key - variants *VariantStorage // processed variants by cache key + srcContent *FileStore // source images by source key + variants *FileStore // processed variants by cache key config Config log *slog.Logger @@ -135,14 +136,14 @@ func NewCache( return c, nil } - srcContent, err := NewContentStorage( + srcContent, err := NewFileStore( filepath.Join(config.StateDir, "cache", "sources"), ) if err != nil { return nil, fmt.Errorf("failed to create source content storage: %w", err) } - variants, err := NewVariantStorage( + variants, err := NewFileStore( filepath.Join(config.StateDir, "cache", "variants"), ) if err != nil { @@ -169,14 +170,21 @@ func (c *Cache) Variant( return nil, 0, "", ErrNotFound } - reader, size, contentType, err := c.variants.LoadWithContentType(cacheKey) + file, size, err := c.variants.Open(string(cacheKey)) if err != nil { return nil, 0, "", err } + contentType, err := outputContentType(file) + if err != nil { + _ = file.Close() + + return nil, 0, "", err + } + c.touchVariant(cacheKey) - return reader, size, contentType, nil + return file, size, contentType, nil } // PutVariant stores a processed variant by its cache key and records @@ -190,11 +198,18 @@ func (c *Cache) PutVariant( return nil } - size, err := c.variants.Store(cacheKey, content) + data, err := io.ReadAll(content) + if err != nil { + return fmt.Errorf("failed to read content: %w", err) + } + + err = c.variants.Write(string(cacheKey), data) if err != nil { return err } + size := int64(len(data)) + _, err = c.db.ExecContext(ctx, ` INSERT INTO variant_content (cache_key, size_bytes) VALUES (?, ?) @@ -238,14 +253,14 @@ func (c *Cache) Source( return nil, 0, ErrNotFound } - reader, size, err := c.srcContent.LoadWithSize(key) + file, size, err := c.srcContent.Open(string(key)) if err != nil { return nil, 0, err } c.touchSourceContent(key) - return reader, size, nil + return file, size, nil } // PutSource stores fetched source content: it writes the file named by @@ -270,11 +285,13 @@ func (c *Cache) PutSource( key := src.Key() - size, err := c.srcContent.Store(key, data) + err = c.srcContent.Write(string(key), data) if err != nil { return "", fmt.Errorf("failed to store source content: %w", err) } + size := int64(len(data)) + _, err = c.db.ExecContext(ctx, ` INSERT INTO source_content (source_key, size_bytes) VALUES (?, ?) @@ -283,7 +300,7 @@ func (c *Cache) PutSource( last_accessed_at = CURRENT_TIMESTAMP `, string(key), size) if err != nil { - deleteErr := c.srcContent.Delete(key) + deleteErr := c.srcContent.Delete(string(key)) if deleteErr != nil { c.log.Warn("failed to remove source file whose row was not written", "source_key", key, "error", deleteErr) @@ -422,3 +439,30 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) { return deleted, nil } + +// errNotAnOutputFormat is returned for a variant file whose first bytes are +// those of a format pixa reads but never writes, such as SVG. +var errNotAnOutputFormat = errors.New("variant file is not in an output format") + +// outputContentType reads the first bytes of content, without moving its +// offset, and returns the content type of the output format they show. +func outputContentType(content io.ReaderAt) (string, error) { + firstBytes := make([]byte, magic.MinMagicBytes) + + _, err := content.ReadAt(firstBytes, 0) + if err != nil { + return "", fmt.Errorf("failed to read the first bytes: %w", err) + } + + detected, err := magic.DetectFormat(firstBytes) + if err != nil { + return "", err + } + + _, isOutputFormat := magic.MIMEToImageFormat(string(detected)) + if !isOutputFormat { + return "", fmt.Errorf("%w: %s", errNotAnOutputFormat, detected) + } + + return string(detected), nil +} diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index 068bce0..23e2e07 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -104,7 +104,7 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) { for i, tt := range tests { key := imagerequest.VariantKey(fmt.Sprintf("aabbccdd20%02d", i)) - _, err := cache.variants.Store(key, bytes.NewReader(tt.content)) + err := cache.variants.Write(string(key), tt.content) if err != nil { t.Fatalf("%s: storing the variant file: %v", tt.name, err) } diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 8b576ad..ef5f346 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -48,7 +48,7 @@ const staleTempFileAge = time.Hour const sqliteTimestampLayout = "2006-01-02 15:04:05" // tempFilePrefix is the prefix os.CreateTemp uses for in-flight cache -// writes (".tmp-*" patterns in the storage layer). +// writes (the ".tmp-*" pattern in writeFileSynced). const tempFilePrefix = ".tmp-" // UsageBytes returns the total number of bytes of cache content @@ -306,7 +306,7 @@ func (c *Cache) evictVariant( return fmt.Errorf("failed to delete variant accounting row: %w", err) } - err = c.variants.Delete(cacheKey) + err = c.variants.Delete(string(cacheKey)) if err != nil { return err } @@ -325,7 +325,7 @@ func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) return fmt.Errorf("failed to delete source content row: %w", err) } - return c.srcContent.Delete(key) + return c.srcContent.Delete(string(key)) } // notifyWritePressure wakes the background evictor after a store, so @@ -636,11 +636,12 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error { return ctx.Err() } - if c.variants.Exists(key) { + _, err := c.variants.Size(string(key)) + if err == nil { continue } - _, err := c.db.ExecContext(ctx, + _, err = c.db.ExecContext(ctx, `DELETE FROM variant_content WHERE cache_key = ?`, string(key)) if err != nil { return fmt.Errorf("failed to drop stale variant accounting row: %w", err) diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 898d990..7be430e 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -209,7 +209,8 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) { t.Fatalf("failed to scan source_key: %v", err) } - if !cache.srcContent.Exists(imagerequest.SourceKey(key)) { + _, err = os.Stat(cache.srcContent.keyToPath(key)) + if err != nil { t.Errorf("source_content references content %s but the file is missing", key) } @@ -236,7 +237,8 @@ func assertNoDanglingReferences(t *testing.T, cache *Cache) { t.Fatalf("failed to scan cache_key: %v", err) } - if !cache.variants.Exists(imagerequest.VariantKey(key)) { + _, err = os.Stat(cache.variants.keyToPath(key)) + if err != nil { t.Errorf("variant_content references key %s but the file is missing", key) } } @@ -334,7 +336,8 @@ func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) { t.Errorf("usage after eviction = %d, want <= %d", usage, limit) } - if cache.variants.Exists(keys[0]) { + _, err = os.Stat(cache.variants.keyToPath(string(keys[0]))) + if err == nil { t.Errorf("least recently used variant %s must be evicted", keys[0]) } @@ -345,7 +348,8 @@ func TestEvictToLimitEvictsLeastRecentlyUsedFirst(t *testing.T) { } for _, key := range keys[1:] { - if !cache.variants.Exists(key) { + _, err = os.Stat(cache.variants.keyToPath(string(key))) + if err != nil { t.Errorf("more recently used variant %s must survive eviction", key) } } @@ -373,7 +377,8 @@ func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) { } for _, key := range keys { - if !cache.srcContent.Exists(key) { + _, err = os.Stat(cache.srcContent.keyToPath(string(key))) + if err != nil { t.Errorf("source %s must not be evicted while usage is under the limit", key) } @@ -384,7 +389,8 @@ func TestEvictionKeepsEverythingWhenUnderLimit(t *testing.T) { } } - if !cache.variants.Exists(testVariantKeyOne) { + _, err = os.Stat(cache.variants.keyToPath(string(testVariantKeyOne))) + if err != nil { t.Error("variant must not be evicted while usage is under the limit") } @@ -586,7 +592,7 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) { for i, key := range keys { content := bytes.Repeat([]byte{fills[i]}, 1000) - _, err = cache.variants.Store(key, bytes.NewReader(content)) + err = cache.variants.Write(string(key), content) if err != nil { t.Fatalf("failed to store variant file: %v", err) } @@ -605,7 +611,8 @@ func TestEvictionRunsOnPeriodicSchedule(t *testing.T) { stored = 0 for _, key := range keys { - if cache.variants.Exists(key) { + _, err = os.Stat(cache.variants.keyToPath(string(key))) + if err == nil { stored++ } } @@ -637,9 +644,7 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { // feature existed) must be adopted into the accounting. untracked := bytes.Repeat([]byte{0x31}, 1000) - _, err := cache.variants.Store( - testVariantKeyOne, bytes.NewReader(untracked), - ) + err := cache.variants.Write(string(testVariantKeyOne), untracked) if err != nil { t.Fatalf("failed to store untracked variant file: %v", err) } @@ -738,9 +743,7 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) { // insert had failed and only the file write had succeeded. untracked := bytes.Repeat([]byte{0x41}, 900) - _, err = cache.variants.Store( - "aabbccdd0099", bytes.NewReader(untracked), - ) + err = cache.variants.Write("aabbccdd0099", untracked) if err != nil { t.Fatalf("failed to store untracked variant file: %v", err) } @@ -1046,13 +1049,15 @@ func assertOnlyOldestSourceEvicted( ) { t.Helper() - if cache.srcContent.Exists(keys[0]) { + _, err := os.Stat(cache.srcContent.keyToPath(string(keys[0]))) + if err == nil { t.Errorf("source %s, evicted when the context was cancelled, "+ "is still on disk", keys[0]) } for _, key := range keys[1:] { - if !cache.srcContent.Exists(key) { + _, err = os.Stat(cache.srcContent.keyToPath(string(key))) + if err != nil { t.Errorf("source %s was evicted after the context was cancelled", key) } diff --git a/internal/imgcache/filestore.go b/internal/imgcache/filestore.go new file mode 100644 index 0000000..779ff65 --- /dev/null +++ b/internal/imgcache/filestore.go @@ -0,0 +1,185 @@ +package imgcache + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// Storage constants. +const ( + // StorageDirPerm is the permission mode for storage directories. + StorageDirPerm = 0750 + // StorageFilePerm is the permission mode for storage files. + StorageFilePerm = 0600 + // MinHashLength is the minimum hash length for path splitting. + MinHashLength = 4 +) + +// Storage errors. +var ( + ErrNotFound = errors.New("content not found") +) + +// FileStore is a directory of files, each named by a hex key. The file for +// a key is at ///, where and are the key's +// first and second pairs of characters. The cache keeps its source images in +// one FileStore and its variants in another. +type FileStore struct { + baseDir string +} + +// NewFileStore creates the directory baseDir if it is missing and returns +// the FileStore of the files in it. +func NewFileStore(baseDir string) (*FileStore, error) { + err := os.MkdirAll(baseDir, StorageDirPerm) + if err != nil { + return nil, fmt.Errorf("failed to create storage directory: %w", err) + } + + return &FileStore{baseDir: baseDir}, nil +} + +// Write writes data with writeFileSynced to the file for key, replacing any +// file already there. +func (s *FileStore) Write(key string, data []byte) error { + path := s.keyToPath(key) + + err := os.MkdirAll(filepath.Dir(path), StorageDirPerm) + if err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + return writeFileSynced(path, data) +} + +// Open opens the file for key and returns it with its size in bytes. A +// missing file is ErrNotFound. +func (s *FileStore) Open(key string) (*os.File, int64, error) { + f, err := os.Open(s.keyToPath(key)) + if err != nil { + if os.IsNotExist(err) { + return nil, 0, ErrNotFound + } + + return nil, 0, fmt.Errorf("failed to open content: %w", err) + } + + stat, err := f.Stat() + if err != nil { + _ = f.Close() + + return nil, 0, fmt.Errorf("failed to stat content: %w", err) + } + + return f, stat.Size(), nil +} + +// Size returns the size in bytes of the file for key. A missing file is +// ErrNotFound. +func (s *FileStore) Size(key string) (int64, error) { + stat, err := os.Stat(s.keyToPath(key)) + if err != nil { + if os.IsNotExist(err) { + return 0, ErrNotFound + } + + return 0, fmt.Errorf("failed to stat content: %w", err) + } + + return stat.Size(), nil +} + +// Delete removes the file for key. A missing file is not an error. +func (s *FileStore) Delete(key string) error { + err := os.Remove(s.keyToPath(key)) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to delete content: %w", err) + } + + return nil +} + +// writeFileSynced writes data to path through a temp file in the same +// directory: the temp file is synced to disk, renamed to path, and the +// directory synced after, so that neither a concurrent reader nor a +// crash finds a partial file under path. When the directory sync fails, +// the file is removed from path, so that a failed write leaves no file +// there. +func writeFileSynced(path string, data []byte) error { + dir := filepath.Dir(path) + + tmpFile, err := os.CreateTemp(dir, ".tmp-*") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + + tmpPath := tmpFile.Name() + + _, err = tmpFile.Write(data) + if err != nil { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + + return fmt.Errorf("failed to write temp file: %w", err) + } + + err = tmpFile.Sync() + if err != nil { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + + return fmt.Errorf("failed to sync temp file: %w", err) + } + + err = tmpFile.Close() + if err != nil { + _ = os.Remove(tmpPath) + + return fmt.Errorf("failed to close temp file: %w", err) + } + + err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path)) + if err != nil { + _ = os.Remove(tmpPath) + + return fmt.Errorf("failed to rename temp file: %w", err) + } + + err = syncDirectory(dir) + if err != nil { + _ = os.Remove(path) + + return err + } + + return nil +} + +// syncDirectory syncs the directory dir to disk, so that a file just +// renamed into it is still there after a crash. +func syncDirectory(dir string) error { + d, err := os.Open(filepath.Clean(dir)) + if err != nil { + return fmt.Errorf("failed to open directory: %w", err) + } + + err = d.Sync() + _ = d.Close() + + if err != nil { + return fmt.Errorf("failed to sync directory: %w", err) + } + + return nil +} + +// keyToPath converts a key to a file path: /// +func (s *FileStore) keyToPath(key string) string { + if len(key) < MinHashLength { + return filepath.Join(s.baseDir, key) + } + + return filepath.Join(s.baseDir, key[0:2], key[2:4], key) +} diff --git a/internal/imgcache/filestore_internal_test.go b/internal/imgcache/filestore_internal_test.go new file mode 100644 index 0000000..06264c9 --- /dev/null +++ b/internal/imgcache/filestore_internal_test.go @@ -0,0 +1,184 @@ +package imgcache + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +func TestFileStore_WriteAndOpen(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + storage, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("NewFileStore() error = %v", err) + } + + content := []byte("hello world") + key := string(imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()) + + err = storage.Write(key, content) + if err != nil { + t.Fatalf("Write() error = %v", err) + } + + // Verify file exists at expected path + expectedPath := filepath.Join(tmpDir, key[0:2], key[2:4], key) + + _, err = os.Stat(expectedPath) + if err != nil { + t.Errorf("File not at expected path %s: %v", expectedPath, err) + } + + // Open and verify size and content + r, size, err := storage.Open(key) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + + defer func() { _ = r.Close() }() + + if size != int64(len(content)) { + t.Errorf("Open() size = %d, want %d", size, len(content)) + } + + loaded, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + + if !bytes.Equal(loaded, content) { + t.Errorf("Open() content = %q, want %q", loaded, content) + } +} + +func TestFileStore_OpenNotFound(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + storage, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("NewFileStore() error = %v", err) + } + + _, _, err = storage.Open("nonexistent") + if !errors.Is(err, ErrNotFound) { + t.Errorf("Open() error = %v, want ErrNotFound", err) + } +} + +// TestFileStore_Size checks that Size returns the size of a written file, +// and ErrNotFound for a key with no file. +func TestFileStore_Size(t *testing.T) { + t.Parallel() + + storage, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatalf("NewFileStore() error = %v", err) + } + + content := []byte("the file's content") + key := string(imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()) + + err = storage.Write(key, content) + if err != nil { + t.Fatalf("Write() error = %v", err) + } + + size, err := storage.Size(key) + if err != nil || size != int64(len(content)) { + t.Errorf("Size() = %d, error %v; want %d", size, err, len(content)) + } + + _, err = storage.Size("nonexistent") + if !errors.Is(err, ErrNotFound) { + t.Errorf("Size() of a missing file error = %v, want ErrNotFound", err) + } +} + +func TestFileStore_Delete(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + storage, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("NewFileStore() error = %v", err) + } + + content := []byte("to be deleted") + key := string(imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()) + path := storage.keyToPath(key) + + err = storage.Write(key, content) + if err != nil { + t.Fatalf("Write() error = %v", err) + } + + _, err = os.Stat(path) + if err != nil { + t.Errorf("file missing before Delete(): %v", err) + } + + err = storage.Delete(key) + if err != nil { + t.Fatalf("Delete() error = %v", err) + } + + _, err = os.Stat(path) + if err == nil { + t.Error("file still on disk after Delete()") + } +} + +func TestFileStore_DeleteNonexistent(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + storage, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("NewFileStore() error = %v", err) + } + + // Should not error + err = storage.Delete("nonexistent") + if err != nil { + t.Errorf("Delete() error = %v, want nil", err) + } +} + +func TestFileStore_KeyToPath(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + + storage, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("NewFileStore() error = %v", err) + } + + // Test by writing and verifying the resulting path structure + content := []byte("test content for path verification") + key := string(imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()) + + err = storage.Write(key, content) + if err != nil { + t.Fatalf("Write() error = %v", err) + } + + expectedPath := filepath.Join(tmpDir, key[0:2], key[2:4], key) + + _, err = os.Stat(expectedPath) + if err != nil { + t.Errorf("File not at expected path %s: %v", expectedPath, err) + } +} diff --git a/internal/imgcache/partial_cache_files_internal_test.go b/internal/imgcache/partial_cache_files_internal_test.go index b89b720..bcabf2a 100644 --- a/internal/imgcache/partial_cache_files_internal_test.go +++ b/internal/imgcache/partial_cache_files_internal_test.go @@ -67,8 +67,8 @@ func TestReconciliationRemovesDamagedFiles(t *testing.T) { sourceKey := storeEvictionTestSource(t, cache, testHostCDN, testPathCat, content) storeEvictionTestVariant(t, cache, testVariantKeyOne, content) - sourcePath := cache.srcContent.keyToPath(sourceKey) - variantPath := cache.variants.keyToPath(testVariantKeyOne) + sourcePath := cache.srcContent.keyToPath(string(sourceKey)) + variantPath := cache.variants.keyToPath(string(testVariantKeyOne)) for _, path := range []string{sourcePath, variantPath} { err := os.Truncate(path, size) @@ -82,11 +82,13 @@ func TestReconciliationRemovesDamagedFiles(t *testing.T) { assertWarningNames(t, logs, sourcePath) assertWarningNames(t, logs, variantPath) - if cache.srcContent.Exists(sourceKey) { + _, err := os.Stat(sourcePath) + if err == nil { t.Error("the damaged source file is still on disk") } - if cache.variants.Exists(testVariantKeyOne) { + _, err = os.Stat(variantPath) + if err == nil { t.Error("the damaged variant file is still on disk") } @@ -116,16 +118,19 @@ func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) { cache, _ := newEvictionTestCache(t, 1<<30) - _, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil)) + err := cache.variants.Write(string(testVariantKeyOne), nil) if err != nil { t.Fatalf("failed to store an empty variant file: %v", err) } + variantPath := cache.variants.keyToPath(string(testVariantKeyOne)) + logs := reconcileAsAtStartup(t, cache) - assertWarningNames(t, logs, cache.variants.keyToPath(testVariantKeyOne)) + assertWarningNames(t, logs, variantPath) - if cache.variants.Exists(testVariantKeyOne) { + _, err = os.Stat(variantPath) + if err == nil { t.Error("the empty variant file is still on disk") } @@ -137,19 +142,19 @@ func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) { } } -// TestContentStorage_StoreReplacesFileOfAnotherSize checks that a -// store over a file of another size at the source's path, such as a -// partial file a crash left, replaces it with the content. -func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) { +// TestFileStore_WriteReplacesFileOfAnotherSize checks that a write over a +// file of another size at the source's path, such as a partial file a crash +// left, replaces it with the content. +func TestFileStore_WriteReplacesFileOfAnotherSize(t *testing.T) { t.Parallel() - storage, err := NewContentStorage(t.TempDir()) + storage, err := NewFileStore(t.TempDir()) if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) + t.Fatalf("NewFileStore() error = %v", err) } content := []byte("the whole source image") - key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() + key := string(imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key()) path := storage.keyToPath(key) err = os.MkdirAll(filepath.Dir(path), StorageDirPerm) @@ -162,14 +167,14 @@ func TestContentStorage_StoreReplacesFileOfAnotherSize(t *testing.T) { t.Fatalf("writing a partial file: %v", err) } - _, err = storage.Store(key, content) + err = storage.Write(key, content) if err != nil { - t.Fatalf("Store() error = %v", err) + t.Fatalf("Write() error = %v", err) } - stored, _, err := storage.LoadWithSize(key) + stored, _, err := storage.Open(key) if err != nil { - t.Fatalf("LoadWithSize() error = %v", err) + t.Fatalf("Open() error = %v", err) } defer func() { _ = stored.Close() }() diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go index 3d12835..b9d7adb 100644 --- a/internal/imgcache/source_content_internal_test.go +++ b/internal/imgcache/source_content_internal_test.go @@ -74,7 +74,8 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { t.Fatalf("evictSourceBlob() error = %v", err) } - if cache.srcContent.Exists(evicted) { + _, err = os.Stat(cache.srcContent.keyToPath(string(evicted))) + if err == nil { t.Error("the evicted source's file is still on disk") } @@ -129,7 +130,8 @@ func TestPutSource_FailedRowWriteLeavesNoFile(t *testing.T) { t.Error("PutSource() error = nil, want the failed row write") } - if cache.srcContent.Exists(req.Source.Key()) { + _, err = os.Stat(cache.srcContent.keyToPath(string(req.Source.Key()))) + if err == nil { t.Error("the source's file is on disk although its row was not written") } } diff --git a/internal/imgcache/storage.go b/internal/imgcache/storage.go deleted file mode 100644 index d0da00f..0000000 --- a/internal/imgcache/storage.go +++ /dev/null @@ -1,326 +0,0 @@ -package imgcache - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - - "sneak.berlin/go/pixa/internal/imagerequest" - "sneak.berlin/go/pixa/internal/magic" -) - -// Storage constants. -const ( - // StorageDirPerm is the permission mode for storage directories. - StorageDirPerm = 0750 - // StorageFilePerm is the permission mode for storage files. - StorageFilePerm = 0600 - // MinHashLength is the minimum hash length for path splitting. - MinHashLength = 4 -) - -// Storage errors. -var ( - ErrNotFound = errors.New("content not found") -) - -// ContentStorage handles source image file storage, one file per source -// URL. Files are stored at: /// -type ContentStorage struct { - baseDir string -} - -// NewContentStorage creates a new content storage at the given base directory. -func NewContentStorage(baseDir string) (*ContentStorage, error) { - err := os.MkdirAll(baseDir, StorageDirPerm) - if err != nil { - return nil, fmt.Errorf("failed to create storage directory: %w", err) - } - - return &ContentStorage{baseDir: baseDir}, nil -} - -// Store writes data with writeFileSynced to the path derived from key, -// replacing any file already there, and returns its size. -func (s *ContentStorage) Store( - key imagerequest.SourceKey, data []byte, -) (int64, error) { - path := s.keyToPath(key) - - err := os.MkdirAll(filepath.Dir(path), StorageDirPerm) - if err != nil { - return 0, fmt.Errorf("failed to create directory: %w", err) - } - - err = writeFileSynced(path, data) - if err != nil { - return 0, err - } - - return int64(len(data)), nil -} - -// LoadWithSize returns a reader and file size for the content with the given key. -func (s *ContentStorage) LoadWithSize( - key imagerequest.SourceKey, -) (io.ReadCloser, int64, error) { - path := s.keyToPath(key) - - f, err := os.Open(path) //nolint:gosec // path derived from source key - if err != nil { - if os.IsNotExist(err) { - return nil, 0, ErrNotFound - } - - return nil, 0, fmt.Errorf("failed to open content: %w", err) - } - - stat, err := f.Stat() - if err != nil { - _ = f.Close() - - return nil, 0, fmt.Errorf("failed to stat content: %w", err) - } - - return f, stat.Size(), nil -} - -// Delete removes content with the given key. -func (s *ContentStorage) Delete(key imagerequest.SourceKey) error { - path := s.keyToPath(key) - - err := os.Remove(path) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete content: %w", err) - } - - return nil -} - -// Exists checks if content with the given key exists. -func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool { - path := s.keyToPath(key) - _, err := os.Stat(path) - - return err == nil -} - -// writeFileSynced writes data to path through a temp file in the same -// directory: the temp file is synced to disk, renamed to path, and the -// directory synced after, so that neither a concurrent reader nor a -// crash finds a partial file under path. When the directory sync fails, -// the file is removed from path, so that a failed write leaves no file -// there. -func writeFileSynced(path string, data []byte) error { - dir := filepath.Dir(path) - - tmpFile, err := os.CreateTemp(dir, ".tmp-*") - if err != nil { - return fmt.Errorf("failed to create temp file: %w", err) - } - - tmpPath := tmpFile.Name() - - _, err = tmpFile.Write(data) - if err != nil { - _ = tmpFile.Close() - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to write temp file: %w", err) - } - - err = tmpFile.Sync() - if err != nil { - _ = tmpFile.Close() - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to sync temp file: %w", err) - } - - err = tmpFile.Close() - if err != nil { - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to close temp file: %w", err) - } - - err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path)) - if err != nil { - _ = os.Remove(tmpPath) - - return fmt.Errorf("failed to rename temp file: %w", err) - } - - err = syncDirectory(dir) - if err != nil { - _ = os.Remove(path) - - return err - } - - return nil -} - -// syncDirectory syncs the directory dir to disk, so that a file just -// renamed into it is still there after a crash. -func syncDirectory(dir string) error { - d, err := os.Open(filepath.Clean(dir)) - if err != nil { - return fmt.Errorf("failed to open directory: %w", err) - } - - err = d.Sync() - _ = d.Close() - - if err != nil { - return fmt.Errorf("failed to sync directory: %w", err) - } - - return nil -} - -// keyToPath converts a key to a file path: /// -func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string { - k := string(key) - if len(k) < MinHashLength { - return filepath.Clean(filepath.Join(s.baseDir, k)) - } - - return filepath.Clean(filepath.Join(s.baseDir, k[0:2], k[2:4], k)) -} - -// VariantStorage handles key-based file storage for processed image variants. -// Files are stored at: /// -type VariantStorage struct { - baseDir string -} - -// errNotAnOutputFormat is returned for a variant file whose first bytes are -// those of a format pixa reads but never writes, such as SVG. -var errNotAnOutputFormat = errors.New("variant file is not in an output format") - -// NewVariantStorage creates a new variant storage at the given base directory. -func NewVariantStorage(baseDir string) (*VariantStorage, error) { - err := os.MkdirAll(baseDir, StorageDirPerm) - if err != nil { - return nil, fmt.Errorf("failed to create variant storage directory: %w", err) - } - - return &VariantStorage{baseDir: baseDir}, nil -} - -// Store writes content to storage at the given key with writeFileSynced. -func (s *VariantStorage) Store( - key imagerequest.VariantKey, r io.Reader, -) (int64, error) { - data, err := io.ReadAll(r) - if err != nil { - return 0, fmt.Errorf("failed to read content: %w", err) - } - - path := s.keyToPath(key) - - err = os.MkdirAll(filepath.Dir(path), StorageDirPerm) - if err != nil { - return 0, fmt.Errorf("failed to create directory: %w", err) - } - - err = writeFileSynced(path, data) - if err != nil { - return 0, err - } - - return int64(len(data)), nil -} - -// LoadWithContentType opens the content at the given key and returns a -// reader, the file size and the content type, which is read from the -// file's first bytes. A missing file is ErrNotFound. A file whose first -// bytes are not those of an output format, an empty one included, is -// closed again and the reason returned. -func (s *VariantStorage) LoadWithContentType( - key imagerequest.VariantKey, -) (io.ReadCloser, int64, string, error) { - path := s.keyToPath(key) - - f, err := os.Open(path) //nolint:gosec // path derived from cache key - if err != nil { - if os.IsNotExist(err) { - return nil, 0, "", ErrNotFound - } - - return nil, 0, "", fmt.Errorf("failed to open content: %w", err) - } - - stat, err := f.Stat() - if err != nil { - _ = f.Close() - - return nil, 0, "", fmt.Errorf("failed to stat content: %w", err) - } - - contentType, err := outputContentType(f) - if err != nil { - _ = f.Close() - - return nil, 0, "", err - } - - return f, stat.Size(), contentType, nil -} - -// outputContentType reads the first bytes of content, without moving its -// offset, and returns the content type of the output format they show. -func outputContentType(content io.ReaderAt) (string, error) { - firstBytes := make([]byte, magic.MinMagicBytes) - - _, err := content.ReadAt(firstBytes, 0) - if err != nil { - return "", fmt.Errorf("failed to read the first bytes: %w", err) - } - - detected, err := magic.DetectFormat(firstBytes) - if err != nil { - return "", err - } - - _, isOutputFormat := magic.MIMEToImageFormat(string(detected)) - if !isOutputFormat { - return "", fmt.Errorf("%w: %s", errNotAnOutputFormat, detected) - } - - return string(detected), nil -} - -// Exists checks if content exists at the given key. -func (s *VariantStorage) Exists(key imagerequest.VariantKey) bool { - path := s.keyToPath(key) - _, err := os.Stat(path) - - return err == nil -} - -// Delete removes the content at the given key. A missing file is not an -// error. -func (s *VariantStorage) Delete(key imagerequest.VariantKey) error { - path := s.keyToPath(key) - - err := os.Remove(path) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete content: %w", err) - } - - return nil -} - -// keyToPath converts a key to a file path: /// -func (s *VariantStorage) keyToPath(key imagerequest.VariantKey) string { - k := string(key) - if len(k) < MinHashLength { - return filepath.Join(s.baseDir, k) - } - - return filepath.Join(s.baseDir, k[0:2], k[2:4], k) -} diff --git a/internal/imgcache/storage_internal_test.go b/internal/imgcache/storage_internal_test.go deleted file mode 100644 index a3e0e78..0000000 --- a/internal/imgcache/storage_internal_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package imgcache - -import ( - "bytes" - "errors" - "io" - "os" - "path/filepath" - "testing" - - "sneak.berlin/go/pixa/internal/imagerequest" -) - -func TestContentStorage_StoreAndLoadWithSize(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewContentStorage(tmpDir) - if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) - } - - content := []byte("hello world") - key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() - - size, err := storage.Store(key, content) - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - if size != int64(len(content)) { - t.Errorf("Store() size = %d, want %d", size, len(content)) - } - - // Verify file exists at expected path - keyStr := string(key) - - expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr) - - _, err = os.Stat(expectedPath) - if err != nil { - t.Errorf("File not at expected path %s: %v", expectedPath, err) - } - - // Load and verify content - r, _, err := storage.LoadWithSize(key) - if err != nil { - t.Fatalf("LoadWithSize() error = %v", err) - } - - defer func() { _ = r.Close() }() - - loaded, err := io.ReadAll(r) - if err != nil { - t.Fatalf("ReadAll() error = %v", err) - } - - if !bytes.Equal(loaded, content) { - t.Errorf("LoadWithSize() content = %q, want %q", loaded, content) - } -} - -func TestContentStorage_LoadWithSizeNotFound(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewContentStorage(tmpDir) - if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) - } - - _, _, err = storage.LoadWithSize(imagerequest.SourceKey("nonexistent")) - if !errors.Is(err, ErrNotFound) { - t.Errorf("LoadWithSize() error = %v, want ErrNotFound", err) - } -} - -func TestContentStorage_Delete(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewContentStorage(tmpDir) - if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) - } - - content := []byte("to be deleted") - key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() - - _, err = storage.Store(key, content) - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - if !storage.Exists(key) { - t.Error("Exists() = false, want true") - } - - err = storage.Delete(key) - if err != nil { - t.Fatalf("Delete() error = %v", err) - } - - if storage.Exists(key) { - t.Error("Exists() = true after delete, want false") - } -} - -func TestContentStorage_DeleteNonexistent(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewContentStorage(tmpDir) - if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) - } - - // Should not error - err = storage.Delete(imagerequest.SourceKey("nonexistent")) - if err != nil { - t.Errorf("Delete() error = %v, want nil", err) - } -} - -func TestContentStorage_KeyToPath(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - - storage, err := NewContentStorage(tmpDir) - if err != nil { - t.Fatalf("NewContentStorage() error = %v", err) - } - - // Test by storing and verifying the resulting path structure - content := []byte("test content for path verification") - key := imagerequest.Source{Host: testHostCDN, Path: testPathCat}.Key() - - _, err = storage.Store(key, content) - if err != nil { - t.Fatalf("Store() error = %v", err) - } - - keyStr := string(key) - - expectedPath := filepath.Join(tmpDir, keyStr[0:2], keyStr[2:4], keyStr) - - _, err = os.Stat(expectedPath) - if err != nil { - t.Errorf("File not at expected path %s: %v", expectedPath, err) - } -} diff --git a/internal/imgcache/testdata/red.avif b/internal/imgcache/testdata/red.avif deleted file mode 100644 index ae4624ba21e6803af856e275967a8231fa1b9a55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 281 zcmZQzV30{GsVqn=%S>Ycg51nBLl8SRGZDnUmYZ6V2oeVZ#f+4kA_$X#p&&E41jdHZ zdkW|Xd$xjCHTpSn}fe-|KL1_jCc_2?YGcyltDOgk#$Vp`asRF7^EHf|! zF~c$oiVA?VMP@;AK9J@~EHiX&bjSh8iDVWRq=Fbgzyier1(_9@AOU6`9v&bimRXPs zb3f1#%#BRUKvDJFl*AGt1}=^WmfBuMUbcruf)WgT7LQvP7&Zupyfs+zn&%Gyqc|_Q From b73bbde46ed0b6b1aa9dc3396a91cfc30e281372 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 21:21:39 +0100 Subject: [PATCH 109/131] Detect a fetched image's format from its bytes once (closes #319) magic.DetectFormat returns an imagerequest.Format. ValidateMagicBytes returns the format it checked against the upstream Content-Type. The cache reads a cached source's format from its file's first bytes, as it reads a variant's. The proxy hands the format to Process. Process refuses a format that is not a source format, and input libvips would load with another loader, such as an SVG passing as AVIF. Format.MIMEType declares each MIME string once, and the fetcher, the cache's variant read and the Accept matching use it. magic.MIMEType, magic.ImageFormat, MIMEToImageFormat, magic's SVG detection, detectFormat, formatFromString, FormatToMIME and Result.InputFormat are gone. Model: opus-5-5 --- README.md | 9 +- TODO.md | 24 +++ internal/handlers/format_auto.go | 8 +- .../unsupported_source_internal_test.go | 6 +- internal/httpfetcher/fetch_internal_test.go | 6 +- internal/httpfetcher/httpfetcher.go | 27 +-- .../httpfetcher/httpfetcher_internal_test.go | 22 +- internal/httpfetcher/mock.go | 18 +- .../httpfetcher/request_id_internal_test.go | 3 +- .../avif_too_large_internal_test.go | 33 +-- .../computed_side_internal_test.go | 4 +- .../imageprocessor/deadline_internal_test.go | 5 +- .../empty_format_internal_test.go | 2 +- .../encoder_settings_internal_test.go | 17 +- .../imageprocessor/fit_cover_internal_test.go | 6 +- .../fit_outside_internal_test.go | 4 +- internal/imageprocessor/imageprocessor.go | 147 +++++-------- .../imageprocessor_internal_test.go | 60 +++--- .../jpegxl_output_internal_test.go | 5 +- ...max_concurrent_processing_internal_test.go | 10 +- .../orig_format_internal_test.go | 7 +- .../source_formats_internal_test.go | 33 ++- internal/imageproxy/imageproxy.go | 50 +++-- .../imageproxy/imageproxy_internal_test.go | 17 +- ...max_concurrent_processing_internal_test.go | 14 +- .../negative_cache_internal_test.go | 2 +- .../partial_cache_files_internal_test.go | 37 +++- internal/imagerequest/imagerequest.go | 34 ++- .../imagerequest/mimetype_internal_test.go | 30 +++ internal/imgcache/cache.go | 56 +++-- internal/imgcache/cache_hit_internal_test.go | 6 +- internal/imgcache/eviction_internal_test.go | 2 +- .../pendingaccesstimes_internal_test.go | 9 +- .../imgcache/source_content_internal_test.go | 105 +++++++++- internal/magic/jpegxl_internal_test.go | 27 +-- internal/magic/magic.go | 119 ++--------- internal/magic/magic_internal_test.go | 195 +++--------------- 37 files changed, 570 insertions(+), 589 deletions(-) create mode 100644 internal/imagerequest/mimetype_internal_test.go diff --git a/README.md b/README.md index 0f3271a..5e1437c 100644 --- a/README.md +++ b/README.md @@ -191,10 +191,11 @@ allowlisted source hosts for open access. name. Each source image and each transformed image has one row in the database, -holding its size and when it was last used. Two source URLs that serve the same -bytes are stored twice, once under each URL. A source image is stored by writing -its file and then its row; when the row cannot be written, the file is removed -again. +holding its size and when it was last used. pixa reads a cached source image's +format from its file's first bytes. When they match none of the source formats, +pixa fetches the source again. Two source URLs that serve the same bytes are +stored twice, once under each URL. A source image is stored by writing its file +and then its row; when the row cannot be written, the file is removed again. Source and transformed image files are written under a temporary name, synced to disk, and only then renamed into place, so a crash leaves no file under its own diff --git a/TODO.md b/TODO.md index e76626d..39e0f30 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,30 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 a fetched image's format is detected from its bytes once, as an + `imagerequest.Format` (closes #319). The upstream `Content-Type` check in the + fetcher stays as the filter before downloading. + - `magic.DetectFormat` returns an `imagerequest.Format`, and + `magic.ValidateMagicBytes` returns the format it checked against the + `Content-Type`. + - `Format.MIMEType` gives each format's MIME type. The fetcher, the mock + fetcher, the cache's read of a variant's type and the handler's `Accept` + matching use it. + - `Cache.Source` returns a source's format, read from its file's first bytes + as `Cache.Variant` reads a variant's. A file in no source format is an + error, and the proxy fetches the source again. + - The proxy hands the format to `ImageProcessor.Process`. The processor + refuses input that libvips would load as another format, as before, with + `ErrUnsupportedInputFormat`. It refuses a format that is not one of the + six source formats the same way. + - `internal/magic` no longer detects SVG, which pixa does not read. SVG data + is `ErrUnknownFormat`. + - Removed: + - `magic.MIMEType`, `magic.ImageFormat` and `magic.MIMEToImageFormat` + - `detectFormat`, `formatFromString`, `FormatToMIME` and + `Result.InputFormat` of the processor + - the fetcher's MIME constants of the six formats + - `errNotAnOutputFormat` of the cache - 2026-10-26 the cache keeps its source images in one `imgcache.FileStore` and its variants in another (closes #318). A `FileStore` is a directory of files, each named by a hex key, at `//`. diff --git a/internal/handlers/format_auto.go b/internal/handlers/format_auto.go index 96a1b06..c54bebd 100644 --- a/internal/handlers/format_auto.go +++ b/internal/handlers/format_auto.go @@ -65,20 +65,20 @@ func formatForAccept(accept string) (imagerequest.Format, error) { return imagerequest.FormatJPEG, nil } - if qualities["image/jxl"] > 0 { + if qualities[imagerequest.FormatJXL.MIMEType()] > 0 { return imagerequest.FormatJXL, nil } - if qualities["image/avif"] > 0 { + if qualities[imagerequest.FormatAVIF.MIMEType()] > 0 { return imagerequest.FormatAVIF, nil } - if qualities["image/webp"] > 0 { + if qualities[imagerequest.FormatWebP.MIMEType()] > 0 { return imagerequest.FormatWebP, nil } // For JPEG, the most specific entry the header has decides - quality, named := qualities["image/jpeg"] + quality, named := qualities[imagerequest.FormatJPEG.MIMEType()] if !named { quality, named = qualities["image/*"] } diff --git a/internal/handlers/unsupported_source_internal_test.go b/internal/handlers/unsupported_source_internal_test.go index 030274f..dbb05b3 100644 --- a/internal/handlers/unsupported_source_internal_test.go +++ b/internal/handlers/unsupported_source_internal_test.go @@ -16,8 +16,8 @@ import ( // TestUnsupportedSourceFormatAnswers415 checks that respondImageError, which // answers the errors of both image routes, answers 415 with a clear error when // the upstream answers with a content type pixa does not accept, such as SVG, -// and when orig is asked of a source in a format the processor cannot name, -// wrapped as the service wraps them. +// and when libvips does not read a source as the format its first bytes +// show, wrapped as the service wraps them. func TestUnsupportedSourceFormatAnswers415(t *testing.T) { t.Parallel() @@ -29,7 +29,7 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) { for _, err := range []error{ fmt.Errorf("upstream fetch failed: %w: image/svg+xml", httpfetcher.ErrInvalidContentType), - fmt.Errorf("image processing failed: %w: unknown", + fmt.Errorf("image processing failed: %w: libvips does not read the input as avif", imageprocessor.ErrUnsupportedInputFormat), } { rec := httptest.NewRecorder() diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index 0d515d7..7c5199f 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -83,7 +83,7 @@ func startUpstream(t *testing.T) *httptest.Server { mux := http.NewServeMux() mux.HandleFunc("/image", func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeJPEG) + w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType()) _, _ = io.WriteString(w, imagePayload) }) mux.HandleFunc("/status/500", func(w http.ResponseWriter, _ *http.Request) { @@ -358,7 +358,7 @@ func TestFetchContentLengthBodyAtMaxResponseSize(t *testing.T) { res := fetchWithMaxResponseSize(t, maxResponseSize, func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", contentTypeJPEG) + w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType()) w.Header().Set("Content-Length", strconv.Itoa(tc.size)) _, _ = io.WriteString(w, body) }) @@ -406,7 +406,7 @@ func TestFetchChunkedBodyAtMaxResponseSize(t *testing.T) { res := fetchWithMaxResponseSize(t, maxResponseSize, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", contentTypeJPEG) + w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType()) _, _ = io.WriteString(w, body) // With no Content-Length, flushing sends the body // chunked; its end is sent when the handler returns. diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 08eeee1..d532288 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -38,18 +38,6 @@ const userAgent = "pixa/1.0" // MaxConnections fetches are already in progress. const ConnectionWaitTimeout = 10 * time.Second -// MIME content types. -const ( - contentTypeJPEG = "image/jpeg" - contentTypePNG = "image/png" - contentTypeGIF = "image/gif" - contentTypeWebP = "image/webp" - contentTypeAVIF = "image/avif" - contentTypeJXL = "image/jxl" - contentTypeSVG = "image/svg+xml" - contentTypeOctetStream = "application/octet-stream" -) - // Loopback addresses blocked by SSRF protection. const ( localhostIPv4 = "127.0.0.1" @@ -121,15 +109,16 @@ type FetchResult struct { } // allowedContentTypes returns the content types an upstream response may -// have, which Fetch also sends as its Accept header. +// have, which Fetch also sends as its Accept header: the MIME types of the +// formats pixa reads. func allowedContentTypes() []string { return []string{ - contentTypeJPEG, - contentTypePNG, - contentTypeGIF, - contentTypeWebP, - contentTypeAVIF, - contentTypeJXL, + imagerequest.FormatJPEG.MIMEType(), + imagerequest.FormatPNG.MIMEType(), + imagerequest.FormatGIF.MIMEType(), + imagerequest.FormatWebP.MIMEType(), + imagerequest.FormatAVIF.MIMEType(), + imagerequest.FormatJXL.MIMEType(), } } diff --git a/internal/httpfetcher/httpfetcher_internal_test.go b/internal/httpfetcher/httpfetcher_internal_test.go index 8b71419..e1400a2 100644 --- a/internal/httpfetcher/httpfetcher_internal_test.go +++ b/internal/httpfetcher/httpfetcher_internal_test.go @@ -30,9 +30,9 @@ func TestIsAllowedContentType(t *testing.T) { contentType string want bool }{ - {contentTypeJPEG, true}, - {contentTypePNG, true}, - {contentTypeWebP, true}, + {imagerequest.FormatJPEG.MIMEType(), true}, + {imagerequest.FormatPNG.MIMEType(), true}, + {imagerequest.FormatWebP.MIMEType(), true}, {"image/jpeg; charset=utf-8", true}, {"IMAGE/JPEG", true}, {"text/html", false}, @@ -217,7 +217,7 @@ func TestMockFetcher_FetchesFile(t *testing.T) { defer func() { _ = result.Content.Close() }() - if result.ContentType != contentTypeJPEG { + if result.ContentType != imagerequest.FormatJPEG.MIMEType() { t.Errorf("ContentType = %q, want image/jpeg", result.ContentType) } @@ -272,13 +272,13 @@ func TestDetectContentTypeFromPath(t *testing.T) { path string want string }{ - {"foo/bar.jpg", contentTypeJPEG}, - {"foo/bar.JPG", contentTypeJPEG}, - {"foo/bar.jpeg", contentTypeJPEG}, - {"foo/bar.png", contentTypePNG}, - {"foo/bar.gif", contentTypeGIF}, - {"foo/bar.webp", contentTypeWebP}, - {"foo/bar.avif", contentTypeAVIF}, + {"foo/bar.jpg", imagerequest.FormatJPEG.MIMEType()}, + {"foo/bar.JPG", imagerequest.FormatJPEG.MIMEType()}, + {"foo/bar.jpeg", imagerequest.FormatJPEG.MIMEType()}, + {"foo/bar.png", imagerequest.FormatPNG.MIMEType()}, + {"foo/bar.gif", imagerequest.FormatGIF.MIMEType()}, + {"foo/bar.webp", imagerequest.FormatWebP.MIMEType()}, + {"foo/bar.avif", imagerequest.FormatAVIF.MIMEType()}, {"foo/bar.svg", contentTypeSVG}, {"foo/bar.bin", contentTypeOctetStream}, {"foo/bar", contentTypeOctetStream}, diff --git a/internal/httpfetcher/mock.go b/internal/httpfetcher/mock.go index 40ac3e6..1e3268b 100644 --- a/internal/httpfetcher/mock.go +++ b/internal/httpfetcher/mock.go @@ -66,23 +66,29 @@ func (m *MockFetcher) Fetch( }, nil } +// The content types the mock fetcher gives files in no format pixa reads. +const ( + contentTypeSVG = "image/svg+xml" + contentTypeOctetStream = "application/octet-stream" +) + // detectContentTypeFromPath returns the MIME type based on file extension. func detectContentTypeFromPath(path string) string { path = strings.ToLower(path) switch { case strings.HasSuffix(path, ".jpg"), strings.HasSuffix(path, ".jpeg"): - return contentTypeJPEG + return imagerequest.FormatJPEG.MIMEType() case strings.HasSuffix(path, ".png"): - return contentTypePNG + return imagerequest.FormatPNG.MIMEType() case strings.HasSuffix(path, ".gif"): - return contentTypeGIF + return imagerequest.FormatGIF.MIMEType() case strings.HasSuffix(path, ".webp"): - return contentTypeWebP + return imagerequest.FormatWebP.MIMEType() case strings.HasSuffix(path, ".avif"): - return contentTypeAVIF + return imagerequest.FormatAVIF.MIMEType() case strings.HasSuffix(path, ".jxl"): - return contentTypeJXL + return imagerequest.FormatJXL.MIMEType() case strings.HasSuffix(path, ".svg"): return contentTypeSVG default: diff --git a/internal/httpfetcher/request_id_internal_test.go b/internal/httpfetcher/request_id_internal_test.go index 62a654d..b6e87b8 100644 --- a/internal/httpfetcher/request_id_internal_test.go +++ b/internal/httpfetcher/request_id_internal_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/go-chi/chi/v5/middleware" + "sneak.berlin/go/pixa/internal/imagerequest" ) // TestFetchSendsRequestID verifies that a fetch sends the ID of the request @@ -25,7 +26,7 @@ func TestFetchSendsRequestID(t *testing.T) { func(w http.ResponseWriter, r *http.Request) { received <- r.Header.Get("X-Request-Id") - w.Header().Set("Content-Type", contentTypeJPEG) + w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType()) _, _ = io.WriteString(w, imagePayload) })) t.Cleanup(srv.Close) diff --git a/internal/imageprocessor/avif_too_large_internal_test.go b/internal/imageprocessor/avif_too_large_internal_test.go index 15b184c..7850e66 100644 --- a/internal/imageprocessor/avif_too_large_internal_test.go +++ b/internal/imageprocessor/avif_too_large_internal_test.go @@ -39,30 +39,31 @@ func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) { tests := []struct { name string input []byte + sourceFormat imagerequest.Format format imagerequest.Format width, height int fit imagerequest.Fit refused bool }{ - {"AVIF at the limit", cutOff, imagerequest.FormatAVIF, - 4096, 4096, imagerequest.FitCover, false}, - {"AVIF above the limit", cutOff, imagerequest.FormatAVIF, - 4096, 4097, imagerequest.FitCover, true}, - {"WebP above the AVIF limit", cutOff, imagerequest.FormatWebP, - 4096, 4097, imagerequest.FitCover, false}, + {"AVIF at the limit", cutOff, imagerequest.FormatPNG, + imagerequest.FormatAVIF, 4096, 4096, imagerequest.FitCover, false}, + {"AVIF above the limit", cutOff, imagerequest.FormatPNG, + imagerequest.FormatAVIF, 4096, 4097, imagerequest.FitCover, true}, + {"WebP above the AVIF limit", cutOff, imagerequest.FormatPNG, + imagerequest.FormatWebP, 4096, 4097, imagerequest.FitCover, false}, // FitInside does not enlarge, so the output stays 64x32 - {"AVIF inside a larger size", cutOff, imagerequest.FormatAVIF, - 8192, 8192, imagerequest.FitInside, false}, + {"AVIF inside a larger size", cutOff, imagerequest.FormatPNG, + imagerequest.FormatAVIF, 8192, 8192, imagerequest.FitInside, false}, // FitOutside scales 64x32 by 128 to cover 8192x1: 8192x4096 - {"AVIF outside a smaller size", cutOff, imagerequest.FormatAVIF, - 8192, 1, imagerequest.FitOutside, true}, + {"AVIF outside a smaller size", cutOff, imagerequest.FormatPNG, + imagerequest.FormatAVIF, 8192, 1, imagerequest.FitOutside, true}, // With no size, the output keeps the source's 4097x4096 {"AVIF of the source's size", largeAVIF, imagerequest.FormatAVIF, - 0, 0, imagerequest.FitCover, true}, - {"orig of an AVIF source", largeAVIF, imagerequest.FormatOriginal, - 0, 0, imagerequest.FitCover, true}, - {"JPEG of the source's size", largeAVIF, imagerequest.FormatJPEG, - 0, 0, imagerequest.FitCover, false}, + imagerequest.FormatAVIF, 0, 0, imagerequest.FitCover, true}, + {"orig of an AVIF source", largeAVIF, imagerequest.FormatAVIF, + imagerequest.FormatOriginal, 0, 0, imagerequest.FitCover, true}, + {"JPEG of the source's size", largeAVIF, imagerequest.FormatAVIF, + imagerequest.FormatJPEG, 0, 0, imagerequest.FitCover, false}, } for _, tt := range tests { @@ -72,7 +73,7 @@ func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) { _, err := New(Params{ MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, - }).Process(t.Context(), bytes.NewReader(tt.input), + }).Process(t.Context(), bytes.NewReader(tt.input), tt.sourceFormat, &imagerequest.Transform{ Size: imagerequest.Size{Width: tt.width, Height: tt.height}, Format: tt.format, diff --git a/internal/imageprocessor/computed_side_internal_test.go b/internal/imageprocessor/computed_side_internal_test.go index 6bfb23a..496b4cb 100644 --- a/internal/imageprocessor/computed_side_internal_test.go +++ b/internal/imageprocessor/computed_side_internal_test.go @@ -42,7 +42,7 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatJPEG, &imagerequest.Transform{ Size: tt.size, Format: imagerequest.FormatJPEG, @@ -111,7 +111,7 @@ func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatJPEG, &imagerequest.Transform{ Size: tt.size, Format: imagerequest.FormatJPEG, diff --git a/internal/imageprocessor/deadline_internal_test.go b/internal/imageprocessor/deadline_internal_test.go index 9f6d394..b63e79e 100644 --- a/internal/imageprocessor/deadline_internal_test.go +++ b/internal/imageprocessor/deadline_internal_test.go @@ -7,6 +7,8 @@ import ( "io" "testing" "time" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // readerAfterDeadline is a Process input whose first Read waits until done @@ -86,7 +88,8 @@ func checkProcessStopsAtDeadline( proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) - result, err := proc.Process(ctx, input, smallJPEGTransform()) + result, err := proc.Process(ctx, input, imagerequest.FormatJPEG, + smallJPEGTransform()) if result != nil { t.Logf("Process() made a %dx%d image after its deadline passed", result.Width, result.Height) diff --git a/internal/imageprocessor/empty_format_internal_test.go b/internal/imageprocessor/empty_format_internal_test.go index 48bba8e..de54111 100644 --- a/internal/imageprocessor/empty_format_internal_test.go +++ b/internal/imageprocessor/empty_format_internal_test.go @@ -19,7 +19,7 @@ func TestImageProcessor_EmptyFormatRefused(t *testing.T) { MaxConcurrentProcessing: 1, }).Process( t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), - &imagerequest.Transform{}, + imagerequest.FormatJPEG, &imagerequest.Transform{}, ) if !errors.Is(err, ErrUnsupportedOutputFormat) { t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat) diff --git a/internal/imageprocessor/encoder_settings_internal_test.go b/internal/imageprocessor/encoder_settings_internal_test.go index dc93d76..b414e20 100644 --- a/internal/imageprocessor/encoder_settings_internal_test.go +++ b/internal/imageprocessor/encoder_settings_internal_test.go @@ -11,17 +11,18 @@ import ( "sneak.berlin/go/pixa/internal/imagerequest" ) -// processedSize runs input through Process and returns the output's size in -// bytes. +// processedSize runs input, an image in sourceFormat, through Process and +// returns the output's size in bytes. func processedSize( - t *testing.T, input []byte, transform *imagerequest.Transform, + t *testing.T, input []byte, sourceFormat imagerequest.Format, + transform *imagerequest.Transform, ) int64 { t.Helper() result, err := New(Params{ MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, - }).Process(t.Context(), bytes.NewReader(input), transform) + }).Process(t.Context(), bytes.NewReader(input), sourceFormat, transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -74,7 +75,7 @@ func TestImageProcessor_PNGIsCompressed(t *testing.T) { } } - size := processedSize(t, encodePNG(t, flat), + size := processedSize(t, encodePNG(t, flat), imagerequest.FormatPNG, &imagerequest.Transform{Format: imagerequest.FormatPNG}) const rawBytes = width * height * 3 @@ -92,7 +93,7 @@ func TestImageProcessor_WebPAtEffort2(t *testing.T) { input := createTestJPEG(t, 200, 150) - size := processedSize(t, input, + size := processedSize(t, input, imagerequest.FormatJPEG, &imagerequest.Transform{Format: imagerequest.FormatWebP, Quality: 85}) want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{ @@ -134,7 +135,7 @@ func TestImageProcessor_AVIFAtEffort1(t *testing.T) { input := encodePNG(t, source) - size := processedSize(t, input, + size := processedSize(t, input, imagerequest.FormatPNG, &imagerequest.Transform{Format: imagerequest.FormatAVIF, Quality: 85}) want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{ @@ -161,7 +162,7 @@ func TestImageProcessor_JPEGXLAtEffort4(t *testing.T) { input := createTestJPEG(t, 200, 150) - size := processedSize(t, input, + size := processedSize(t, input, imagerequest.FormatJPEG, &imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: 85}) params := vips.NewJxlExportParams() diff --git a/internal/imageprocessor/fit_cover_internal_test.go b/internal/imageprocessor/fit_cover_internal_test.go index a12d155..ea347ac 100644 --- a/internal/imageprocessor/fit_cover_internal_test.go +++ b/internal/imageprocessor/fit_cover_internal_test.go @@ -40,7 +40,7 @@ func TestImageProcessor_FitCoverServesThinSource(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatJPEG, &imagerequest.Transform{ Size: tt.size, Format: imagerequest.FormatJPEG, @@ -113,7 +113,7 @@ func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatPNG, &imagerequest.Transform{ Size: imagerequest.Size{Width: 50, Height: 50}, Format: imagerequest.FormatPNG, @@ -240,7 +240,7 @@ func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatPNG, &imagerequest.Transform{ Size: tt.size, Format: imagerequest.FormatPNG, diff --git a/internal/imageprocessor/fit_outside_internal_test.go b/internal/imageprocessor/fit_outside_internal_test.go index 3c72d59..12e7e23 100644 --- a/internal/imageprocessor/fit_outside_internal_test.go +++ b/internal/imageprocessor/fit_outside_internal_test.go @@ -45,7 +45,7 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatJPEG, &imagerequest.Transform{ Size: tt.size, Format: imagerequest.FormatJPEG, @@ -102,7 +102,7 @@ func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatJPEG, &imagerequest.Transform{ Size: tt.size, Format: imagerequest.FormatJPEG, diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index afe216e..18fdd12 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -70,8 +70,6 @@ type Result struct { InputWidth int // InputHeight is the original image height before processing. InputHeight int - // InputFormat is the detected input format (e.g., "jpeg", "png"). - InputFormat string } // MaxInputDimension is the maximum allowed width or height for input images. @@ -113,9 +111,10 @@ var ErrAVIFOutputTooLarge = errors.New("AVIF output has too many pixels") var ErrUnsupportedOutputFormat = errors.New("unsupported output format") // ErrUnsupportedInputFormat is returned, before the input is decoded, when -// libvips reads the input's first bytes as another format than the source -// formats, or as none: for example an SVG whose first bytes pass the -// first-bytes check for AVIF. +// libvips reads the input's first bytes as another format than its source +// format, or as none. An SVG whose first bytes pass the first-bytes check for +// AVIF is one example. A source format other than the six pixa reads is also +// refused with it. var ErrUnsupportedInputFormat = errors.New("unsupported input format") // ErrTooManyImages is returned when MaxConcurrentProcessing images are being @@ -157,9 +156,10 @@ func New(params Params) *ImageProcessor { } } -// Process transforms an image according to transform. When -// MaxConcurrentProcessing images are already being processed, it waits up -// to ProcessingWaitTimeout for one to finish, then fails with +// Process transforms the image read from input according to transform. +// sourceFormat is the input's format, which magic detected from its first +// bytes. When MaxConcurrentProcessing images are already being processed, +// it waits up to ProcessingWaitTimeout for one to finish, then fails with // ErrTooManyImages. When ctx has ended once Process has a slot, or before it // encodes, it returns ctx's error and gives the slot back. govips gives pixa // no way to stop libvips once an encode has begun, so an encode keeps its @@ -168,6 +168,7 @@ func New(params Params) *ImageProcessor { func (p *ImageProcessor) Process( ctx context.Context, input io.Reader, + sourceFormat imagerequest.Format, transform *imagerequest.Transform, ) (*Result, error) { release, err := p.acquireSlot(ctx) @@ -182,13 +183,9 @@ func (p *ImageProcessor) Process( return nil, err } - // govips picks the libvips loader from the input's first bytes with - // DetermineImageType. Input whose first bytes it reads as no source - // format, such as SVG, which librsvg would render, is refused before - // libvips reads it. - inputFormat := p.detectFormat(vips.DetermineImageType(data)) - - sourceFormat, err := p.formatFromString(inputFormat) + // Input libvips would not load as sourceFormat is refused before it is + // decoded + err = checkInputFormat(data, sourceFormat) if err != nil { return nil, err } @@ -254,12 +251,11 @@ func (p *ImageProcessor) Process( return &Result{ Content: io.NopCloser(bytes.NewReader(output)), ContentLength: int64(len(output)), - ContentType: FormatToMIME(outputFormat), + ContentType: outputFormat.MIMEType(), Width: img.Width(), Height: img.Height(), InputWidth: origWidth, InputHeight: origHeight, - InputFormat: inputFormat, }, nil } @@ -294,38 +290,6 @@ func targetDimensions( return width, height, nil } -// MIME types for the supported image formats. -const ( - mimeJPEG = "image/jpeg" - mimePNG = "image/png" - mimeGIF = "image/gif" - mimeWebP = "image/webp" - mimeAVIF = "image/avif" - mimeJXL = "image/jxl" -) - -// FormatToMIME converts a Format to its MIME type string. -func FormatToMIME(format imagerequest.Format) string { - switch format { - case imagerequest.FormatJPEG: - return mimeJPEG - case imagerequest.FormatPNG: - return mimePNG - case imagerequest.FormatWebP: - return mimeWebP - case imagerequest.FormatGIF: - return mimeGIF - case imagerequest.FormatAVIF: - return mimeAVIF - case imagerequest.FormatJXL: - return mimeJXL - case imagerequest.FormatOriginal, imagerequest.FormatAuto: - return "application/octet-stream" - default: - return "application/octet-stream" - } -} - // WaitForProcessing waits until no image is being processed, or until ctx // ends, and returns how many images were still being processed then. It // waits by taking each slot in processingSemaphore as it frees up until it @@ -394,27 +358,49 @@ func (p *ImageProcessor) readInput(input io.Reader) ([]byte, error) { return data, nil } -// detectFormat returns the format string for a govips image type. -func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string { - switch imageType { - case vips.ImageTypeJPEG: - return "jpeg" - case vips.ImageTypePNG: - return "png" - case vips.ImageTypeGIF: - return "gif" - case vips.ImageTypeWEBP: - return "webp" - case vips.ImageTypeAVIF, vips.ImageTypeHEIF: - return string(imagerequest.FormatAVIF) - case vips.ImageTypeJXL: - return string(imagerequest.FormatJXL) - case vips.ImageTypeUnknown, vips.ImageTypeMagick, vips.ImageTypePDF, - vips.ImageTypeSVG, vips.ImageTypeTIFF, vips.ImageTypeBMP, - vips.ImageTypeJP2K: - return "unknown" +// checkInputFormat returns ErrUnsupportedInputFormat unless libvips would +// load data with the loader of sourceFormat. govips picks the libvips loader +// from the input's first bytes with DetermineImageType. It refuses an SVG +// whose first bytes pass the first-bytes check for AVIF, which librsvg would +// otherwise render. +func checkInputFormat(data []byte, sourceFormat imagerequest.Format) error { + // A format with no loader, such as orig or auto, has the type govips also + // gives input it cannot read, which the check below would let through. + sourceType := vipsImageType(sourceFormat) + if sourceType == vips.ImageTypeUnknown { + return fmt.Errorf("%w: %q is not a source format", + ErrUnsupportedInputFormat, sourceFormat) + } + + if vips.DetermineImageType(data) != sourceType { + return fmt.Errorf("%w: libvips does not read the input as %s", + ErrUnsupportedInputFormat, sourceFormat) + } + + return nil +} + +// vipsImageType returns the govips image type whose libvips loader reads +// format. orig and auto stand for another format, so they have +// vips.ImageTypeUnknown, which govips has no loader for. +func vipsImageType(format imagerequest.Format) vips.ImageType { + switch format { + case imagerequest.FormatJPEG: + return vips.ImageTypeJPEG + case imagerequest.FormatPNG: + return vips.ImageTypePNG + case imagerequest.FormatGIF: + return vips.ImageTypeGIF + case imagerequest.FormatWebP: + return vips.ImageTypeWEBP + case imagerequest.FormatAVIF: + return vips.ImageTypeAVIF + case imagerequest.FormatJXL: + return vips.ImageTypeJXL + case imagerequest.FormatOriginal, imagerequest.FormatAuto: + return vips.ImageTypeUnknown default: - return "unknown" + return vips.ImageTypeUnknown } } @@ -768,26 +754,3 @@ func jxlDistance(quality int) float64 { return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0 } - -// formatFromString converts a format string from detectFormat to Format. A -// format detectFormat could not name is ErrUnsupportedInputFormat. -func (p *ImageProcessor) formatFromString( - format string, -) (imagerequest.Format, error) { - switch format { - case "jpeg": - return imagerequest.FormatJPEG, nil - case "png": - return imagerequest.FormatPNG, nil - case "gif": - return imagerequest.FormatGIF, nil - case "webp": - return imagerequest.FormatWebP, nil - case string(imagerequest.FormatAVIF): - return imagerequest.FormatAVIF, nil - case string(imagerequest.FormatJXL): - return imagerequest.FormatJXL, nil - default: - return "", fmt.Errorf("%w: %s", ErrUnsupportedInputFormat, format) - } -} diff --git a/internal/imageprocessor/imageprocessor_internal_test.go b/internal/imageprocessor/imageprocessor_internal_test.go index e9b3656..dca5645 100644 --- a/internal/imageprocessor/imageprocessor_internal_test.go +++ b/internal/imageprocessor/imageprocessor_internal_test.go @@ -94,23 +94,23 @@ func isAVIF(data []byte) bool { // detectMIME is a minimal magic-byte detector for test assertions. func detectMIME(data []byte) string { if len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { - return mimeJPEG + return imagerequest.FormatJPEG.MIMEType() } if len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n" { - return mimePNG + return imagerequest.FormatPNG.MIMEType() } if len(data) >= 4 && string(data[:4]) == "GIF8" { - return mimeGIF + return imagerequest.FormatGIF.MIMEType() } if len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP" { - return mimeWebP + return imagerequest.FormatWebP.MIMEType() } if isAVIF(data) { - return mimeAVIF + return imagerequest.FormatAVIF.MIMEType() } return "" @@ -131,7 +131,8 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) { FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -157,7 +158,7 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) { } mime := detectMIME(data) - if mime != mimeJPEG { + if mime != imagerequest.FormatJPEG.MIMEType() { t.Errorf("Output format = %v, want image/jpeg", mime) } } @@ -176,7 +177,8 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) { FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -189,7 +191,7 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) { } mime := detectMIME(data) - if mime != mimePNG { + if mime != imagerequest.FormatPNG.MIMEType() { t.Errorf("Output format = %v, want image/png", mime) } } @@ -213,7 +215,8 @@ func processAndCheckSize( FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -253,7 +256,8 @@ func TestImageProcessor_FitContain(t *testing.T) { FitMode: imagerequest.FitContain, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -296,7 +300,8 @@ func TestImageProcessor_ProcessPNG(t *testing.T) { FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatPNG, transform) if err != nil { t.Fatalf("Process() error = %v", err) } @@ -341,7 +346,8 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) { FitMode: imagerequest.FitCover, } - _, err := proc.Process(ctx, bytes.NewReader(input), transform) + _, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err == nil { t.Error("Process() should reject oversized input images") } @@ -369,7 +375,8 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf( "Process() should accept images at MaxInputDimension, got error: %v", @@ -399,7 +406,8 @@ func encodeAndCheck( FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf("Process() error = %v, want nil", err) } @@ -430,7 +438,7 @@ func encodeAndCheck( func TestImageProcessor_EncodeWebP(t *testing.T) { t.Parallel() - encodeAndCheck(t, imagerequest.FormatWebP, 80, mimeWebP) + encodeAndCheck(t, imagerequest.FormatWebP, 80, imagerequest.FormatWebP.MIMEType()) } func TestImageProcessor_DecodeAVIF(t *testing.T) { @@ -453,7 +461,8 @@ func TestImageProcessor_DecodeAVIF(t *testing.T) { FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatAVIF, transform) if err != nil { t.Fatalf("Process() error = %v, want nil (AVIF decoding should work)", err) } @@ -467,7 +476,7 @@ func TestImageProcessor_DecodeAVIF(t *testing.T) { } mime := detectMIME(data) - if mime != mimeJPEG { + if mime != imagerequest.FormatJPEG.MIMEType() { t.Errorf("Output format = %v, want image/jpeg", mime) } } @@ -494,7 +503,8 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) { FitMode: imagerequest.FitCover, } - _, err := proc.Process(ctx, bytes.NewReader(input), transform) + _, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err == nil { t.Fatal("Process() should reject input exceeding maxInputBytes") } @@ -521,7 +531,8 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) { FitMode: imagerequest.FitCover, } - result, err := proc.Process(ctx, bytes.NewReader(input), transform) + result, err := proc.Process(ctx, bytes.NewReader(input), + imagerequest.FormatJPEG, transform) if err != nil { t.Fatalf("Process() error = %v, want nil", err) } @@ -532,11 +543,11 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) { func TestImageProcessor_EncodeAVIF(t *testing.T) { t.Parallel() - encodeAndCheck(t, imagerequest.FormatAVIF, 85, mimeAVIF) + encodeAndCheck(t, imagerequest.FormatAVIF, 85, imagerequest.FormatAVIF.MIMEType()) } -// processAndDecode runs input through Process and decodes the output with -// vips, so a test can inspect the image a client would receive. +// processAndDecode runs input, a JPEG, through Process and decodes the output +// with vips, so a test can inspect the image a client would receive. func processAndDecode( t *testing.T, input []byte, transform *imagerequest.Transform, ) *vips.ImageRef { @@ -546,7 +557,8 @@ func processAndDecode( MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process( - context.Background(), bytes.NewReader(input), transform, + context.Background(), bytes.NewReader(input), imagerequest.FormatJPEG, + transform, ) if err != nil { t.Fatalf("Process() error = %v", err) diff --git a/internal/imageprocessor/jpegxl_output_internal_test.go b/internal/imageprocessor/jpegxl_output_internal_test.go index 29afb32..1f041ee 100644 --- a/internal/imageprocessor/jpegxl_output_internal_test.go +++ b/internal/imageprocessor/jpegxl_output_internal_test.go @@ -25,7 +25,8 @@ func TestImageProcessor_EncodeJPEGXL(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process( - t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), transform, + t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), + imagerequest.FormatJPEG, transform, ) if err != nil { t.Fatalf("Process() error = %v", err) @@ -71,7 +72,7 @@ func TestImageProcessor_JPEGXLQuality(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process(t.Context(), - bytes.NewReader(input), + bytes.NewReader(input), imagerequest.FormatJPEG, &imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: quality}) if err != nil { t.Fatalf("Process() at quality %d error = %v", quality, err) diff --git a/internal/imageprocessor/max_concurrent_processing_internal_test.go b/internal/imageprocessor/max_concurrent_processing_internal_test.go index fff69d0..e9ce734 100644 --- a/internal/imageprocessor/max_concurrent_processing_internal_test.go +++ b/internal/imageprocessor/max_concurrent_processing_internal_test.go @@ -93,7 +93,8 @@ func processInBackground( proc *ImageProcessor, reader *gatedReader, results chan<- error, ) { go func() { - result, err := proc.Process(context.Background(), reader, smallJPEGTransform()) + result, err := proc.Process(context.Background(), reader, + imagerequest.FormatJPEG, smallJPEGTransform()) if err == nil { _ = result.Content.Close() } @@ -206,7 +207,7 @@ func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { start := time.Now() _, err := proc.Process(context.Background(), bytes.NewReader(input), - smallJPEGTransform()) + imagerequest.FormatJPEG, smallJPEGTransform()) if !errors.Is(err, ErrTooManyImages) { t.Fatalf("Process() error = %v, want ErrTooManyImages", err) } @@ -274,7 +275,8 @@ func TestProcessReleasesSlotOnError(t *testing.T) { proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1}) proc.processingWaitTimeout = 100 * time.Millisecond - _, err := proc.Process(context.Background(), tc.input, tc.transform) + _, err := proc.Process(context.Background(), tc.input, + imagerequest.FormatJPEG, tc.transform) if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { t.Fatalf("Process() error = %v, want %v", err, tc.want) } @@ -284,7 +286,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) { } result, err := proc.Process(context.Background(), bytes.NewReader(valid), - smallJPEGTransform()) + imagerequest.FormatJPEG, smallJPEGTransform()) if err != nil { t.Fatalf("Process() after the error = %v, want nil", err) } diff --git a/internal/imageprocessor/orig_format_internal_test.go b/internal/imageprocessor/orig_format_internal_test.go index 70b97ef..142eabb 100644 --- a/internal/imageprocessor/orig_format_internal_test.go +++ b/internal/imageprocessor/orig_format_internal_test.go @@ -10,8 +10,9 @@ import ( ) // TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF -// input, which is not a source format, is refused with -// ErrUnsupportedInputFormat before it is decoded, and not saved as JPEG. +// input is refused with ErrUnsupportedInputFormat before it is decoded, and +// not saved as JPEG. TIFF is not a source format, and the test passes the +// input to Process as a JPEG. func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) { t.Parallel() @@ -31,7 +32,7 @@ func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) { MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1, }).Process( - t.Context(), bytes.NewReader(tiff), + t.Context(), bytes.NewReader(tiff), imagerequest.FormatJPEG, &imagerequest.Transform{Format: imagerequest.FormatOriginal}, ) if result != nil { diff --git a/internal/imageprocessor/source_formats_internal_test.go b/internal/imageprocessor/source_formats_internal_test.go index aaf3aec..ac0255a 100644 --- a/internal/imageprocessor/source_formats_internal_test.go +++ b/internal/imageprocessor/source_formats_internal_test.go @@ -2,6 +2,7 @@ package imageprocessor import ( "bytes" + "errors" "io" "testing" @@ -21,7 +22,7 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, } { saved, err := proc.Process(t.Context(), - bytes.NewReader(createTestPNG(t, 20, 20)), + bytes.NewReader(createTestPNG(t, 20, 20)), imagerequest.FormatPNG, &imagerequest.Transform{Format: format}) if err != nil { t.Fatalf("saving the test image as %s: %v", format, err) @@ -32,7 +33,7 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { t.Fatalf("reading the %s image: %v", format, err) } - result, err := proc.Process(t.Context(), bytes.NewReader(source), + result, err := proc.Process(t.Context(), bytes.NewReader(source), format, &imagerequest.Transform{Format: imagerequest.FormatOriginal}) if err != nil { t.Errorf("orig of the %s image: %v", format, err) @@ -42,9 +43,33 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { t.Logf("orig of the %s image: %s", format, result.ContentType) - if result.ContentType != FormatToMIME(format) { + if result.ContentType != format.MIMEType() { t.Errorf("orig of the %s image is %s, want %s", - format, result.ContentType, FormatToMIME(format)) + format, result.ContentType, format.MIMEType()) + } + } +} + +// TestImageProcessor_NotASourceFormatRefused verifies that a source format +// other than the six, such as orig, auto or none, is refused with +// ErrUnsupportedInputFormat. The input is text, which libvips reads as no +// format. +func TestImageProcessor_NotASourceFormatRefused(t *testing.T) { + t.Parallel() + + proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) + + for _, sourceFormat := range []imagerequest.Format{ + imagerequest.FormatOriginal, imagerequest.FormatAuto, "", + } { + result, err := proc.Process(t.Context(), + bytes.NewReader([]byte("this is text, not an image")), sourceFormat, + &imagerequest.Transform{Format: imagerequest.FormatJPEG}) + t.Logf("source format %q: error = %v", sourceFormat, err) + + if result != nil || !errors.Is(err, ErrUnsupportedInputFormat) { + t.Errorf("Process() with source format %q = %v, error %v; want %v", + sourceFormat, result, err, ErrUnsupportedInputFormat) } } } diff --git a/internal/imageproxy/imageproxy.go b/internal/imageproxy/imageproxy.go index 7df0381..ae17947 100644 --- a/internal/imageproxy/imageproxy.go +++ b/internal/imageproxy/imageproxy.go @@ -303,21 +303,22 @@ func withoutCancelKeepingDeadline( return context.WithDeadline(context.WithoutCancel(ctx), deadline) } -// loadCachedSource opens src's cached source, without reading it, and -// returns it with its size; nil if it is not in the cache, cannot be read, -// is empty or exceeds maxResponseSize. +// loadCachedSource opens src's cached source, reading only the first bytes, +// which give its format, and returns it with its size and its format. It +// returns nil if the source is not in the cache, cannot be read, is in no +// source format or exceeds maxResponseSize. func (p *Proxy) loadCachedSource( ctx context.Context, src imagerequest.Source, -) (io.ReadCloser, int64) { - reader, size, err := p.cache.Source(ctx, src) +) (io.ReadCloser, int64, imagerequest.Format) { + reader, size, format, err := p.cache.Source(ctx, src) if errors.Is(err, imgcache.ErrNotFound) { - return nil, 0 + return nil, 0, "" } if err != nil { p.log.Warn("failed to load cached source, fetching", "error", err) - return nil, 0 + return nil, 0, "" } p.log.Debug("using cached source", "source_key", src.Key()) @@ -330,16 +331,10 @@ func (p *Proxy) loadCachedSource( "max_bytes", p.maxResponseSize, ) - return nil, 0 + return nil, 0, "" } - if size == 0 { - _ = reader.Close() - - return nil, 0 - } - - return reader, size + return reader, size, format } // processFromSourceOrFetch processes an image, using cached source content @@ -349,7 +344,7 @@ func (p *Proxy) processFromSourceOrFetch( req *imagerequest.Request, cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { - source, sourceSize := p.loadCachedSource(ctx, req.Source) + source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source) // Fetch from upstream if we don't have source data or it's empty if source == nil { @@ -361,7 +356,7 @@ func (p *Proxy) processFromSourceOrFetch( // Process using cached source; nothing was fetched from upstream. The // image processor reads the source only once it has a processing slot, // so a request waiting for one holds none of it in memory. - return p.processAndStore(ctx, req, cacheKey, source, sourceSize) + return p.processAndStore(ctx, req, cacheKey, source, sourceFormat, sourceSize) } // fetchAndProcess fetches from upstream, processes, and caches the result. @@ -442,10 +437,11 @@ func (p *Proxy) fetchAndProcess( "cipher", fetchResult.TLSCipherSuite, ) - // Validate magic bytes match content type. A body that does not is the - // upstream's error, so it gets a negative cache entry, as an upstream - // error status does. - err = magic.ValidateMagicBytes(sourceData, fetchResult.ContentType) + // The source's format, detected from its first bytes, which must match + // its content type. A body that does not is the upstream's error, so it + // gets a negative cache entry, as an upstream error status does. + sourceFormat, err := magic.ValidateMagicBytes(sourceData, + fetchResult.ContentType) if err != nil { p.log.Warn("source refused", "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, "path", req.Source.Path, @@ -463,7 +459,7 @@ func (p *Proxy) fetchAndProcess( } return p.processAndStore( - ctx, req, cacheKey, bytes.NewReader(sourceData), fetchBytes, + ctx, req, cacheKey, bytes.NewReader(sourceData), sourceFormat, fetchBytes, ) } @@ -497,13 +493,14 @@ type processedVariant struct { fetchedBytes int64 } -// processAndStore processes the image read from source and stores the -// result. +// processAndStore processes the image read from source, which is in +// sourceFormat, and stores the result. func (p *Proxy) processAndStore( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, source io.Reader, + sourceFormat imagerequest.Format, fetchBytes int64, ) (*processedVariant, error) { // Process the image @@ -511,7 +508,8 @@ func (p *Proxy) processAndStore( // ctx has the deadline of the request doing the processing, which can // end while a request waiting for its image has time left - processResult, err := p.processor.Process(ctx, source, &req.Transform) + processResult, err := p.processor.Process(ctx, source, sourceFormat, + &req.Transform) if errors.Is(err, context.DeadlineExceeded) { return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err) } @@ -546,7 +544,7 @@ func (p *Proxy) processAndStore( "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, "path", req.Source.Path, - "src_format", processResult.InputFormat, + "src_format", sourceFormat, "dst_format", req.Transform.Format, "src_bytes", fetchBytes, "dst_bytes", outputSize, diff --git a/internal/imageproxy/imageproxy_internal_test.go b/internal/imageproxy/imageproxy_internal_test.go index 5e392ea..be2e46a 100644 --- a/internal/imageproxy/imageproxy_internal_test.go +++ b/internal/imageproxy/imageproxy_internal_test.go @@ -191,23 +191,14 @@ func TestProxy_Get_FormatConversion(t *testing.T) { t.Fatalf("failed to read response: %v", err) } - detectedMIME, err := magic.DetectFormat(data) + detectedFormat, err := magic.DetectFormat(data) if err != nil { t.Fatalf("failed to detect format: %v", err) } - expectedFormat, ok := magic.MIMEToImageFormat(tt.wantMIME) - if !ok { - t.Fatalf("unknown format for MIME type: %s", tt.wantMIME) - } - - detectedFormat, ok := magic.MIMEToImageFormat(string(detectedMIME)) - if !ok { - t.Fatalf("unknown format for detected MIME type: %s", detectedMIME) - } - - if detectedFormat != expectedFormat { - t.Errorf("detected format = %q, want %q", detectedFormat, expectedFormat) + if detectedFormat.MIMEType() != tt.wantMIME { + t.Errorf("detected format = %q, want the format of %s", + detectedFormat, tt.wantMIME) } }) } diff --git a/internal/imageproxy/max_concurrent_processing_internal_test.go b/internal/imageproxy/max_concurrent_processing_internal_test.go index 5d49d16..affa3cb 100644 --- a/internal/imageproxy/max_concurrent_processing_internal_test.go +++ b/internal/imageproxy/max_concurrent_processing_internal_test.go @@ -41,7 +41,8 @@ func holdProcessingSlot( input, feed := io.Pipe() go func() { - _, _ = proc.Process(t.Context(), input, &imagerequest.Transform{}) + _, _ = proc.Process(t.Context(), input, imagerequest.FormatJPEG, + &imagerequest.Transform{}) }() _, err := feed.Write([]byte{0}) @@ -57,10 +58,11 @@ func holdProcessingSlot( // TestProxy_Get_WaitsForSlotBeforeReadingCachedSource checks that a // request whose source is cached holds none of it while it waits for a -// processing slot: it reads the cached file only once it has a slot. With -// the only slot held, a request for a new width of the cached 100x100 photo -// waits; the cached file is then rewritten as a 100x50 image before the slot -// is freed, so the request must answer with that image scaled to 40x20. +// processing slot: past the first bytes, which give the source's format, it +// reads the cached file only once it has a slot. With the only slot held, a +// request for a new width of the cached 100x100 photo waits; the cached file +// is then rewritten as a 100x50 image before the slot is freed, so the +// request must answer with that image scaled to 40x20. func TestProxy_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { t.Parallel() @@ -79,7 +81,7 @@ func TestProxy_Get_WaitsForSlotBeforeReadingCachedSource(t *testing.T) { cachedSource := widthOnlyRequest(fixtures, 50).Source - reader, _, err := svc.cache.Source(t.Context(), cachedSource) + reader, _, _, err := svc.cache.Source(t.Context(), cachedSource) if err != nil { t.Fatalf("Source() error = %v; want the cached source", err) } diff --git a/internal/imageproxy/negative_cache_internal_test.go b/internal/imageproxy/negative_cache_internal_test.go index 23ca916..96257cc 100644 --- a/internal/imageproxy/negative_cache_internal_test.go +++ b/internal/imageproxy/negative_cache_internal_test.go @@ -55,7 +55,7 @@ func TestProxy_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { source := photoVariant(fixtures, 85, imagerequest.FitCover).Source - reader, _, err := svc.cache.Source(t.Context(), source) + reader, _, _, err := svc.cache.Source(t.Context(), source) if err != nil { t.Fatalf("Source() error = %v, want the cached source", err) } diff --git a/internal/imageproxy/partial_cache_files_internal_test.go b/internal/imageproxy/partial_cache_files_internal_test.go index f1b7c4f..13f37bc 100644 --- a/internal/imageproxy/partial_cache_files_internal_test.go +++ b/internal/imageproxy/partial_cache_files_internal_test.go @@ -7,6 +7,7 @@ import ( "testing" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/imgcache" ) // getImage calls Get for req and returns the image it answered with and @@ -49,7 +50,7 @@ func cachedSourcePath( ) string { t.Helper() - reader, _, err := svc.cache.Source(t.Context(), req.Source) + reader, _, _, err := svc.cache.Source(t.Context(), req.Source) if err != nil { t.Fatalf("Source() error = %v; want the cached source", err) } @@ -104,6 +105,40 @@ func TestProxy_Get_FetchesAgainForRemovedCachedSource(t *testing.T) { } } +// TestProxy_Get_FetchesAgainSourceOfNoSourceFormat replaces the cached +// source's file with text. The next request for a new size of the source +// must fetch it again and cache it whole. +func TestProxy_Get_FetchesAgainSourceOfNoSourceFormat(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + + getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + + path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover)) + wholeSize := fileSize(t, path) + + err := os.WriteFile(path, []byte("this is text, not an image"), + imgcache.StorageFilePerm) + if err != nil { + t.Fatalf("replacing the source's file: %v", err) + } + + getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) + + fetches := readCounters(t, svc).upstreamFetchCount + t.Logf("upstream fetches = %d", fetches) + + if fetches != 2 { + t.Errorf("upstream fetches = %d, want 2", fetches) + } + + size := fileSize(t, path) + if size != wholeSize { + t.Errorf("cached source is %d bytes, want %d", size, wholeSize) + } +} + // TestProxy_Get_ConvertsAgainForRemovedCachedVariant caches a variant of // the photo and removes its row and file, as the reconciliation pass does // when the file is empty or cut short. internal/imgcache tests that the diff --git a/internal/imagerequest/imagerequest.go b/internal/imagerequest/imagerequest.go index d9b2bac..3ae3c6c 100644 --- a/internal/imagerequest/imagerequest.go +++ b/internal/imagerequest/imagerequest.go @@ -1,7 +1,8 @@ // Package imagerequest holds the image request type, its parts, its checks, // the parsing of the /v1/image/ path, the variant key, the source key and the -// source's upstream URL. It imports no other package of this repo, so every -// package that handles an image request can import it. +// source's upstream URL. It also holds the image format type and the MIME type +// of each format. It imports no other package of this repo, so every package +// that handles an image request can import it. package imagerequest import ( @@ -12,10 +13,11 @@ import ( // ErrInvalidFitMode is returned when an invalid fit mode is provided. var ErrInvalidFitMode = errors.New("invalid fit mode") -// Format represents supported output image formats. +// Format is an image format. pixa reads and writes the formats from JPEG to +// GIF below. In a request, orig and auto stand for one of them. type Format string -// Supported image output formats. +// Supported image formats. const ( FormatOriginal Format = "orig" FormatJPEG Format = "jpeg" @@ -31,6 +33,30 @@ const ( FormatAuto Format = "auto" ) +// MIMEType returns the MIME type of the format, such as image/jpeg for +// FormatJPEG. orig and auto stand for another format, so they have none: +// MIMEType returns "" for them. +func (f Format) MIMEType() string { + switch f { + case FormatJPEG: + return "image/jpeg" + case FormatPNG: + return "image/png" + case FormatWebP: + return "image/webp" + case FormatAVIF: + return "image/avif" + case FormatJXL: + return "image/jxl" + case FormatGIF: + return "image/gif" + case FormatOriginal, FormatAuto: + return "" + default: + return "" + } +} + // Size represents requested image dimensions type Size struct { Width int diff --git a/internal/imagerequest/mimetype_internal_test.go b/internal/imagerequest/mimetype_internal_test.go new file mode 100644 index 0000000..172c5e9 --- /dev/null +++ b/internal/imagerequest/mimetype_internal_test.go @@ -0,0 +1,30 @@ +package imagerequest + +import "testing" + +// TestFormat_MIMEType checks the MIME type of each format pixa reads and +// writes, and that orig and auto have none. +func TestFormat_MIMEType(t *testing.T) { + t.Parallel() + + tests := []struct { + format Format + want string + }{ + {FormatJPEG, "image/jpeg"}, + {FormatPNG, "image/png"}, + {FormatWebP, "image/webp"}, + {FormatAVIF, "image/avif"}, + {FormatJXL, "image/jxl"}, + {FormatGIF, "image/gif"}, + {FormatOriginal, ""}, + {FormatAuto, ""}, + } + + for _, tt := range tests { + got := tt.format.MIMEType() + if got != tt.want { + t.Errorf("%s MIMEType() = %q, want %q", tt.format, got, tt.want) + } + } +} diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index d1376b2..ab8a5d2 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -175,7 +175,7 @@ func (c *Cache) Variant( return nil, 0, "", err } - contentType, err := outputContentType(file) + format, err := readFormat(file) if err != nil { _ = file.Close() @@ -184,7 +184,7 @@ func (c *Cache) Variant( c.touchVariant(cacheKey) - return file, size, contentType, nil + return file, size, format.MIMEType(), nil } // PutVariant stores a processed variant by its cache key and records @@ -227,15 +227,17 @@ func (c *Cache) PutVariant( return nil } -// Source opens the cached source of src and returns a reader and its size -// in bytes. A source whose row or file is missing is ErrNotFound, as is any -// source of a disabled cache. A source it opens has its access time handed -// to the background writer without waiting (see touchSourceContent). +// Source opens the cached source of src and returns a reader, its size in +// bytes and its format, read from the file's first bytes. A source whose row +// or file is missing is ErrNotFound, as is any source of a disabled cache; a +// file whose first bytes are not those of a source format is another error. +// A source it opens has its access time handed to the background writer +// without waiting (see touchSourceContent). func (c *Cache) Source( ctx context.Context, src imagerequest.Source, -) (io.ReadCloser, int64, error) { +) (io.ReadCloser, int64, imagerequest.Format, error) { if c.config.MaxBytes == 0 { - return nil, 0, ErrNotFound + return nil, 0, "", ErrNotFound } key := src.Key() @@ -246,21 +248,28 @@ func (c *Cache) Source( SELECT EXISTS(SELECT 1 FROM source_content WHERE source_key = ?) `, string(key)).Scan(&rowExists) if err != nil { - return nil, 0, fmt.Errorf("failed to lookup source: %w", err) + return nil, 0, "", fmt.Errorf("failed to lookup source: %w", err) } if !rowExists { - return nil, 0, ErrNotFound + return nil, 0, "", ErrNotFound } file, size, err := c.srcContent.Open(string(key)) if err != nil { - return nil, 0, err + return nil, 0, "", err + } + + format, err := readFormat(file) + if err != nil { + _ = file.Close() + + return nil, 0, "", err } c.touchSourceContent(key) - return file, size, nil + return file, size, format, nil } // PutSource stores fetched source content: it writes the file named by @@ -440,13 +449,10 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) { return deleted, nil } -// errNotAnOutputFormat is returned for a variant file whose first bytes are -// those of a format pixa reads but never writes, such as SVG. -var errNotAnOutputFormat = errors.New("variant file is not in an output format") - -// outputContentType reads the first bytes of content, without moving its -// offset, and returns the content type of the output format they show. -func outputContentType(content io.ReaderAt) (string, error) { +// readFormat reads the first bytes of a source or variant file, without +// moving its offset, and returns the format they show. Sources and variants +// are in the same six formats. +func readFormat(content io.ReaderAt) (imagerequest.Format, error) { firstBytes := make([]byte, magic.MinMagicBytes) _, err := content.ReadAt(firstBytes, 0) @@ -454,15 +460,5 @@ func outputContentType(content io.ReaderAt) (string, error) { return "", fmt.Errorf("failed to read the first bytes: %w", err) } - detected, err := magic.DetectFormat(firstBytes) - if err != nil { - return "", err - } - - _, isOutputFormat := magic.MIMEToImageFormat(string(detected)) - if !isOutputFormat { - return "", fmt.Errorf("%w: %s", errNotAnOutputFormat, detected) - } - - return string(detected), nil + return magic.DetectFormat(firstBytes) } diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index 23e2e07..828ce50 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -32,7 +32,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) { imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, } { result, err := processor.Process(t.Context(), bytes.NewReader(source), - &imagerequest.Transform{ + imagerequest.FormatJPEG, &imagerequest.Transform{ Size: imagerequest.Size{Width: 20, Height: 15}, Format: format, Quality: 85, @@ -68,7 +68,7 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) { t.Logf("%s: %d bytes served as %s", format, size, contentType) - if want := imageprocessor.FormatToMIME(format); contentType != want { + if want := format.MIMEType(); contentType != want { t.Errorf("%s variant content type = %q, want %q", format, contentType, want) } @@ -96,7 +96,7 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) { }{ {"text", []byte("this is text, not an image"), magic.ErrUnknownFormat}, {"SVG", []byte(``), - errNotAnOutputFormat}, + magic.ErrUnknownFormat}, {"JPEG cut to 10 bytes", jpegImage[:10], io.EOF}, {"empty", nil, io.EOF}, } diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 7be430e..15b2ebc 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -455,7 +455,7 @@ func assertDisabledCacheReadsAlwaysMiss( ctx := t.Context() - reader, _, err := cache.Source(ctx, req.Source) + reader, _, _, err := cache.Source(ctx, req.Source) if !errors.Is(err, ErrNotFound) { t.Fatalf("Source on disabled cache error = %v, want %v", err, ErrNotFound) } diff --git a/internal/imgcache/pendingaccesstimes_internal_test.go b/internal/imgcache/pendingaccesstimes_internal_test.go index b21ca78..d9ceb65 100644 --- a/internal/imgcache/pendingaccesstimes_internal_test.go +++ b/internal/imgcache/pendingaccesstimes_internal_test.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "errors" + "image/color" "io" "log/slog" "strings" @@ -204,8 +205,8 @@ func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) - key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg", - []byte("source bytes")) + content := generateTestJPEG(t, 10, 10, color.RGBA{0, 0, 255, 255}) + key := storeEvictionTestSource(t, cache, "example.com", "/photo.jpg", content) anHourAgo := time.Now().Add(-time.Hour) setSourceLastAccessed(t, cache, key, anHourAgo) @@ -215,7 +216,7 @@ func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { ` anHourAgoInDatabase := anHourAgo.UTC().Format(sqliteTimestampFormat) - reader, _, err := cache.Source(t.Context(), + reader, _, _, err := cache.Source(t.Context(), imagerequest.Source{Host: "example.com", Path: "/photo.jpg"}) if err != nil { t.Fatalf("Source() error = %v; want the source of %q", err, key) @@ -224,7 +225,7 @@ func TestSource_HandsAccessTimeToTheBackgroundWriter(t *testing.T) { found, err := io.ReadAll(reader) _ = reader.Close() - if err != nil || string(found) != "source bytes" { + if err != nil || !bytes.Equal(found, content) { t.Fatalf("Source() reads %q, %v; want the source of %q", found, err, key) } diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go index b9d7adb..60a9be4 100644 --- a/internal/imgcache/source_content_internal_test.go +++ b/internal/imgcache/source_content_internal_test.go @@ -2,12 +2,17 @@ package imgcache import ( "bytes" + "errors" + "fmt" + "image/color" "io" "os" "path/filepath" "testing" + "sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/magic" ) // TestPutSource_StoresIdenticalBytesOfTwoURLsTwice stores the same bytes @@ -64,7 +69,7 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { t.Parallel() cache, _ := newEvictionTestCache(t, 1<<30) - content := bytes.Repeat([]byte{0x5B}, 400) + content := generateTestJPEG(t, 20, 20, color.RGBA{0x5B, 0x5B, 0x5B, 255}) evicted := storeEvictionTestSource(t, cache, testHostCDN, "/evicted.jpg", content) storeEvictionTestSource(t, cache, testHostCDN, "/kept.jpg", content) @@ -85,7 +90,7 @@ func TestEvictSourceBlob_LeavesSourceWithIdenticalBytesServed(t *testing.T) { t.Errorf("the evicted source still has %d source_content rows", n) } - reader, _, err := cache.Source(t.Context(), + reader, _, _, err := cache.Source(t.Context(), imagerequest.Source{Host: testHostCDN, Path: "/kept.jpg"}) t.Logf("Source() of the kept source: error %v", err) @@ -135,3 +140,99 @@ func TestPutSource_FailedRowWriteLeavesNoFile(t *testing.T) { t.Error("the source's file is on disk although its row was not written") } } + +// TestSource_FormatFromFirstBytes stores an image in each source format +// under one source URL, in turn. After each store, Source must return that +// image whole, with its format read from the file. +func TestSource_FormatFromFirstBytes(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + processor := imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 1, + }) + jpegImage := generateTestJPEG(t, 40, 30, color.RGBA{200, 40, 40, 255}) + src := imagerequest.Source{Host: testHostCDN, Path: testPathCat} + + for _, format := range []imagerequest.Format{ + imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, + imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, + } { + result, err := processor.Process(t.Context(), bytes.NewReader(jpegImage), + imagerequest.FormatJPEG, &imagerequest.Transform{Format: format}) + if err != nil { + t.Fatalf("making a %s image: %v", format, err) + } + + image, err := io.ReadAll(result.Content) + if err != nil { + t.Fatalf("reading the %s image: %v", format, err) + } + + _, err = cache.PutSource(t.Context(), src, bytes.NewReader(image)) + if err != nil { + t.Fatalf("PutSource(%s) error = %v", format, err) + } + + reader, size, got, err := cache.Source(t.Context(), src) + if err != nil { + t.Fatalf("Source() of the %s image error = %v", format, err) + } + + served, err := io.ReadAll(reader) + _ = reader.Close() + + if err != nil { + t.Fatalf("reading the %s source: %v", format, err) + } + + t.Logf("%s: %d bytes read as %s", format, size, got) + + if got != format { + t.Errorf("Source() of the %s image format = %q, want %q", + format, got, format) + } + + if size != int64(len(image)) || !bytes.Equal(served, image) { + t.Errorf("%s source read as %d bytes, size %d, want the %d bytes stored", + format, len(served), size, len(image)) + } + } +} + +// TestSource_RefusesFileOfNoSourceFormat checks that Source refuses a +// cached source whose file's first bytes are not those of a source format, +// with an error other than ErrNotFound. +func TestSource_RefusesFileOfNoSourceFormat(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + jpegImage := generateTestJPEG(t, 10, 10, color.RGBA{0, 0, 255, 255}) + + tests := []struct { + name string + content []byte + wantErr error + }{ + {"text", []byte("this is text, not an image"), magic.ErrUnknownFormat}, + {"SVG", []byte(``), + magic.ErrUnknownFormat}, + {"JPEG cut to 10 bytes", jpegImage[:10], io.EOF}, + {"empty file", nil, io.EOF}, + } + + for i, tt := range tests { + path := fmt.Sprintf("/refused%d.jpg", i) + storeEvictionTestSource(t, cache, testHostCDN, path, tt.content) + + reader, _, format, err := cache.Source(t.Context(), + imagerequest.Source{Host: testHostCDN, Path: path}) + t.Logf("%s: Source() error = %v", tt.name, err) + + if reader != nil || format != "" || !errors.Is(err, tt.wantErr) { + t.Errorf("%s: Source() = reader %v, format %q, error %v; "+ + "want no reader and %v", tt.name, reader, format, err, tt.wantErr) + } + } +} diff --git a/internal/magic/jpegxl_internal_test.go b/internal/magic/jpegxl_internal_test.go index 45d9197..e7aa910 100644 --- a/internal/magic/jpegxl_internal_test.go +++ b/internal/magic/jpegxl_internal_test.go @@ -1,12 +1,17 @@ package magic -import "testing" +import ( + "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" +) // testMIMEJXL is the MIME type of JPEG XL. const testMIMEJXL = "image/jxl" // TestDetectFormatJPEGXL verifies that both JPEG XL signatures, that of a -// bare codestream and that of the container, are detected as image/jxl. +// bare codestream and that of the container, are detected as JPEG XL and +// match the MIME type image/jxl. func TestDetectFormatJPEGXL(t *testing.T) { t.Parallel() @@ -25,25 +30,15 @@ func TestDetectFormatJPEGXL(t *testing.T) { t.Parallel() got, err := DetectFormat(tt.data) - if err != nil || got != MIMETypeJXL { - t.Errorf("DetectFormat() = %q, %v, want %q", got, err, testMIMEJXL) + if err != nil || got != imagerequest.FormatJXL { + t.Errorf("DetectFormat() = %q, %v, want %q", + got, err, imagerequest.FormatJXL) } - err = ValidateMagicBytes(tt.data, testMIMEJXL) + _, err = ValidateMagicBytes(tt.data, testMIMEJXL) if err != nil { t.Errorf("ValidateMagicBytes(%q) error = %v", testMIMEJXL, err) } }) } } - -// TestJPEGXLMIMEType verifies that image/jxl maps to the format jxl. -func TestJPEGXLMIMEType(t *testing.T) { - t.Parallel() - - format, ok := MIMEToImageFormat(testMIMEJXL) - if format != FormatJXL || !ok { - t.Errorf("MIMEToImageFormat(%q) = %q, %v, want %q, true", - testMIMEJXL, format, ok, FormatJXL) - } -} diff --git a/internal/magic/magic.go b/internal/magic/magic.go index db7781a..4d069ba 100644 --- a/internal/magic/magic.go +++ b/internal/magic/magic.go @@ -6,6 +6,8 @@ import ( "bytes" "errors" "strings" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // Magic byte errors. @@ -15,34 +17,6 @@ var ( ErrNotEnoughData = errors.New("not enough data to detect format") ) -// MIMEType represents a supported MIME type for input images. -type MIMEType string - -// Supported input MIME types. -const ( - MIMETypeJPEG = MIMEType("image/jpeg") - MIMETypePNG = MIMEType("image/png") - MIMETypeWebP = MIMEType("image/webp") - MIMETypeGIF = MIMEType("image/gif") - MIMETypeAVIF = MIMEType("image/avif") - MIMETypeJXL = MIMEType("image/jxl") - MIMETypeSVG = MIMEType("image/svg+xml") -) - -// ImageFormat represents supported output image formats. -// This mirrors the type in imgcache to avoid circular imports. -type ImageFormat string - -// Supported image output formats. -const ( - FormatJPEG ImageFormat = "jpeg" - FormatPNG ImageFormat = "png" - FormatWebP ImageFormat = "webp" - FormatAVIF ImageFormat = "avif" - FormatJXL ImageFormat = "jxl" - FormatGIF ImageFormat = "gif" -) - // MinMagicBytes is the minimum number of bytes needed to detect format. const MinMagicBytes = 12 @@ -78,32 +52,32 @@ var ( avisBrand = []byte{0x61, 0x76, 0x69, 0x73} // "avis" (AVIF sequence) ) -// DetectFormat detects the image format from magic bytes. -// Returns the MIME type and nil error on success. -func DetectFormat(data []byte) (MIMEType, error) { +// DetectFormat detects the image format from magic bytes. Data in none of +// the formats pixa reads, SVG included, is ErrUnknownFormat. +func DetectFormat(data []byte) (imagerequest.Format, error) { if len(data) < MinMagicBytes { return "", ErrNotEnoughData } // Check JPEG (FFD8FF) if bytes.HasPrefix(data, magicJPEG) { - return MIMETypeJPEG, nil + return imagerequest.FormatJPEG, nil } // Check PNG (89504E47 0D0A1A0A) if bytes.HasPrefix(data, magicPNG) { - return MIMETypePNG, nil + return imagerequest.FormatPNG, nil } // Check GIF (GIF87a or GIF89a) if bytes.HasPrefix(data, magicGIF) { - return MIMETypeGIF, nil + return imagerequest.FormatGIF, nil } // Check WebP (RIFF....WEBP) if bytes.HasPrefix(data, magicWebP) && len(data) >= 12 { if bytes.Equal(data[8:12], webpIdent) { - return MIMETypeWebP, nil + return imagerequest.FormatWebP, nil } } @@ -112,66 +86,39 @@ func DetectFormat(data []byte) (MIMEType, error) { if len(data) >= 12 && bytes.Equal(data[4:8], magicFtyp) { brand := data[8:12] if bytes.Equal(brand, avifBrand) || bytes.Equal(brand, avisBrand) { - return MIMETypeAVIF, nil + return imagerequest.FormatAVIF, nil } } // Check JPEG XL (FF0A, or the 12-byte container signature) if bytes.HasPrefix(data, magicJXLCodestream) || bytes.HasPrefix(data, magicJXLContainer) { - return MIMETypeJXL, nil - } - - // Check SVG - look for XML declaration or SVG tag - if detectSVG(data) { - return MIMETypeSVG, nil + return imagerequest.FormatJXL, nil } return "", ErrUnknownFormat } -// detectSVG checks if data appears to be SVG content. -func detectSVG(data []byte) bool { - // Skip BOM if present - content := skipBOM(data) - - // Convert to string for easier pattern matching - s := strings.ToLower(string(content)) - - // Skip leading whitespace - s = strings.TrimSpace(s) - - // Check for XML declaration or SVG element - return strings.HasPrefix(s, "= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF { - return data[3:] - } - - return data -} - -// ValidateMagicBytes validates that the content matches the declared MIME type. -func ValidateMagicBytes(data []byte, declaredType string) error { +// ValidateMagicBytes detects the format of data from its magic bytes, and +// returns it when it is the format the declared MIME type names. Data in +// another format is ErrMagicByteMismatch. +func ValidateMagicBytes( + data []byte, declaredType string, +) (imagerequest.Format, error) { detected, err := DetectFormat(data) if err != nil { - return err + return "", err } // Normalize the declared type (remove parameters like charset) normalizedDeclared := normalizeMIMEType(declaredType) // Check if they match - if string(detected) != normalizedDeclared { - return ErrMagicByteMismatch + if detected.MIMEType() != normalizedDeclared { + return "", ErrMagicByteMismatch } - return nil + return detected, nil } // normalizeMIMEType extracts just the media type, removing parameters. @@ -183,27 +130,3 @@ func normalizeMIMEType(mimeType string) string { return strings.TrimSpace(strings.ToLower(mimeType)) } - -// MIMEToImageFormat converts a MIME type to an ImageFormat. -func MIMEToImageFormat(mimeType string) (ImageFormat, bool) { - normalized := normalizeMIMEType(mimeType) - switch MIMEType(normalized) { - case MIMETypeJPEG: - return FormatJPEG, true - case MIMETypePNG: - return FormatPNG, true - case MIMETypeWebP: - return FormatWebP, true - case MIMETypeGIF: - return FormatGIF, true - case MIMETypeAVIF: - return FormatAVIF, true - case MIMETypeJXL: - return FormatJXL, true - case MIMETypeSVG: - // SVG has no corresponding output format. - return "", false - default: - return "", false - } -} diff --git a/internal/magic/magic_internal_test.go b/internal/magic/magic_internal_test.go index fc081ea..5d9b25f 100644 --- a/internal/magic/magic_internal_test.go +++ b/internal/magic/magic_internal_test.go @@ -1,9 +1,10 @@ package magic import ( - "bytes" "errors" "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" ) // Shared test fixture strings. @@ -12,9 +13,6 @@ const ( testMIMEJPEG = "image/jpeg" testMIMEJPEGParams = "image/jpeg; charset=utf-8" testMIMEPNG = "image/png" - testMIMEWebP = "image/webp" - testMIMEGIF = "image/gif" - testMIMEAVIF = "image/avif" ) // pad appends zero bytes so data is comfortably above MinMagicBytes. @@ -36,37 +34,22 @@ func TestDetectFormat(t *testing.T) { avis := pad(0x00, 0x00, 0x00, 0x1C, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x73) tests := []struct { - name string - data []byte - wantMIME MIMEType - wantErr error + name string + data []byte + wantFormat imagerequest.Format + wantErr error }{ - {name: "JPEG", data: jpeg, wantMIME: MIMETypeJPEG}, - {name: "PNG", data: png, wantMIME: MIMETypePNG}, - {name: "GIF87a", data: gif87a, wantMIME: MIMETypeGIF}, - {name: "GIF89a", data: gif89a, wantMIME: MIMETypeGIF}, - {name: "WebP", data: webp, wantMIME: MIMETypeWebP}, - {name: "AVIF", data: avif, wantMIME: MIMETypeAVIF}, - {name: "AVIF sequence", data: avis, wantMIME: MIMETypeAVIF}, + {name: "JPEG", data: jpeg, wantFormat: imagerequest.FormatJPEG}, + {name: "PNG", data: png, wantFormat: imagerequest.FormatPNG}, + {name: "GIF87a", data: gif87a, wantFormat: imagerequest.FormatGIF}, + {name: "GIF89a", data: gif89a, wantFormat: imagerequest.FormatGIF}, + {name: "WebP", data: webp, wantFormat: imagerequest.FormatWebP}, + {name: "AVIF", data: avif, wantFormat: imagerequest.FormatAVIF}, + {name: "AVIF sequence", data: avis, wantFormat: imagerequest.FormatAVIF}, { - name: "SVG with XML declaration", - data: []byte(``), - wantMIME: MIMETypeSVG, - }, - { - name: "SVG without declaration", - data: []byte(``), - wantMIME: MIMETypeSVG, - }, - { - name: "SVG with whitespace", - data: []byte(` `), - wantMIME: MIMETypeSVG, - }, - { - name: "SVG with BOM", - data: append([]byte{0xEF, 0xBB, 0xBF}, []byte(``)...), - wantMIME: MIMETypeSVG, + name: "SVG", + data: []byte(``), + wantErr: ErrUnknownFormat, }, { name: "unknown format", @@ -88,8 +71,8 @@ func TestDetectFormat(t *testing.T) { return } - if got != tt.wantMIME { - t.Errorf("DetectFormat() = %v, want %v", got, tt.wantMIME) + if got != tt.wantFormat { + t.Errorf("DetectFormat() = %v, want %v", got, tt.wantFormat) } }) } @@ -105,24 +88,28 @@ func TestValidateMagicBytes(t *testing.T) { name string data []byte declaredType string + wantFormat imagerequest.Format wantErr error }{ { name: "matching JPEG", data: jpegData, declaredType: testMIMEJPEG, + wantFormat: imagerequest.FormatJPEG, wantErr: nil, }, { name: "matching JPEG with params", data: jpegData, declaredType: testMIMEJPEGParams, + wantFormat: imagerequest.FormatJPEG, wantErr: nil, }, { name: "matching PNG", data: pngData, declaredType: testMIMEPNG, + wantFormat: imagerequest.FormatPNG, wantErr: nil, }, { @@ -143,45 +130,14 @@ func TestValidateMagicBytes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := ValidateMagicBytes(tt.data, tt.declaredType) + got, err := ValidateMagicBytes(tt.data, tt.declaredType) if !errors.Is(err, tt.wantErr) { t.Errorf("ValidateMagicBytes() error = %v, wantErr %v", err, tt.wantErr) } - }) - } -} - -func TestMIMEToImageFormat(t *testing.T) { - t.Parallel() - - tests := []struct { - mimeType string - wantFormat ImageFormat - wantOk bool - }{ - {testMIMEJPEG, FormatJPEG, true}, - {testMIMEPNG, FormatPNG, true}, - {testMIMEWebP, FormatWebP, true}, - {testMIMEGIF, FormatGIF, true}, - {testMIMEAVIF, FormatAVIF, true}, - {"image/svg+xml", "", false}, // SVG doesn't convert to ImageFormat - {"image/tiff", "", false}, - {"text/plain", "", false}, - } - - for _, tt := range tests { - t.Run(tt.mimeType, func(t *testing.T) { - t.Parallel() - - got, ok := MIMEToImageFormat(tt.mimeType) - - if ok != tt.wantOk { - t.Errorf("MIMEToImageFormat(%q) ok = %v, want %v", tt.mimeType, ok, tt.wantOk) - } if got != tt.wantFormat { - t.Errorf("MIMEToImageFormat(%q) = %v, want %v", tt.mimeType, got, tt.wantFormat) + t.Errorf("ValidateMagicBytes() = %q, want %q", got, tt.wantFormat) } }) } @@ -214,109 +170,6 @@ func TestNormalizeMIMEType(t *testing.T) { } } -func TestDetectSVG(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - data string - want bool - }{ - {"xml declaration", ``, true}, - {"svg element", ``, true}, - { - "doctype", - ``, - true, - }, - {"with whitespace", ` - `, true}, - {"uppercase", ``, true}, - {"not svg", ``, false}, - {"random text", `hello world`, false}, - {testNameEmpty, ``, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := detectSVG([]byte(tt.data)) - - if got != tt.want { - t.Errorf("detectSVG(%q) = %v, want %v", tt.data, got, tt.want) - } - }) - } -} - -func TestSkipBOM(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - data []byte - want []byte - }{ - {"with BOM", []byte{0xEF, 0xBB, 0xBF, 'h', 'e', 'l', 'l', 'o'}, []byte("hello")}, - {"without BOM", []byte("hello"), []byte("hello")}, - {testNameEmpty, []byte{}, []byte{}}, - {"only BOM", []byte{0xEF, 0xBB, 0xBF}, []byte{}}, - {"partial BOM", []byte{0xEF, 0xBB}, []byte{0xEF, 0xBB}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := skipBOM(tt.data) - - if !bytes.Equal(got, tt.want) { - t.Errorf("skipBOM() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestRealWorldSVGPatterns(t *testing.T) { - t.Parallel() - - // Test various real-world SVG patterns - svgPatterns := []string{ - ` - - -`, - ` - -`, - `` + ` - -`, - } - - for i, pattern := range svgPatterns { - data := []byte(pattern) - if len(data) < MinMagicBytes { - // Pad short SVGs for detection - data = append(data, make([]byte, MinMagicBytes-len(data))...) - } - - got, err := DetectFormat(data) - if err != nil { - t.Errorf("Pattern %d: DetectFormat() error = %v", i, err) - - continue - } - - if got != MIMETypeSVG { - t.Errorf("Pattern %d: DetectFormat() = %v, want %v", i, got, MIMETypeSVG) - } - } -} - func TestDetectFormatRIFFNotWebP(t *testing.T) { t.Parallel() From 93b13d6638d0d477241f325fb6ff9faf35ea760b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 22:32:03 +0100 Subject: [PATCH 110/131] Load every setting from one table in internal/config (closes #320) Config.settings lists each setting once: its key, its environment variable and the function that loads it. Most rows load through field, whose arguments give the type, the default and the check. db_url and cache_max_bytes keep their own load functions. Loading, the unknown-key and unknown-variable checks and the names in error messages all read the table. allowlist_hosts now loads through parseHostList. isKnownConfigKey, envVarNames, strictLoader, validate, validateAllowlistHostsValue and getStringSlice are gone, with the getStringSlice tests. The key constants move to the tests, their only users. README.md lists the settings one item each instead of in a table, with shorter sentences above the list. A new test checks that this list and configs/config.example.yml name exactly the keys of the table. Model: opus-5-5 --- README.md | 123 ++- TODO.md | 24 + internal/config/config.go | 927 +++++++----------- internal/config/config_internal_test.go | 85 -- .../config/config_validation_internal_test.go | 30 +- .../documented_settings_internal_test.go | 127 +++ internal/config/env_internal_test.go | 7 +- 7 files changed, 604 insertions(+), 719 deletions(-) create mode 100644 internal/config/documented_settings_internal_test.go diff --git a/README.md b/README.md index 5e1437c..b748270 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: filesystem's free space and what the cache holds, and at least 500 MiB, so the cache can grow to three quarters or more of the space they all have left, and to all of it when that is 500 MiB or less. - - the rest are in the table under Configuration below + - the rest are in the list under Configuration below - **Memory:** pixa can use up to `max_concurrent_processing` times 4.2 GiB for the images it processes at once, plus `upstream_connections` times `upstream_max_response_size` for the source images it has fetched that wait @@ -678,54 +678,85 @@ refused with 400. Every setting can be given as an environment variable, in a YAML config file (`--config`), or both. A variable present in the environment wins over the file, even when it is empty, and the file wins over the built-in default. The one -exception is a variable named in the file's `env:` section: it is set while the -file loads, so it overrides both the environment the process was started with -and the file's own key. A variable's value is parsed as the same text in the -file would be. The three lists take comma-separated entries, with the spaces -around each trimmed; an empty variable is an empty list. A value that does not -parse or is invalid aborts startup, naming the variable. A variable whose name -starts with `PIXA_` but is not in the table below, such as a misspelled one or -`PIXA_PORT`, aborts startup naming it, as an unknown config key does. The one -other accepted name is `PIXA_CONFIG_PATH`, the config file's path (like -`--config`). The variables set by the file's `env:` section are checked the same +exception is a variable set in the file's `env:` section. pixa sets that +variable while loading the file, so the variable wins over the environment pixa +started with. It also wins over the file's key for the same setting. A +variable's value is parsed as the same text in the file would be. A setting that +holds a list takes comma-separated entries, with the spaces around each trimmed. +An empty variable is an empty list. A value that does not parse or is invalid +aborts startup, naming the variable. pixa refuses to start when a variable +starts with `PIXA_` but is not in the list below, such as a misspelled one or +`PIXA_PORT`. The error names the variable, as it names an unknown config key. +pixa also accepts `PIXA_CONFIG_PATH`, which names the config file, like +`--config`. The variables set by the file's `env:` section are checked the same way. -pixa reads at most one config file: the one given with `--config` (or `-c`), -otherwise the one `PIXA_CONFIG_PATH` names, otherwise the first of these that -pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`, -`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml` and -`config.yaml` in the working directory. A named file that does not exist, cannot -be read or does not parse aborts startup. Of the files pixa looks for on its -own, only one that does not exist is passed over, without a message. One that -pixa cannot read or parse aborts startup, naming the file. So does one in a -directory pixa may not enter, whether or not it is there, since pixa cannot -tell. With no file, pixa uses the environment and the defaults. +pixa reads at most one config file. It reads the file given with `--config` (or +`-c`), or else the one `PIXA_CONFIG_PATH` names. pixa refuses to start when that +file does not exist, cannot be read or does not parse. Without `--config` or +`PIXA_CONFIG_PATH`, pixa looks in `/etc/pixa/`, then in `~/.config/pixa/`, then +in the working directory. In each directory it looks for `config.yml`, then +`config.yaml`. It reads the first file it finds. It skips a missing file without +a message. pixa refuses to start when a file it finds cannot be read or does not +parse, and the error names the file. It also refuses to start when it may not +enter one of these directories, whether or not the file is there, since it +cannot tell. With no file, pixa uses the environment and the defaults. -| Variable | Config key | Meaning | -| ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- | -| `PIXA_SIGNING_KEY` | `signing_key` | Required: secret for signed and encrypted URLs and login, 32+ characters | -| `PORT` | `port` | Port to listen on; default `8080` | -| `PIXA_STATE_DIR` | `state_dir` | Directory for the database and the disk cache; default `/var/lib/pixa` | -| `PIXA_DB_URL` | `db_url` | SQLite database URL; default `state.sqlite3` in the state directory | -| `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of (free + cached) | -| `PIXA_ALLOWLIST_HOSTS` | `allowlist_hosts` | Upstream hosts served without a signature | -| `PIXA_REFERER_BLOCKLIST` | `referer_blocklist` | Hosts whose pages the image routes refuse with 403, by `Referer` | -| `PIXA_BLOCKED_NETWORKS` | `blocked_networks` | CIDR ranges never fetched from, on top of the built-in ones | -| `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 | -| `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` | -| `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` | -| `PIXA_UPSTREAM_CONNECTIONS` | `upstream_connections` | Concurrent connections to all upstream hosts together; default `64` | -| `PIXA_MAX_CONCURRENT_PROCESSING` | `max_concurrent_processing` | Images processed at once; default half the CPUs, at least 1 | -| `PIXA_IMAGE_REQUESTS_PER_MINUTE` | `image_requests_per_minute` | Image requests per client per minute; `0` turns it off; default `600` | -| `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` | -| `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB | -| `PIXA_DOWNSTREAM_TIMEOUT` | `downstream_timeout` | Time allowed for answering one client request; default `60s` | -| `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN` | `access_control_allow_origin` | CORS origin allowed to read image responses: `*` or one origin; default `*` | -| `PIXA_METRICS_USERNAME` | `metrics.username` | Username for `/metrics`, which is served only when both are set | -| `PIXA_METRICS_PASSWORD` | `metrics.password` | Password for `/metrics`; set together with the username | -| `PIXA_SENTRY_DSN` | `sentry_dsn` | Sentry DSN for error reporting; empty disables it | -| `PIXA_DEBUG` | `debug` | Debug logging and plain-HTTP local development; default `false` | -| `PIXA_MAINTENANCE_MODE` | `maintenance_mode` | Answer image requests with 503; the health check stays 200; default `false` | +Each item below is one setting, named by its config key: + +- `signing_key`: variable `PIXA_SIGNING_KEY`, required. The secret for signed + URLs, encrypted URLs and login. It must be 32 characters or longer. +- `port`: variable `PORT`, default `8080`. The port to listen on. +- `state_dir`: variable `PIXA_STATE_DIR`, default `/var/lib/pixa`. The directory + for the database and the disk cache. +- `db_url`: variable `PIXA_DB_URL`, default `state.sqlite3` in the state + directory. The URL of the SQLite database. +- `cache_max_bytes`: variable `PIXA_CACHE_MAX_BYTES`, default 75% of the sum of + the free space and what the cache holds. The disk cache limit in bytes. `0` + turns the disk cache off. +- `allowlist_hosts`: variable `PIXA_ALLOWLIST_HOSTS`, default none. The upstream + hosts whose images pixa serves without a signature. +- `referer_blocklist`: variable `PIXA_REFERER_BLOCKLIST`, default none. The + image routes refuse a request with 403 when its `Referer` header names one of + these hosts. +- `blocked_networks`: variable `PIXA_BLOCKED_NETWORKS`, default none. The CIDR + ranges pixa never fetches from, on top of the built-in ones. +- `trusted_proxies`: variable `PIXA_TRUSTED_PROXIES`, default the RFC 1918 + ranges. The CIDR ranges of the proxies whose `X-Forwarded-For` header pixa + believes. +- `allow_http`: variable `PIXA_ALLOW_HTTP`, default `false`. Allows plain-HTTP + upstream hosts. It is for testing only. +- `upstream_connections_per_host`: variable + `PIXA_UPSTREAM_CONNECTIONS_PER_HOST`, default `20`. The most connections to + one upstream host at once. +- `upstream_connections`: variable `PIXA_UPSTREAM_CONNECTIONS`, default `64`. + The most connections to all upstream hosts together at once. +- `max_concurrent_processing`: variable `PIXA_MAX_CONCURRENT_PROCESSING`, + default half the CPUs, at least 1. The most images processed at once. +- `image_requests_per_minute`: variable `PIXA_IMAGE_REQUESTS_PER_MINUTE`, + default `600`. How many image requests one client may send per minute. `0` + turns the limit off. +- `upstream_fetch_timeout`: variable `PIXA_UPSTREAM_FETCH_TIMEOUT`, default + `30s`. The time allowed for one fetch from an upstream host. +- `upstream_max_response_size`: variable `PIXA_UPSTREAM_MAX_RESPONSE_SIZE`, + default 50 MiB. The largest upstream response accepted, in bytes. +- `downstream_timeout`: variable `PIXA_DOWNSTREAM_TIMEOUT`, default `60s`. The + time allowed for answering one client request. +- `access_control_allow_origin`: variable `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN`, + default `*`. The CORS origin allowed to read image responses. It is `*` or one + origin. +- `metrics.username`: variable `PIXA_METRICS_USERNAME`, default empty. The + username for `/metrics`. pixa serves `/metrics` only when the username and the + password are both set. +- `metrics.password`: variable `PIXA_METRICS_PASSWORD`, default empty. The + password for `/metrics`. Set it together with the username. +- `sentry_dsn`: variable `PIXA_SENTRY_DSN`, default empty. The Sentry DSN for + error reporting. An empty DSN turns error reporting off. +- `debug`: variable `PIXA_DEBUG`, default `false`. Turns on debug logging and + plain-HTTP local development. +- `maintenance_mode`: variable `PIXA_MAINTENANCE_MODE`, default `false`. While + `true`, pixa answers image requests with 503. The health check still + answers 200. Key settings in more detail: diff --git a/TODO.md b/TODO.md index 39e0f30..f0cba91 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,30 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 one table in `internal/config` lists every setting once (closes + #320). `Config.settings` gives each setting's key, its environment variable + and the function that loads it. + - Most rows load through `field`, whose arguments give the setting's type, + its default and its check. + - `db_url` and `cache_max_bytes` have their own load functions. The default + of `db_url` comes from `state_dir`, and `cache_max_bytes` records whether + it is set. + - Loading, the unknown-key check, the unknown-variable check and the names + in error messages all read the table. + - `allowlist_hosts` loads as `referer_blocklist` does, through + `parseHostList`. + - The metrics username and password are checked together once the table has + loaded. + - When a config holds more than one invalid value, the error names the one + that comes first in the table. + - `README.md` lists the settings one item each, where it had a table. A test + checks that this list and `configs/config.example.yml` name exactly the + keys of the table. + - The key constants moved to the tests, their only users. + - Removed: + - `isKnownConfigKey`, `envVarNames`, `strictLoader` and `validate` + - `validateAllowlistHostsValue` and `getStringSlice`, with the three + tests of `getStringSlice` - 2026-10-26 a fetched image's format is detected from its bytes once, as an `imagerequest.Format` (closes #319). The upstream `Content-Type` check in the fetcher stays as the filter before downloading. diff --git a/internal/config/config.go b/internal/config/config.go index 3f484b7..2f433c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -76,34 +76,6 @@ func DefaultCacheMaxBytes(freeBytes uint64, heldBytes int64) int64 { return max(int64(computed), DefaultCacheMaxBytesFloor) } -// Configuration key names. -const ( - keyDebug = "debug" - keyMaintenanceMode = "maintenance_mode" - keyPort = "port" - keyStateDir = "state_dir" - keySentryDSN = "sentry_dsn" - keyDBURL = "db_url" - keyMetrics = "metrics" - keyMetricsUsername = "metrics.username" - keyMetricsPassword = "metrics.password" - keySigningKey = "signing_key" - keyAllowlistHosts = "allowlist_hosts" - keyRefererBlocklist = "referer_blocklist" - keyAllowHTTP = "allow_http" - keyUpstreamConnectionsPerHost = "upstream_connections_per_host" - keyUpstreamConnections = "upstream_connections" - keyMaxConcurrentProcessing = "max_concurrent_processing" - keyCacheMaxBytes = "cache_max_bytes" - keyBlockedNetworks = "blocked_networks" - keyTrustedProxies = "trusted_proxies" - keyAccessControlAllowOrigin = "access_control_allow_origin" - keyUpstreamFetchTimeout = "upstream_fetch_timeout" - keyUpstreamMaxResponseSize = "upstream_max_response_size" - keyDownstreamTimeout = "downstream_timeout" - keyImageRequestsPerMinute = "image_requests_per_minute" -) - // placeholderSigningKey is the dummy signing_key shipped in // configs/config.example.yml. It is 45 characters, so it passes the length // check, but it is public in this repository and must be rejected at @@ -123,7 +95,7 @@ var ( errNotAStringList = errors.New("not a list of strings") errNotAValidCIDR = errors.New("not a valid CIDR network") errMappedTooShort = errors.New("in the IPv4-mapped form but shorter than /96") - errNotAMetricsMap = errors.New("not a map of metrics settings") + errNotAMap = errors.New("not a map") errEmptyListEntry = errors.New("list contains an empty entry") errEmptyEntry = errors.New("contains an empty entry") errNotAValidURL = errors.New("not a valid URL") @@ -284,7 +256,7 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) { // newFromSmartConfig constructs a Config from the environment and a // loaded smartconfig instance, and validates it. A nil sc means no -// config file was found, in which case every option the environment +// config file was found, in which case every setting the environment // does not set takes its default value. A key that is present but // unparseable or invalid is an error: defaults apply only to omitted // keys, never to invalid explicit values. @@ -294,93 +266,171 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) { return nil, err } - err = validateAllowlistHostsValue(sc) - if err != nil { - return nil, err - } + c := &Config{} - blockedNetworks, err := parseCIDRList(sc, keyBlockedNetworks) - if err != nil { - return nil, err - } - - trustedProxies, err := parseCIDRList(sc, keyTrustedProxies) - if err != nil { - return nil, err - } - - // parseCIDRList returns a nil slice only when the key is absent; an - // explicitly empty list ([]) comes back non-nil and empty. An omitted - // key takes the RFC 1918 default, while an explicit empty list is left - // as trust-nothing. - if trustedProxies == nil { - trustedProxies = defaultTrustedProxies() - } - - loader := &strictLoader{sc: sc} - c := &Config{ - Debug: loader.boolVal(keyDebug, false), - MaintenanceMode: loader.boolVal(keyMaintenanceMode, false), - Port: loader.intVal(keyPort, DefaultPort), - StateDir: loader.stringVal(keyStateDir, DefaultStateDir), - SentryDSN: loader.stringVal(keySentryDSN, ""), - MetricsUsername: loader.stringVal(keyMetricsUsername, ""), - MetricsPassword: loader.stringVal(keyMetricsPassword, ""), - SigningKey: loader.stringVal(keySigningKey, ""), - AllowlistHosts: getStringSlice(sc), - AllowHTTP: loader.boolVal(keyAllowHTTP, false), - UpstreamConnectionsPerHost: loader.intVal( - keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost), - UpstreamConnections: loader.intVal( - keyUpstreamConnections, DefaultUpstreamConnections), - MaxConcurrentProcessing: loader.intVal( - keyMaxConcurrentProcessing, DefaultMaxConcurrentProcessing()), - UpstreamFetchTimeout: loader.durationVal( - keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout), - UpstreamMaxResponseSize: loader.int64Val( - keyUpstreamMaxResponseSize, DefaultUpstreamMaxResponseSize), - AccessControlAllowOrigin: loader.stringVal( - keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin), - DownstreamTimeout: loader.durationVal( - keyDownstreamTimeout, DefaultDownstreamTimeout), - ImageRequestsPerMinute: loader.intVal( - keyImageRequestsPerMinute, DefaultImageRequestsPerMinute), - CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0), - BlockedNetworks: blockedNetworks, - TrustedProxies: trustedProxies, - RefererBlocklist: loader.hostListVal(keyRefererBlocklist), - } - - // The default for an omitted cache_max_bytes is worked out when - // pixad builds the cache; here we only record whether the operator - // set the key explicitly. - _, c.CacheMaxBytesExplicit = lookupValue(sc, keyCacheMaxBytes) - - // Build DBURL from StateDir if not explicitly set. The derived URL - // is a default: it applies only when db_url is omitted, never to an - // explicitly empty value. - c.DBURL = loader.stringVal(keyDBURL, "") - if c.DBURL == "" && loader.err == nil { - if _, present := lookupValue(sc, keyDBURL); present { - return nil, fmt.Errorf("%s: %w; omit it to derive it from state_dir", - settingName(keyDBURL), errValueEmpty) + for _, s := range c.settings() { + err := s.load(lookupValue(sc, s)) + if err != nil { + return nil, err } - - c.DBURL = defaultDBURL(c.StateDir) } - if loader.err != nil { - return nil, loader.err - } - - err = c.validate() - if err != nil { - return nil, err + // The metrics username and password are checked together, once both + // have loaded. + if (c.MetricsUsername == "") != (c.MetricsPassword == "") { + return nil, fmt.Errorf("%s and %s %w", + settingName("metrics.username"), settingName("metrics.password"), + errMustBeSetTogether) } return c, nil } +// A setting is one row of the settings table. key is its key in the +// config file, and envVar is the environment variable that also sets it. +// load reads the value set for it into its Config field. +type setting struct { + key string + envVar string + load func(v value) error +} + +// A value is what the environment or the config file sets for one +// setting. present is false when neither sets it. +type value struct { + key string + raw any + present bool +} + +// settings returns the settings table: every setting pixa reads, once +// each, in the order they load into c. Most rows fill their Config field +// with field, whose arguments give the setting's type, its default and +// its check. Code that needs only the keys and the variables reads the +// table of a new Config. +func (c *Config) settings() []setting { + return []setting{ + {"signing_key", "PIXA_SIGNING_KEY", + field(&c.SigningKey, getString, "", validateSigningKey)}, + {"port", "PORT", field(&c.Port, getInt, DefaultPort, validatePort)}, + {"state_dir", "PIXA_STATE_DIR", + field(&c.StateDir, getString, DefaultStateDir, validateNotEmpty)}, + {"db_url", "PIXA_DB_URL", c.loadDBURL}, + {"cache_max_bytes", "PIXA_CACHE_MAX_BYTES", c.loadCacheMaxBytes}, + {"allowlist_hosts", "PIXA_ALLOWLIST_HOSTS", + field(&c.AllowlistHosts, parseHostList, nil, nil)}, + {"referer_blocklist", "PIXA_REFERER_BLOCKLIST", + field(&c.RefererBlocklist, parseHostList, nil, nil)}, + {"blocked_networks", "PIXA_BLOCKED_NETWORKS", + field(&c.BlockedNetworks, parseCIDRList, nil, nil)}, + {"trusted_proxies", "PIXA_TRUSTED_PROXIES", + field(&c.TrustedProxies, parseCIDRList, defaultTrustedProxies(), nil)}, + {"allow_http", "PIXA_ALLOW_HTTP", field(&c.AllowHTTP, getBool, false, nil)}, + {"upstream_connections_per_host", "PIXA_UPSTREAM_CONNECTIONS_PER_HOST", + field(&c.UpstreamConnectionsPerHost, getInt, + DefaultUpstreamConnectionsPerHost, validateAtLeastOne)}, + {"upstream_connections", "PIXA_UPSTREAM_CONNECTIONS", + field(&c.UpstreamConnections, getInt, + DefaultUpstreamConnections, validateAtLeastOne)}, + {"max_concurrent_processing", "PIXA_MAX_CONCURRENT_PROCESSING", + field(&c.MaxConcurrentProcessing, getInt, + DefaultMaxConcurrentProcessing(), validateAtLeastOne)}, + {"image_requests_per_minute", "PIXA_IMAGE_REQUESTS_PER_MINUTE", + field(&c.ImageRequestsPerMinute, getInt, + DefaultImageRequestsPerMinute, validateNotNegative)}, + {"upstream_fetch_timeout", "PIXA_UPSTREAM_FETCH_TIMEOUT", + field(&c.UpstreamFetchTimeout, getDuration, + DefaultUpstreamFetchTimeout, nil)}, + {"upstream_max_response_size", "PIXA_UPSTREAM_MAX_RESPONSE_SIZE", + field(&c.UpstreamMaxResponseSize, getInt64, + DefaultUpstreamMaxResponseSize, validateUpstreamMaxResponseSize)}, + {"downstream_timeout", "PIXA_DOWNSTREAM_TIMEOUT", + field(&c.DownstreamTimeout, getDuration, + DefaultDownstreamTimeout, nil)}, + {"access_control_allow_origin", "PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", + field(&c.AccessControlAllowOrigin, getString, + DefaultAccessControlAllowOrigin, validateAccessControlAllowOrigin)}, + {"metrics.username", "PIXA_METRICS_USERNAME", + field(&c.MetricsUsername, getString, "", nil)}, + {"metrics.password", "PIXA_METRICS_PASSWORD", + field(&c.MetricsPassword, getString, "", nil)}, + {"sentry_dsn", "PIXA_SENTRY_DSN", + field(&c.SentryDSN, getString, "", validateSentryDSN)}, + {"debug", "PIXA_DEBUG", field(&c.Debug, getBool, false, nil)}, + {"maintenance_mode", "PIXA_MAINTENANCE_MODE", + field(&c.MaintenanceMode, getBool, false, nil)}, + } +} + +// field returns the load function of a setting that fills *dst. read +// reads a value that is set, and so gives the setting's type. A setting +// that is not set takes defaultValue. The value, a default too, must +// then pass check, unless check is nil. +func field[T any]( + dst *T, + read func(v value) (T, error), + defaultValue T, + check func(key string, val T) error, +) func(v value) error { + return func(v value) error { + val := defaultValue + + if v.present { + var err error + + val, err = read(v) + if err != nil { + return err + } + } + + if check != nil { + err := check(v.key, val) + if err != nil { + return err + } + } + + *dst = val + + return nil + } +} + +// loadDBURL loads db_url. An omitted db_url is derived from state_dir, +// so db_url comes after state_dir in the table. An empty db_url is +// refused, since defaults apply only to omitted keys. +func (c *Config) loadDBURL(v value) error { + if !v.present { + c.DBURL = defaultDBURL(c.StateDir) + + return nil + } + + dbURL, err := getString(v) + if err != nil { + return err + } + + if dbURL == "" { + return fmt.Errorf("%s: %w; omit it to derive it from state_dir", + settingName(v.key), errValueEmpty) + } + + c.DBURL = dbURL + + return nil +} + +// loadCacheMaxBytes loads cache_max_bytes, and records whether it is set. +// For an omitted cache_max_bytes pixad works out DefaultCacheMaxBytes +// when it builds the cache. +func (c *Config) loadCacheMaxBytes(v value) error { + c.CacheMaxBytesExplicit = v.present + + return field(&c.CacheMaxBytes, getInt64, 0, validateNotNegative)(v) +} + // defaultDBURL returns the db_url used when db_url is omitted: the file // state.sqlite3 in stateDir, in WAL mode, which the driver sets only // through a _pragma parameter. String() percent-encodes any %, ? or # in @@ -409,50 +459,65 @@ func defaultDBURL(stateDir string) string { // application does not understand, so typos fail at startup instead of // being silently ignored, and rejects keys that are explicitly set to // null: a null is a SET value, never an omission, so it must not -// silently take the default. The env section is permitted because -// smartconfig consumes it for environment variable injection. A nil sc -// means no config file, which has no keys to check. +// silently take the default. A setting whose key has a dot, such as +// metrics.username, is written inside the map of the key's first part. +// The env section is permitted because smartconfig consumes it for +// environment variable injection. A nil sc means no config file, which +// has no keys to check. func validateKnownKeys(sc *smartconfig.Config) error { if sc == nil { return nil } + topLevelKeys := map[string]bool{"env": true} + mapKeys := map[string]bool{} + keysInMaps := map[string]bool{} + + for _, s := range new(Config).settings() { + mapKey, _, inMap := strings.Cut(s.key, ".") + topLevelKeys[mapKey] = true + + if inMap { + mapKeys[mapKey] = true + keysInMaps[s.key] = true + } + } + var unknown, nullKeys []string - for key, value := range sc.Data() { - if !isKnownConfigKey(key) { + for key, val := range sc.Data() { + switch { + case !topLevelKeys[key]: unknown = append(unknown, key) - - continue - } - - if value == nil { + case val == nil: nullKeys = append(nullKeys, key) - - continue - } - - if key == keyMetrics { - metricsMap, ok := value.(map[string]any) + case mapKeys[key]: + settingsMap, ok := val.(map[string]any) if !ok { - return fmt.Errorf("config key %q: value %v is %w", - keyMetrics, value, errNotAMetricsMap) + return fmt.Errorf("config key %q: value %v is %w of %s settings", + key, val, errNotAMap, key) } - for subkey, subvalue := range metricsMap { - if subkey != "username" && subkey != "password" { - unknown = append(unknown, keyMetrics+"."+subkey) + for subkey, subvalue := range settingsMap { + fullKey := key + "." + subkey - continue - } - - if subvalue == nil { - nullKeys = append(nullKeys, keyMetrics+"."+subkey) + switch { + case !keysInMaps[fullKey]: + unknown = append(unknown, fullKey) + case subvalue == nil: + nullKeys = append(nullKeys, fullKey) } } } } + return unknownOrNullKeysError(unknown, nullKeys) +} + +// unknownOrNullKeysError reports the unknown keys validateKnownKeys +// found, or else the keys set to null. It returns nil when there are +// neither. +func unknownOrNullKeysError(unknown, nullKeys []string) error { if len(unknown) > 0 { sort.Strings(unknown) @@ -480,55 +545,6 @@ func errNullConfigValue(key string) error { return fmt.Errorf("config key %q: %w", key, errValueNull) } -// isKnownConfigKey reports whether key is a permitted top-level -// configuration key. -func isKnownConfigKey(key string) bool { - switch key { - case keyDebug, keyMaintenanceMode, keyPort, keyStateDir, keySentryDSN, - keyDBURL, keyMetrics, keySigningKey, keyAllowlistHosts, keyAllowHTTP, - keyUpstreamConnectionsPerHost, keyUpstreamConnections, - keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks, - keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout, - keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist, - keyImageRequestsPerMinute, "env": - return true - } - - return false -} - -// envVarNames returns, for each configuration key, the environment -// variable that also sets it: PIXA_ plus the key in upper case, with "." -// written as "_", except the port, which REPO_POLICIES.md requires to be -// PORT. metrics is set through its two subkeys; env has no variable. -func envVarNames() map[string]string { - return map[string]string{ //nolint:gosec // G101: variable names, not secrets - keyDebug: "PIXA_DEBUG", - keyMaintenanceMode: "PIXA_MAINTENANCE_MODE", - keyPort: "PORT", - keyStateDir: "PIXA_STATE_DIR", - keySentryDSN: "PIXA_SENTRY_DSN", - keyDBURL: "PIXA_DB_URL", - keyMetricsUsername: "PIXA_METRICS_USERNAME", - keyMetricsPassword: "PIXA_METRICS_PASSWORD", - keySigningKey: "PIXA_SIGNING_KEY", - keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS", - keyRefererBlocklist: "PIXA_REFERER_BLOCKLIST", - keyAllowHTTP: "PIXA_ALLOW_HTTP", - keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST", - keyUpstreamConnections: "PIXA_UPSTREAM_CONNECTIONS", - keyMaxConcurrentProcessing: "PIXA_MAX_CONCURRENT_PROCESSING", - keyCacheMaxBytes: "PIXA_CACHE_MAX_BYTES", - keyBlockedNetworks: "PIXA_BLOCKED_NETWORKS", - keyTrustedProxies: "PIXA_TRUSTED_PROXIES", - keyAccessControlAllowOrigin: "PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", - keyUpstreamFetchTimeout: "PIXA_UPSTREAM_FETCH_TIMEOUT", - keyUpstreamMaxResponseSize: "PIXA_UPSTREAM_MAX_RESPONSE_SIZE", - keyDownstreamTimeout: "PIXA_DOWNSTREAM_TIMEOUT", - keyImageRequestsPerMinute: "PIXA_IMAGE_REQUESTS_PER_MINUTE", - } -} - // validateKnownEnvVars rejects environment variables whose names start // with PIXA_ but that are neither a setting's variable nor // PIXA_CONFIG_PATH, so a misspelled variable fails at startup instead of @@ -538,8 +554,8 @@ func envVarNames() map[string]string { func validateKnownEnvVars() error { known := map[string]bool{"PIXA_CONFIG_PATH": true} - for _, name := range envVarNames() { - known[name] = true + for _, s := range new(Config).settings() { + known[s.envVar] = true } var unknown []string @@ -566,28 +582,37 @@ func validateKnownEnvVars() error { return nil } -// lookupValue returns the value set for key and whether one is set. The -// key's environment variable wins when it is present, even when empty; -// its value is a string, read exactly as the same text quoted in the -// config file would be. Otherwise the config file's value is used. -func lookupValue(sc *smartconfig.Config, key string) (any, bool) { - value, present := os.LookupEnv(envVarNames()[key]) +// lookupValue returns the value set for s. Its environment variable wins +// when it is present, even when empty; its value is a string, read +// exactly as the same text quoted in the config file would be. Otherwise +// the config file's value is used. +func lookupValue(sc *smartconfig.Config, s setting) value { + envValue, present := os.LookupEnv(s.envVar) if present { - return value, true + return value{key: s.key, raw: envValue, present: true} } if sc == nil { - return nil, false + return value{key: s.key} } - return sc.Get(key) + fileValue, present := sc.Get(s.key) + + return value{key: s.key, raw: fileValue, present: present} } -// settingName names key in an error message together with its -// environment variable, since either one may have set the value. +// settingName names the setting key in an error message together with +// its environment variable, since either one may have set the value. func settingName(key string) string { - return fmt.Sprintf("config key %q (environment variable %s)", - key, envVarNames()[key]) + var envVar string + + for _, s := range new(Config).settings() { + if s.key == key { + envVar = s.envVar + } + } + + return fmt.Sprintf("config key %q (environment variable %s)", key, envVar) } // ensureStateDirWritable verifies at startup that StateDir can be @@ -596,16 +621,18 @@ func settingName(key string) string { func (c *Config) ensureStateDirWritable() error { const stateDirPerms = 0o750 + name := settingName("state_dir") + err := os.MkdirAll(c.StateDir, stateDirPerms) if err != nil { return fmt.Errorf("%s: cannot create directory %q: %w", - settingName(keyStateDir), c.StateDir, err) + name, c.StateDir, err) } probe, err := os.CreateTemp(c.StateDir, ".startup-write-probe-*") if err != nil { return fmt.Errorf("%s: directory %q is not writable: %w", - settingName(keyStateDir), c.StateDir, err) + name, c.StateDir, err) } probePath := probe.Name() @@ -613,13 +640,13 @@ func (c *Config) ensureStateDirWritable() error { err = probe.Close() if err != nil { return fmt.Errorf("%s: cannot close probe file %q: %w", - settingName(keyStateDir), probePath, err) + name, probePath, err) } err = os.Remove(probePath) if err != nil { return fmt.Errorf("%s: cannot remove probe file %q: %w", - settingName(keyStateDir), probePath, err) + name, probePath, err) } return nil @@ -628,101 +655,79 @@ func (c *Config) ensureStateDirWritable() error { // validateSigningKey checks that the signing key is present, long // enough, and not the public placeholder from configs/config.example.yml. // The key value itself is never echoed in error messages. -func (c *Config) validateSigningKey() error { - if c.SigningKey == "" { - return fmt.Errorf("%s: %w", settingName(keySigningKey), errValueRequired) +func validateSigningKey(key, signingKey string) error { + if signingKey == "" { + return fmt.Errorf("%s: %w", settingName(key), errValueRequired) } // Minimum key length for security (32 bytes = 256 bits) const minKeyLength = 32 - if len(c.SigningKey) < minKeyLength { + if len(signingKey) < minKeyLength { return fmt.Errorf("%s: %w: must be at least %d characters, got %d", - settingName(keySigningKey), errValueTooShort, minKeyLength, - len(c.SigningKey)) + settingName(key), errValueTooShort, minKeyLength, len(signingKey)) } - if c.SigningKey == placeholderSigningKey { - return fmt.Errorf("%s: %w", settingName(keySigningKey), errPlaceholderKey) + if signingKey == placeholderSigningKey { + return fmt.Errorf("%s: %w", settingName(key), errPlaceholderKey) } return nil } -// validate checks that all required configuration values are set and -// that every value is within its valid range. -func (c *Config) validate() error { - err := c.validateSigningKey() - if err != nil { - return err - } - +// validatePort checks that a port is from 1 to 65535. +func validatePort(key string, port int) error { const maxPort = 65535 - if c.Port < 1 || c.Port > maxPort { + if port < 1 || port > maxPort { return fmt.Errorf("%s: value %d is %w 1-%d", - settingName(keyPort), c.Port, errPortOutOfRange, maxPort) + settingName(key), port, errPortOutOfRange, maxPort) } - err = c.validateConcurrencyLimits() - if err != nil { - return err + return nil +} + +// validateNotEmpty checks that a value is not empty. +func validateNotEmpty(key, val string) error { + if val == "" { + return fmt.Errorf("%s: %w", settingName(key), errValueEmpty) } - if c.StateDir == "" { - return fmt.Errorf("%s: %w", settingName(keyStateDir), errValueEmpty) - } + return nil +} - // Zero is valid (it disables the disk cache); only negative - // values are rejected. No floor applies to explicit values. - if c.CacheMaxBytes < 0 { +// validateAtLeastOne checks that a limit, such as the most images +// processed at once, is at least 1. +func validateAtLeastOne(key string, limit int) error { + if limit < 1 { return fmt.Errorf("%s: value %d %w", - settingName(keyCacheMaxBytes), c.CacheMaxBytes, errMustNotBeNegative) + settingName(key), limit, errMustBeAtLeastOne) } - // Zero is valid (it turns the limit off); only negative values are - // rejected. - if c.ImageRequestsPerMinute < 0 { + return nil +} + +// validateNotNegative checks that a number is not negative. Zero is valid +// for the two settings that use it: it turns the disk cache or the limit +// on image requests off. +func validateNotNegative[T int | int64](key string, number T) error { + if number < 0 { return fmt.Errorf("%s: value %d %w", - settingName(keyImageRequestsPerMinute), c.ImageRequestsPerMinute, - errMustNotBeNegative) + settingName(key), number, errMustNotBeNegative) } - err = c.validateUpstreamMaxResponseSize() - if err != nil { - return err - } - - for _, host := range c.AllowlistHosts { - err := validateHostPattern(keyAllowlistHosts, host) - if err != nil { - return err - } - } - - err = c.validateSentryDSN() - if err != nil { - return err - } - - if (c.MetricsUsername == "") != (c.MetricsPassword == "") { - return fmt.Errorf("%s and %s %w", - settingName(keyMetricsUsername), settingName(keyMetricsPassword), - errMustBeSetTogether) - } - - return c.validateAccessControlAllowOrigin() + return nil } // validateSentryDSN checks that sentry_dsn, when set, is a URL with a scheme // and a host. -func (c *Config) validateSentryDSN() error { - if c.SentryDSN == "" { +func validateSentryDSN(key, dsn string) error { + if dsn == "" { return nil } - parsed, err := url.Parse(c.SentryDSN) + parsed, err := url.Parse(dsn) if err != nil || parsed.Scheme == "" || parsed.Host == "" { return fmt.Errorf("%s: value %q is %w", - settingName(keySentryDSN), c.SentryDSN, errNotAValidURL) + settingName(key), dsn, errNotAValidURL) } return nil @@ -732,13 +737,11 @@ func (c *Config) validateSentryDSN() error { // is from 1 byte to 1 GiB. An upstream response is read whole into // memory, and the image processor reads one byte past this limit, which // must not overflow. -func (c *Config) validateUpstreamMaxResponseSize() error { +func validateUpstreamMaxResponseSize(key string, size int64) error { const maxUpstreamMaxResponseSize = 1 << 30 // 1 GiB - if c.UpstreamMaxResponseSize < 1 || - c.UpstreamMaxResponseSize > maxUpstreamMaxResponseSize { + if size < 1 || size > maxUpstreamMaxResponseSize { return fmt.Errorf("%s: value %d is %w 1-%d", - settingName(keyUpstreamMaxResponseSize), c.UpstreamMaxResponseSize, - errSizeOutOfRange, maxUpstreamMaxResponseSize) + settingName(key), size, errSizeOutOfRange, maxUpstreamMaxResponseSize) } return nil @@ -747,14 +750,13 @@ func (c *Config) validateUpstreamMaxResponseSize() error { // validateAccessControlAllowOrigin accepts "*" or an origin exactly as a browser // sends it: http or https, an IP address as netip writes it or a lowercase name // with a letter in its last part, and an optional port 1-65535, not the default. -func (c *Config) validateAccessControlAllowOrigin() error { - origin := c.AccessControlAllowOrigin +func validateAccessControlAllowOrigin(key, origin string) error { if origin == "*" { return nil } errOrigin := fmt.Errorf("%s: value %q is %w", - settingName(keyAccessControlAllowOrigin), origin, errNotAnOrigin) + settingName(key), origin, errNotAnOrigin) parsed, err := url.Parse(origin) if err != nil { @@ -804,30 +806,6 @@ func (c *Config) validateAccessControlAllowOrigin() error { return nil } -// validateConcurrencyLimits checks that the two upstream connection limits -// and the image processing limit are at least 1. -func (c *Config) validateConcurrencyLimits() error { - if c.UpstreamConnectionsPerHost < 1 { - return fmt.Errorf("%s: value %d %w", - settingName(keyUpstreamConnectionsPerHost), - c.UpstreamConnectionsPerHost, errMustBeAtLeastOne) - } - - if c.UpstreamConnections < 1 { - return fmt.Errorf("%s: value %d %w", - settingName(keyUpstreamConnections), - c.UpstreamConnections, errMustBeAtLeastOne) - } - - if c.MaxConcurrentProcessing < 1 { - return fmt.Errorf("%s: value %d %w", - settingName(keyMaxConcurrentProcessing), - c.MaxConcurrentProcessing, errMustBeAtLeastOne) - } - - return nil -} - // hostNamePattern matches a host name: letters, digits, hyphens, underscores // and dots, optionally after one leading dot. var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`) @@ -903,128 +881,31 @@ func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, erro return nil, nil //nolint:nilnil // nil config is valid (use defaults) } -// strictLoader accumulates the first error encountered while reading -// typed values out of a smartconfig instance, so Config construction -// can stay a single struct literal. -type strictLoader struct { - sc *smartconfig.Config - err error -} - -func (l *strictLoader) stringVal(key, defaultVal string) string { - if l.err != nil { - return "" +// getString returns the string v holds. A value that is not a string, or +// is explicitly null, is an error. +func getString(v value) (string, error) { + if v.raw == nil { + return "", errNullConfigValue(v.key) } - val, err := getString(l.sc, key, defaultVal) - if err != nil { - l.err = err - } - - return val -} - -func (l *strictLoader) intVal(key string, defaultVal int) int { - if l.err != nil { - return 0 - } - - val, err := getInt(l.sc, key, defaultVal) - if err != nil { - l.err = err - } - - return val -} - -func (l *strictLoader) int64Val(key string, defaultVal int64) int64 { - if l.err != nil { - return 0 - } - - val, err := getInt64(l.sc, key, defaultVal) - if err != nil { - l.err = err - } - - return val -} - -func (l *strictLoader) durationVal(key string, defaultVal time.Duration) time.Duration { - if l.err != nil { - return 0 - } - - val, err := getDuration(l.sc, key, defaultVal) - if err != nil { - l.err = err - } - - return val -} - -func (l *strictLoader) boolVal(key string, defaultVal bool) bool { - if l.err != nil { - return false - } - - val, err := getBool(l.sc, key, defaultVal) - if err != nil { - l.err = err - } - - return val -} - -func (l *strictLoader) hostListVal(key string) []string { - if l.err != nil { - return nil - } - - val, err := parseHostList(l.sc, key) - if err != nil { - l.err = err - } - - return val -} - -// getString returns the string value for key, or defaultVal if the key -// is omitted. A present value that is not a string, or is explicitly -// null, is an error. -func getString(sc *smartconfig.Config, key, defaultVal string) (string, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return defaultVal, nil - } - - if raw == nil { - return "", errNullConfigValue(key) - } - - str, ok := raw.(string) + str, ok := v.raw.(string) if !ok { return "", fmt.Errorf("config key %q: value %v (%T) is %w", - key, raw, raw, errNotAString) + v.key, v.raw, v.raw, errNotAString) } return str, nil } -// getInt returns the integer value for key, or defaultVal if the key is -// omitted. A present value that is not a whole number, or is explicitly -// null, is an error; fractional values are never truncated. -func getInt(sc *smartconfig.Config, key string, defaultVal int) (int, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return defaultVal, nil +// getInt returns the integer v holds. A value that is not a whole +// number, or is explicitly null, is an error; fractional values are +// never truncated. +func getInt(v value) (int, error) { + if v.raw == nil { + return 0, errNullConfigValue(v.key) } - if raw == nil { - return 0, errNullConfigValue(key) - } - - switch val := raw.(type) { + switch val := v.raw.(type) { case int: return val, nil case int64: @@ -1032,7 +913,7 @@ func getInt(sc *smartconfig.Config, key string, defaultVal int) (int, error) { case float64: if val != math.Trunc(val) { return 0, fmt.Errorf("config key %q: value %v is %w", - key, val, errNotAnInteger) + v.key, val, errNotAnInteger) } return int(val), nil @@ -1040,31 +921,25 @@ func getInt(sc *smartconfig.Config, key string, defaultVal int) (int, error) { parsed, err := strconv.Atoi(strings.TrimSpace(val)) if err != nil { return 0, fmt.Errorf("%s: value %q is %w", - settingName(key), val, errNotAnInteger) + settingName(v.key), val, errNotAnInteger) } return parsed, nil default: return 0, fmt.Errorf("config key %q: value %v (%T) is %w", - key, raw, raw, errNotAnInteger) + v.key, v.raw, v.raw, errNotAnInteger) } } -// getInt64 returns the 64-bit integer value for key, or defaultVal if -// the key is omitted. A present value that is not a whole number, or -// is explicitly null, is an error; fractional values are never -// truncated and out-of-range values are never clamped. -func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return defaultVal, nil +// getInt64 returns the 64-bit integer v holds. A value that is not a +// whole number, or is explicitly null, is an error; fractional values +// are never truncated and out-of-range values are never clamped. +func getInt64(v value) (int64, error) { + if v.raw == nil { + return 0, errNullConfigValue(v.key) } - if raw == nil { - return 0, errNullConfigValue(key) - } - - switch val := raw.(type) { + switch val := v.raw.(type) { case int: return int64(val), nil case int64: @@ -1072,14 +947,14 @@ func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, erro case uint64: if val > math.MaxInt64 { return 0, fmt.Errorf("config key %q: value %d %w", - key, val, errOverflowsInt64) + v.key, val, errOverflowsInt64) } return int64(val), nil case float64: if val != math.Trunc(val) { return 0, fmt.Errorf("config key %q: value %v is %w", - key, val, errNotAnInteger) + v.key, val, errNotAnInteger) } return int64(val), nil @@ -1087,174 +962,70 @@ func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, erro parsed, err := strconv.ParseInt(strings.TrimSpace(val), 10, 64) if err != nil { return 0, fmt.Errorf("%s: value %q is %w", - settingName(key), val, errNotAnInteger) + settingName(v.key), val, errNotAnInteger) } return parsed, nil default: return 0, fmt.Errorf("config key %q: value %v (%T) is %w", - key, raw, raw, errNotAnInteger) + v.key, v.raw, v.raw, errNotAnInteger) } } -// getDuration returns the duration value for key, or defaultVal if the -// key is omitted. A present value must be a positive Go duration string -// such as "30s" or "2m", read with time.ParseDuration; a bare number has -// no unit and is an error, as is an explicit null. -func getDuration( - sc *smartconfig.Config, key string, defaultVal time.Duration, -) (time.Duration, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return defaultVal, nil +// getDuration returns the duration v holds. The value must be a positive +// Go duration string such as "30s" or "2m", read with +// time.ParseDuration; a bare number has no unit and is an error, as is +// an explicit null. +func getDuration(v value) (time.Duration, error) { + if v.raw == nil { + return 0, errNullConfigValue(v.key) } - if raw == nil { - return 0, errNullConfigValue(key) - } - - str, ok := raw.(string) + str, ok := v.raw.(string) if !ok { return 0, fmt.Errorf("config key %q: value %v (%T) is %w", - key, raw, raw, errNotADuration) + v.key, v.raw, v.raw, errNotADuration) } parsed, err := time.ParseDuration(strings.TrimSpace(str)) if err != nil { return 0, fmt.Errorf("%s: value %q is %w", - settingName(key), str, errNotADuration) + settingName(v.key), str, errNotADuration) } if parsed <= 0 { return 0, fmt.Errorf("%s: value %q %w", - settingName(key), str, errMustBePositive) + settingName(v.key), str, errMustBePositive) } return parsed, nil } -// getBool returns the boolean value for key, or defaultVal if the key -// is omitted. A present value that is not a boolean (or a ParseBool-able -// string), or is explicitly null, is an error; numbers are not accepted -// as booleans. -func getBool(sc *smartconfig.Config, key string, defaultVal bool) (bool, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return defaultVal, nil +// getBool returns the boolean v holds. A value that is not a boolean (or +// a ParseBool-able string), or is explicitly null, is an error; numbers +// are not accepted as booleans. +func getBool(v value) (bool, error) { + if v.raw == nil { + return false, errNullConfigValue(v.key) } - if raw == nil { - return false, errNullConfigValue(key) - } - - switch val := raw.(type) { + switch val := v.raw.(type) { case bool: return val, nil case string: parsed, err := strconv.ParseBool(strings.TrimSpace(val)) if err != nil { return false, fmt.Errorf("%s: value %q is %w", - settingName(key), val, errNotABoolean) + settingName(v.key), val, errNotABoolean) } return parsed, nil default: return false, fmt.Errorf("config key %q: value %v (%T) is %w", - key, raw, raw, errNotABoolean) + v.key, v.raw, v.raw, errNotABoolean) } } -// validateAllowlistHostsValue checks the raw shape of the -// allowlist_hosts value before the lenient extraction in getStringSlice -// runs: an explicitly null value, a value that is not a list of strings -// (or a comma-separated string), a non-string entry, or an empty entry -// is an error, never silently skipped. -func validateAllowlistHostsValue(sc *smartconfig.Config) error { - raw, ok := lookupValue(sc, keyAllowlistHosts) - if !ok { - return nil - } - - if raw == nil { - return errNullConfigValue(keyAllowlistHosts) - } - - switch val := raw.(type) { - case []any: - for _, item := range val { - str, ok := item.(string) - if !ok { - return fmt.Errorf("config key %q: list entry %v (%T) is %w", - keyAllowlistHosts, item, item, errNotAString) - } - - if strings.TrimSpace(str) == "" { - return fmt.Errorf("config key %q: %w", - keyAllowlistHosts, errEmptyListEntry) - } - } - case string: - if strings.TrimSpace(val) == "" { - return nil - } - - for part := range strings.SplitSeq(val, ",") { - if strings.TrimSpace(part) == "" { - return fmt.Errorf("%s: value %q %w", - settingName(keyAllowlistHosts), val, errEmptyEntry) - } - } - default: - return fmt.Errorf("config key %q: value %v (%T) is %w", - keyAllowlistHosts, raw, raw, errNotAStringList) - } - - return nil -} - -// getStringSlice returns the allowlist_hosts list of strings, or nil if -// the key is omitted. It accepts a YAML list of strings or a -// comma-separated string (backwards compatibility). Malformed entries -// are rejected beforehand by validateAllowlistHostsValue. -func getStringSlice(sc *smartconfig.Config) []string { - val, ok := lookupValue(sc, keyAllowlistHosts) - if !ok || val == nil { - return nil - } - - // Handle YAML list format - if slice, ok := val.([]any); ok { - result := make([]string, 0, len(slice)) - for _, item := range slice { - if str, ok := item.(string); ok { - trimmed := strings.TrimSpace(str) - if trimmed != "" { - result = append(result, trimmed) - } - } - } - - return result - } - - // Fall back to comma-separated string for backwards compatibility - if str, ok := val.(string); ok && str != "" { - parts := strings.Split(str, ",") - result := make([]string, 0, len(parts)) - - for _, part := range parts { - trimmed := strings.TrimSpace(part) - if trimmed != "" { - result = append(result, trimmed) - } - } - - return result - } - - return nil -} - // defaultTrustedProxies returns the trusted_proxies default: the three RFC // 1918 private ranges. pixa is always deployed behind a TLS-terminating // reverse proxy, which in practice sits on a private network, so its @@ -1270,12 +1041,10 @@ func defaultTrustedProxies() []netip.Prefix { } } -// parseCIDRList parses the value of the named config key into CIDR -// prefixes, or returns nil if the key is omitted. It accepts a YAML list -// of strings or a comma-separated string. An explicitly null value, a -// wrong type, an empty entry, a non-string entry, or an unparseable CIDR -// aborts startup naming the key and the offending value; the default -// (an empty list) applies only to an omitted key. +// parseCIDRList parses the list v holds into CIDR prefixes. It accepts a +// YAML list of strings or a comma-separated string. An explicitly null +// value, a wrong type, an empty entry, a non-string entry, or an +// unparseable CIDR aborts startup naming the key and the offending value. // // An entry written in the IPv4-mapped form (::ffff:a.b.c.d) is kept as the // IPv4 range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. Both @@ -1283,17 +1052,12 @@ func defaultTrustedProxies() []netip.Prefix { // address as the IPv4 address it holds, so the entry as written would match // nothing. Such an entry shorter than /96 covers more than the IPv4 // addresses and aborts startup. -func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return nil, nil +func parseCIDRList(v value) ([]netip.Prefix, error) { + if v.raw == nil { + return nil, errNullConfigValue(v.key) } - if raw == nil { - return nil, errNullConfigValue(key) - } - - entries, err := listEntries(raw, key) + entries, err := listEntries(v.raw, v.key) if err != nil { return nil, err } @@ -1308,12 +1072,12 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { prefix, err := netip.ParsePrefix(entry) if err != nil { return nil, fmt.Errorf("%s: value %q is %w", - settingName(key), entry, errNotAValidCIDR) + settingName(v.key), entry, errNotAValidCIDR) } if prefix.Addr().Is4In6() && prefix.Bits() < mappedPrefixBits { return nil, fmt.Errorf("%s: value %q is %w", - settingName(key), entry, errMappedTooShort) + settingName(v.key), entry, errMappedTooShort) } if prefix.Addr().Is4In6() { @@ -1327,28 +1091,23 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) { return prefixes, nil } -// parseHostList parses the value of the named config key into host patterns, -// or returns nil if the key is omitted. It accepts a YAML list of strings or a -// comma-separated string. An explicitly null value, a wrong type, an empty -// entry, a non-string entry, or an entry validateHostPattern rejects aborts -// startup naming the key and the offending value. -func parseHostList(sc *smartconfig.Config, key string) ([]string, error) { - raw, ok := lookupValue(sc, key) - if !ok { - return nil, nil +// parseHostList parses the list v holds into host patterns. It accepts a +// YAML list of strings or a comma-separated string. An explicitly null +// value, a wrong type, an empty entry, a non-string entry, or an entry +// validateHostPattern rejects aborts startup naming the key and the +// offending value. +func parseHostList(v value) ([]string, error) { + if v.raw == nil { + return nil, errNullConfigValue(v.key) } - if raw == nil { - return nil, errNullConfigValue(key) - } - - entries, err := listEntries(raw, key) + entries, err := listEntries(v.raw, v.key) if err != nil { return nil, err } for _, entry := range entries { - err := validateHostPattern(key, entry) + err := validateHostPattern(v.key, entry) if err != nil { return nil, err } @@ -1359,8 +1118,8 @@ func parseHostList(sc *smartconfig.Config, key string) ([]string, error) { // listEntries extracts the raw entries of the named list key as trimmed, // non-empty strings, from either a YAML list of strings or a comma-separated -// string; an empty string is an empty list, as for allowlist_hosts. Any other -// shape is a configuration error. +// string; an empty string is an empty list. Any other shape is a +// configuration error. func listEntries(raw any, key string) ([]string, error) { switch val := raw.(type) { case []any: diff --git a/internal/config/config_internal_test.go b/internal/config/config_internal_test.go index b655b35..bae94c7 100644 --- a/internal/config/config_internal_test.go +++ b/internal/config/config_internal_test.go @@ -1,97 +1,12 @@ package config import ( - "os" "path/filepath" "testing" "git.eeqj.de/sneak/smartconfig" ) -// writeTestConfig writes yamlContent to a temp config file and returns -// the file path. -func writeTestConfig(t *testing.T, yamlContent string) string { - t.Helper() - - tmpDir := t.TempDir() - configPath := filepath.Join(tmpDir, "config.yml") - - err := os.WriteFile(configPath, []byte(yamlContent), 0o600) - if err != nil { - t.Fatalf("failed to write test config: %v", err) - } - - return configPath -} - -// checkAllowlistHosts loads the config at configPath and asserts that -// getStringSlice returns the three expected hosts. -func checkAllowlistHosts(t *testing.T, configPath string) { - t.Helper() - - sc, err := loadTestConfig(configPath) - if err != nil { - t.Fatalf("failed to load config: %v", err) - } - - hosts := getStringSlice(sc) - - if len(hosts) != 3 { - t.Errorf("expected 3 hosts, got %d: %v", len(hosts), hosts) - } - - expected := []string{"static.sneak.cloud", "sneak.berlin", testHostS3} - for i, want := range expected { - if i >= len(hosts) { - t.Errorf("missing host at index %d: want %q", i, want) - - continue - } - - if hosts[i] != want { - t.Errorf("host[%d] = %q, want %q", i, hosts[i], want) - } - } -} - -func TestGetStringSlice_YAMLList(t *testing.T) { - t.Parallel() - - yamlContent := ` -allowlist_hosts: - - static.sneak.cloud - - sneak.berlin - - s3.sneak.cloud -` - - checkAllowlistHosts(t, writeTestConfig(t, yamlContent)) -} - -func TestGetStringSlice_CommaSeparated(t *testing.T) { - t.Parallel() - - // Backwards compatibility with comma-separated string values. - yamlContent := `allowlist_hosts: "static.sneak.cloud, sneak.berlin, s3.sneak.cloud"` - - checkAllowlistHosts(t, writeTestConfig(t, yamlContent)) -} - -func TestGetStringSlice_Empty(t *testing.T) { - t.Parallel() - - configPath := writeTestConfig(t, `port: 8080`) - - sc, err := loadTestConfig(configPath) - if err != nil { - t.Fatalf("failed to load config: %v", err) - } - - hosts := getStringSlice(sc) - if len(hosts) != 0 { - t.Errorf("expected nil or empty slice, got %v", hosts) - } -} - // TestExampleConfigAllowlistsNoHost loads configs/config.example.yml // with a real signing key, as Getting Started in README.md has it copied // and edited, and checks that it allowlists no host: every image on an diff --git a/internal/config/config_validation_internal_test.go b/internal/config/config_validation_internal_test.go index 1324050..1aaac63 100644 --- a/internal/config/config_validation_internal_test.go +++ b/internal/config/config_validation_internal_test.go @@ -15,8 +15,36 @@ import ( _ "modernc.org/sqlite" // SQLite driver registration ) +// The config keys the tests name in the errors they expect. +const ( + keyDebug = "debug" + keyMaintenanceMode = "maintenance_mode" + keyPort = "port" + keyStateDir = "state_dir" + keySentryDSN = "sentry_dsn" + keyDBURL = "db_url" + keyMetrics = "metrics" + keyMetricsUsername = "metrics.username" + keyMetricsPassword = "metrics.password" + keySigningKey = "signing_key" + keyAllowlistHosts = "allowlist_hosts" + keyRefererBlocklist = "referer_blocklist" + keyAllowHTTP = "allow_http" + keyUpstreamConnectionsPerHost = "upstream_connections_per_host" + keyUpstreamConnections = "upstream_connections" + keyMaxConcurrentProcessing = "max_concurrent_processing" + keyCacheMaxBytes = "cache_max_bytes" + keyBlockedNetworks = "blocked_networks" + keyTrustedProxies = "trusted_proxies" + keyAccessControlAllowOrigin = "access_control_allow_origin" + keyUpstreamFetchTimeout = "upstream_fetch_timeout" + keyUpstreamMaxResponseSize = "upstream_max_response_size" + keyDownstreamTimeout = "downstream_timeout" + keyImageRequestsPerMinute = "image_requests_per_minute" +) + // validTestSigningKey is a 32-character signing key that satisfies the -// minimum length requirement in validate(). +// minimum length requirement in validateSigningKey. const validTestSigningKey = "0123456789abcdef0123456789abcdef" // signingKeyLine is a valid signing_key config line used as the base of diff --git a/internal/config/documented_settings_internal_test.go b/internal/config/documented_settings_internal_test.go new file mode 100644 index 0000000..6670252 --- /dev/null +++ b/internal/config/documented_settings_internal_test.go @@ -0,0 +1,127 @@ +package config + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// TestReadmeNamesEverySetting checks that the list of settings in +// README.md names exactly the keys of the settings table, one item each. +// The list is the first one in the Configuration section, and it ends at +// the first blank line. Each item starts with its key in backticks. +func TestReadmeNamesEverySetting(t *testing.T) { + t.Parallel() + + readme, err := os.ReadFile(filepath.Join("..", "..", "README.md")) + if err != nil { + t.Fatalf("failed to read README.md: %v", err) + } + + _, section, found := strings.Cut(string(readme), "\n### Configuration\n") + if !found { + t.Fatal("README.md has no Configuration section") + } + + // The section ends at the next heading of its level or above. + section, _, _ = strings.Cut(section, "\n## ") + section, _, _ = strings.Cut(section, "\n### ") + + _, list, found := strings.Cut(section, "\n- ") + if !found { + t.Fatal("README.md's Configuration section has no list") + } + + list, _, _ = strings.Cut("- "+list, "\n\n") + + itemKey := regexp.MustCompile("^- `([^`]+)`") + named := map[string]bool{} + + for line := range strings.SplitSeq(list, "\n") { + // A line that does not start with "- " goes on with the item above. + if !strings.HasPrefix(line, "- ") { + continue + } + + match := itemKey.FindStringSubmatch(line) + if match == nil { + t.Errorf("README.md's list of settings has an item "+ + "that does not start with a key in backticks: %s", line) + + continue + } + + if named[match[1]] { + t.Errorf("README.md's list of settings names %s twice", match[1]) + } + + named[match[1]] = true + } + + for _, s := range new(Config).settings() { + if !named[s.key] { + t.Errorf("README.md's list of settings does not name %s", s.key) + } + + delete(named, s.key) + } + + for key := range named { + t.Errorf("README.md's list of settings names %s, "+ + "which is not a setting", key) + } +} + +// TestExampleConfigNamesEverySetting checks that +// configs/config.example.yml names exactly the keys of the settings +// table, each one set or commented out with "# ". A key line holds the +// key, a colon and at most a one-word value, which tells it from a +// comment line that starts with a word and a colon. An indented key is +// inside the map of the key above it, as username is inside metrics. +func TestExampleConfigNamesEverySetting(t *testing.T) { + t.Parallel() + + const examplePath = "configs/config.example.yml" + + example, err := os.ReadFile(filepath.Join("..", "..", examplePath)) + if err != nil { + t.Fatalf("failed to read %s: %v", examplePath, err) + } + + keyLine := regexp.MustCompile(`(?m)^(?:# )?( *)([a-z_]+):(?: \S+)?$`) + named := map[string]bool{} + + var mapKey string + + for _, match := range keyLine.FindAllStringSubmatch(string(example), -1) { + indent, key := match[1], match[2] + if indent == "" { + mapKey = key + named[key] = true + + continue + } + + // The key holding the map is not itself a setting. + delete(named, mapKey) + + named[mapKey+"."+key] = true + } + + // The env section sets environment variables; it is not a setting. + delete(named, "env") + + for _, s := range new(Config).settings() { + if !named[s.key] { + t.Errorf("%s does not name %s", examplePath, s.key) + } + + delete(named, s.key) + } + + for key := range named { + t.Errorf("%s names %s, which is not a setting", examplePath, key) + } +} diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go index 4b23302..435c825 100644 --- a/internal/config/env_internal_test.go +++ b/internal/config/env_internal_test.go @@ -138,7 +138,8 @@ func TestPixaPortAbortsStartupPointingToPort(t *testing.T) { // TestSettingVariablesAndConfigPathAreAccepted checks that every // setting's variable and PIXA_CONFIG_PATH pass the check for unknown -// PIXA_ variables. TestEnvironmentSetsEveryKey pins the names in the list. +// PIXA_ variables. TestEnvironmentSetsEveryKey pins the names in the +// settings table. func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) { // A config file's env section loaded by another test can leave a // PIXA_ variable set for the whole process, so every one is unset @@ -159,8 +160,8 @@ func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) { t.Setenv("PIXA_CONFIG_PATH", "/etc/pixa/config.yml") - for _, name := range envVarNames() { - t.Setenv(name, "") + for _, s := range new(Config).settings() { + t.Setenv(s.envVar, "") } err := validateKnownEnvVars() From 6de69f73496a6f39777eda17cdb257d762e4d202 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 23:19:19 +0100 Subject: [PATCH 111/131] Name pixa's concepts at the start of the README's Design section (closes #321) The Design section starts with a Concepts list. Each concept gets one sentence, under the name the code gives it. Storage calls the transformed images variants, and the sentences it changed for that are shorter. Its paths end in the source key and the variant key, and it lists the image cache's tables in the database. Architecture names internal/providers and lists the packages that work with the concepts. No code or test changes. Model: opus-5-5 --- README.md | 128 +++++++++++++++++++++++++++++++++++++++--------------- TODO.md | 7 +++ 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b748270..5268bc1 100644 --- a/README.md +++ b/README.md @@ -178,47 +178,91 @@ allowlisted source hosts for open access. ## Design +### Concepts + +These terms are the names the code uses: + +- **Source**: the image at one upstream URL, named by its host, path and query. +- **Source key**: the SHA-256 of a source's host, path and query, which names + the source's file and its row in the database. +- **Transform**: the output size, format, quality and fit an image request asks + for; with the format `auto`, pixa chooses the output format from the `Accept` + header. +- **Image request**: a source and a transform, which each image route reads from + its URL. +- **Variant**: the image pixa serves for an image request, made from its source + with its transform. +- **Variant key**: the SHA-256 of a variant's source, size, format, quality and + fit, which names the variant's file and its row in the database, and gives its + `ETag`. +- **Signed URL**: a `/v1/image/` URL whose `sig` signs its image request and its + expiry `exp` with `signing_key`. +- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any + expiry, encrypted with a key derived from `signing_key`. +- **Login session**: an encrypted cookie that keeps a browser logged in to the + generator page for 30 days, or until a logout replaces the random value pixa + keeps in the `logout` table (see Encrypted URLs). +- **Negative cache**: the source URLs pixa does not fetch again for 5 minutes, + after one of these failed fetches: + - the upstream answered with an error status + - the body's first bytes were not those of the format its `Content-Type` + names +- **Image cache**: the part of pixa that keeps the sources, the variants and the + negative cache (see Storage). +- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached + sources and variants may reach together. +- **Image proxy**: the part of pixa that serves a variant from the image cache + when it can, and otherwise makes it from the source. + ### Storage -- **Source images**: - `/cache/sources///`, one - file for each source URL +- **Sources**: `/cache/sources///`, one file for + each source URL - **Database**: `/state.sqlite3` (SQLite) -- **Transformed images**: - `/cache/variants///` +- **Variants**: `/cache/variants///` `` and `` are the first and second pairs of characters of the file's name. -Each source image and each transformed image has one row in the database, -holding its size and when it was last used. pixa reads a cached source image's -format from its file's first bytes. When they match none of the source formats, -pixa fetches the source again. Two source URLs that serve the same bytes are -stored twice, once under each URL. A source image is stored by writing its file -and then its row; when the row cannot be written, the file is removed again. +The image cache keeps these tables in the database: -Source and transformed image files are written under a temporary name, synced to -disk, and only then renamed into place, so a crash leaves no file under its own -name with part of its data. When pixa starts, and every five minutes after, it -removes each cached source or transformed image whose file is empty, or not the -size the database records for it, with a warning naming the file; the next -request for it fetches the source or converts the image again. Looking for those -files reads the database once for every cached file, so pixa skips each -five-minute look that would start less than ten times as long after the last -look ended as that look took. After a look that took one minute, the next one -starts 10 to 15 minutes after that look ended. +- `source_content`: a row for each source, by source key, with its size and when + it was last used +- `variant_content`: a row for each variant, by variant key (column + `cache_key`), with its size and when it was last used +- `cache_usage`: the total size of the cached sources and variants, which + eviction reads +- `negative_cache`: a row for each source URL in the negative cache, with when + it expires + +pixa reads a cached source image's format from its file's first bytes. When they +match none of the source formats, pixa fetches the source again. Two source URLs +that serve the same bytes are stored twice, once under each URL. A source image +is stored by writing its file and then its row; when the row cannot be written, +the file is removed again. + +Each source and variant file is written under a temporary name and synced to +disk before it is renamed into place. So a crash never leaves part of a file +under its own name. When pixa starts, and every five minutes after, it removes +each source or variant whose file is empty, or not the size its row records. It +logs a warning naming the file. A later request for that image fetches the +source or makes the variant again. Looking for those files reads the database +once for every cached file, so pixa skips each five-minute look that would start +less than ten times as long after the last look ended as that look took. After a +look that took one minute, the next one starts 10 to 15 minutes after that look +ended. pixa's target is 1-5k r/s, which has not been measured at that rate (see Load -Test). Toward it, a cache hit opens the transformed image's file and nothing -else, and makes no database call. The content type comes from the file's first -bytes, which are those of the format pixa wrote it in; a file whose first bytes -are not those of a format pixa writes is not served, and the image is converted -again. A hit hands the time it was served to a background writer and does not -wait for the writer. Eviction orders by that time. The hit is counted in memory. -A miss that uses a cached source image also hands that writer the time it used -the image. Once a second that writer writes the waiting times in one -transaction. While 10,000 times of transformed images, or 10,000 of source -images, wait to be written, a further one is dropped, and that image only looks +Test). Toward it, a cache hit opens the variant's file and nothing else, and +makes no database call. The content type comes from the file's first bytes, +which are those of the format pixa wrote it in; a file whose first bytes are not +those of a format pixa writes is not served, and the image is converted again. A +hit hands the time it was served to a background writer and does not wait for +the writer. Eviction orders by that time. The hit is counted in memory. A miss +that uses a cached source image also hands that writer the time it used the +image. Once a second that writer writes the waiting times in one transaction. Up +to 10,000 access times of variants, and as many of sources, can wait to be +written. A time handed over beyond that is dropped. Its image then only looks less recently used to eviction. `script/bench` measures how many hits pixa serves per second (see Load Test). @@ -880,9 +924,25 @@ See `configs/config.example.yml` for all options with defaults. ### Architecture -- **Dependency injection**: Uber fx. `internal/app` builds every part of pixad - from the config in one fx graph. Each setting's default is declared once, in - `internal/config`. +- **Dependency injection**: Uber fx. `internal/app` puts every part of pixad in + one fx graph. `internal/providers` builds the image cache, the image proxy and + the other parts that serve images and logins. Each setting's default is + declared once, in `internal/config`. +- **Packages**: these packages under `internal/` work with the concepts above: + - `handlers`: the routes, which read the image request from the URL, check + the signed or encrypted URL, and make the login session + - `imagerequest`: the image request, with the source key and the variant key + - `imageproxy`: the image proxy + - `imgcache`: the image cache, which keeps its sources in one `FileStore` + and its variants in another + - `httpfetcher`: fetching sources from upstream hosts + - `magic`: reading an image's format from its first bytes + - `imageprocessor`: making a variant from a source with libvips + - `signature`: checking the `sig` of a signed URL + - `encurl`: making and reading the tokens of encrypted URLs + - `seal`: the keys derived from `signing_key`, and the encryption of those + tokens + - `session`: the login session - **HTTP router**: go-chi - **Image processing**: govips (CGO wrapper for libvips) - **Database**: SQLite via modernc.org/sqlite diff --git a/TODO.md b/TODO.md index f0cba91..92228e4 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,13 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 `README.md`'s Design section starts with a Concepts list (closes + #321). Each concept gets one sentence, under the name the code gives it. + - Storage calls the transformed images variants, in shorter sentences. + - Storage's paths end in the source key and the variant key. + - Storage lists the image cache's tables in the database. + - Architecture names `internal/providers`, and lists the packages that work + with the concepts. - 2026-10-26 one table in `internal/config` lists every setting once (closes #320). `Config.settings` gives each setting's key, its environment variable and the function that loads it. From d33316caf3fdfc36cb16f17a664ce43138c17aef Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 23:26:07 +0000 Subject: [PATCH 112/131] Test that a client leaving while another request makes its image gets 499 On both image routes, a request waits for an image another request is making, and its client goes away. The first test wants that request answered 499 and logged once at info level. It fails on next, which answers 500 and logs it at error level. The second test pins the 500 that a cancellation keeps while the request's context is live; it passes on next. The fix follows in the next commit. Model: opus-5-5 --- .../handlers/client_gone_internal_test.go | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 internal/handlers/client_gone_internal_test.go diff --git a/internal/handlers/client_gone_internal_test.go b/internal/handlers/client_gone_internal_test.go new file mode 100644 index 0000000..108a5a9 --- /dev/null +++ b/internal/handlers/client_gone_internal_test.go @@ -0,0 +1,177 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" + + "github.com/go-chi/chi/v5/middleware" + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// heldFetcher sends on started when a fetch begins, and holds the fetch until +// release is closed. It then fails the fetch as an upstream error. +type heldFetcher struct { + started chan struct{} + release chan struct{} +} + +// Fetch sends on f.started, waits for f.release to be closed, and returns +// httpfetcher.ErrUpstreamError. +func (f heldFetcher) Fetch( + context.Context, imagerequest.Source, +) (*httpfetcher.FetchResult, error) { + f.started <- struct{}{} + + <-f.release + + return nil, httpfetcher.ErrUpstreamError +} + +// readLogLines decodes the JSON log lines in logs. +func readLogLines(t *testing.T, logs *bytes.Buffer) []map[string]any { + t.Helper() + + t.Logf("logged:\n%s", logs.String()) + + var lines []map[string]any + + dec := json.NewDecoder(logs) + for dec.More() { + var line map[string]any + + err := dec.Decode(&line) + if err != nil { + t.Fatalf("decoding log line: %v", err) + } + + lines = append(lines, line) + } + + return lines +} + +// TestImageRoutes_ClientGoneWhileWaitingAnswers499 checks that both image +// routes answer 499 to a request whose client goes away while it waits for an +// image another request is making. A first request's fetch is held, so the +// image is being made, and a second request for the same image waits for it +// until its client goes away. The second request must be logged once at info +// level with its request ID and the source host and path. Nothing may be +// logged at error level. +func TestImageRoutes_ClientGoneWhileWaitingAnswers499(t *testing.T) { + t.Parallel() + + const requestID = "test-request-id" + + for route, imageURL := range photoURLsByRoute() { + t.Run(route, func(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + fetcher := heldFetcher{started: make(chan struct{}), release: make(chan struct{})} + h, routes := newRoutesFetchingThrough(t, fetcher, + slog.New(slog.NewJSONHandler(&logs, nil))) + target := imageURL(t, h) + + firstDone := make(chan struct{}) + + go func() { + defer close(firstDone) + + sendGet(t, routes, target) + }() + + <-fetcher.started + + ctx, cancel := context.WithCancel(context.WithValue(t.Context(), + middleware.RequestIDKey, requestID)) + rec := httptest.NewRecorder() + secondDone := make(chan struct{}) + + go func() { + defer close(secondDone) + + routes.ServeHTTP(rec, httptest.NewRequestWithContext( + ctx, http.MethodGet, target, nil)) + }() + + // Give the second request time to start waiting for the first + // request's image. The fetch is still held, so the second request + // can end only because its client went away + time.Sleep(100 * time.Millisecond) + cancel() + <-secondDone + + close(fetcher.release) + <-firstDone + + if rec.Code != 499 { + t.Errorf("status = %d, want 499", rec.Code) + } + + var clientGoneLines int + + for _, line := range readLogLines(t, &logs) { + if line["level"] == slog.LevelError.String() { + t.Errorf("logged at error level: %v", line["msg"]) + } + + if line["level"] == slog.LevelInfo.String() && + line["msg"] == "client went away before the image was ready" && + line["request_id"] == requestID && + line["host"] == signedHost && + line["path"] == photoPath { + clientGoneLines++ + } + } + + if clientGoneLines != 1 { + t.Errorf("info line naming the request logged %d times, want 1", + clientGoneLines) + } + }) + } +} + +// TestImageRoutes_CanceledWithLiveRequestAnswers500 checks that both image +// routes answer 500, logged as "failed to get image" at error level, for an +// image service error that wraps context.Canceled while the request's own +// context is live. Its client has not gone away, so the cancellation is +// pixa's own failure. +func TestImageRoutes_CanceledWithLiveRequestAnswers500(t *testing.T) { + t.Parallel() + + for route, imageURL := range photoURLsByRoute() { + t.Run(route, func(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + h, routes := newRoutesFetchingThrough(t, + failingFetcher{err: context.Canceled}, + slog.New(slog.NewJSONHandler(&logs, nil))) + + rec := sendGet(t, routes, imageURL(t, h)) + if rec.Code != http.StatusInternalServerError { + t.Errorf("status = %d, want %d", + rec.Code, http.StatusInternalServerError) + } + + if !slices.ContainsFunc(readLogLines(t, &logs), + func(line map[string]any) bool { + return line["level"] == slog.LevelError.String() && + line["msg"] == "failed to get image" + }) { + t.Error(`no "failed to get image" line logged at error level`) + } + }) + } +} From 233765033afa3a1d11f5ee0408fffcc4cb44cc0e Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 23:26:36 +0000 Subject: [PATCH 113/131] Answer 499 to a client that goes away while waiting for an image (closes #339) When a request waits for an image another request is making and its client goes away, the image proxy returns context.Canceled. serveImage now answers it 499, the status nginx calls "client closed request". It logs it once at info level with the request_id, host and path. It was answered 500 and logged at error level, as pixa's own failure. A cancellation while the request's context is live is still answered 500. The request making the image is still answered as before. README.md lists the 499 on both image routes. Model: opus-5-5 --- README.md | 3 +++ TODO.md | 11 +++++++++++ internal/handlers/image.go | 25 ++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5268bc1..a643476 100644 --- a/README.md +++ b/README.md @@ -310,6 +310,8 @@ path under `/v1/` answers 200, in maintenance mode too. - 403 when the upstream host, or a host it redirects to, has an address in a blocked network (see `blocked_networks`). - 429 over `image_requests_per_minute` (below). + - 499, with no body, when the client goes away before its image is ready, + unless its request is the one making the image (below). - 415 when the upstream answers with a `Content-Type` that is not one of the source formats (below). - 415 when the upstream sends a body whose first bytes are not those of the @@ -341,6 +343,7 @@ path under `/v1/` answers 200, in maintenance mode too. - 403 - 415 - 429 + - 499, with no body - 502 - 503 - 504 diff --git a/TODO.md b/TODO.md index 92228e4..f00e81f 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 on both image routes, a request is answered 499 and logged at info + level when its client goes away before its image is ready, unless it is the + request making the image (closes #339). It was answered 500 and logged at + error level, as pixa's own failure. + - The line is "client went away before the image was ready". It carries the + `request_id`, the host and the path. + - The 499 has no body, since nobody reads it. + - `serveImage` answers it before calling `respondImageError`, which cannot + see the request's context. + - An error wrapping `context.Canceled` while the request's context is live + is still answered 500 and logged at error level. - 2026-10-26 `README.md`'s Design section starts with a Concepts list (closes #321). Each concept gets one sentence, under the name the code gives it. - Storage calls the transformed images variants, in shorter sentences. diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 68c2fee..ff11b6e 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "errors" "fmt" "io" @@ -19,6 +20,12 @@ import ( "sneak.berlin/go/pixa/internal/magic" ) +// statusClientClosedRequest is the status nginx calls "client closed +// request". The image routes answer it when the client goes away before its +// image is ready, unless its request is the one making the image. Nobody reads +// it; it is for logs and metrics. +const statusClientClosedRequest = 499 + // HandleImage handles the main image proxy route: // /v1/image///x., or with no format // /v1/image///x @@ -345,7 +352,8 @@ func notModified(w http.ResponseWriter, r *http.Request, etag string) bool { // serveImage serves the image that req asks for. Both image routes call it // once they have the request and its URL's expiry, which ends the // Cache-Control max-age. It chooses the format for the format auto, answers -// If-None-Match with 304 and HEAD with the headers only, and answers an error +// If-None-Match with 304 and HEAD with the headers only. It answers 499 when +// the image service stopped because the client went away, and any other error // of the image service with respondImageError. func (s *Handlers) serveImage( w http.ResponseWriter, r *http.Request, @@ -362,6 +370,21 @@ func (s *Handlers) serveImage( startTime := time.Now() resp, err := s.imgSvc.Get(r.Context(), req) + + // The client went away before its image was ready, which is not pixa's + // failure. A cancellation while the request's context is live is pixa's, + // and respondImageError answers it 500. + if errors.Is(err, context.Canceled) && r.Context().Err() != nil { + s.log.Info("client went away before the image was ready", + "request_id", middleware.GetReqID(r.Context()), + "host", req.Source.Host, + "path", req.Source.Path, + ) + w.WriteHeader(statusClientClosedRequest) + + return + } + if err != nil { s.respondImageError(w, req, err) From 701ed9b94b9cfccbf7b7cf2d94699c1c06dc420c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 23:54:24 +0000 Subject: [PATCH 114/131] Test that reconciliation drops a variant's row only when its file is missing reconcileVariantRows is given two tracked variants. The first one's file is a symbolic link to itself, so reading its size fails with an error other than a missing file. The test wants its row kept, with a warning naming its key and the error. The second one has no file, and the test wants its row dropped with the info line for a missing file. The test fails on next, which drops both rows and logs both files as missing. The fix follows in the next commit. Model: opus-5-5 --- internal/imgcache/eviction_internal_test.go | 86 +++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index 15b2ebc..ed7b4eb 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -699,6 +699,92 @@ func TestStartEvictionReconcilesAccountingWithDisk(t *testing.T) { } } +// logLineNaming returns the first line of logs, written by slog's JSON +// handler, that is at level and names the variant key, or "" if none is. +func logLineNaming(logs, level string, key imagerequest.VariantKey) string { + for line := range strings.Lines(logs) { + if strings.Contains(line, `"level":"`+level+`"`) && + strings.Contains(line, `"cache_key":"`+string(key)+`"`) { + return line + } + } + + return "" +} + +// TestReconcileVariantRowsDropsOnlyRowsOfMissingFiles gives +// reconcileVariantRows two tracked variants. The first one's file is a +// symbolic link to itself, so its size cannot be read. Its row must stay, +// with a warning naming its key and the error. The second one has no file. +// Its row must go, with the info line for a missing file. +func TestReconcileVariantRowsDropsOnlyRowsOfMissingFiles(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + storeEvictionTestVariant(t, cache, testVariantKeyOne, []byte("variant one")) + storeEvictionTestVariant(t, cache, testVariantKeyTwo, []byte("variant two")) + + unreadablePath := cache.variants.keyToPath(string(testVariantKeyOne)) + + err := os.Remove(unreadablePath) + if err != nil { + t.Fatalf("failed to remove the first variant's file: %v", err) + } + + err = os.Symlink(unreadablePath, unreadablePath) + if err != nil { + t.Fatalf("failed to link the first variant's path to itself: %v", err) + } + + _, statErr := os.Stat(unreadablePath) + if statErr == nil || errors.Is(statErr, fs.ErrNotExist) { + t.Fatalf("os.Stat() of the first variant: error = %v, "+ + "want an error other than a missing file", statErr) + } + + err = os.Remove(cache.variants.keyToPath(string(testVariantKeyTwo))) + if err != nil { + t.Fatalf("failed to remove the second variant's file: %v", err) + } + + var logBuf bytes.Buffer + + cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil)) + + err = cache.reconcileVariantRows(t.Context()) + t.Logf("reconcileVariantRows() error = %v, log: %s", err, logBuf.String()) + + if err != nil { + t.Fatalf("reconcileVariantRows() error = %v", err) + } + + rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?` + + if n := countRows(t, cache, rowQuery, string(testVariantKeyOne)); n != 1 { + t.Errorf("the variant whose size cannot be read has %d rows, want 1", n) + } + + if n := countRows(t, cache, rowQuery, string(testVariantKeyTwo)); n != 0 { + t.Errorf("the variant with no file has %d rows, want 0", n) + } + + logs := logBuf.String() + + warning := logLineNaming(logs, "WARN", testVariantKeyOne) + if !strings.Contains(warning, statErr.Error()) { + t.Errorf("no warning names %s and the error %q", testVariantKeyOne, statErr) + } + + if line := logLineNaming(logs, "INFO", testVariantKeyOne); line != "" { + t.Errorf("the variant whose size cannot be read was logged at info: %s", line) + } + + dropped := logLineNaming(logs, "INFO", testVariantKeyTwo) + if !strings.Contains(dropped, "dropped accounting row for missing variant file") { + t.Errorf("no info line says the file of %s is missing", testVariantKeyTwo) + } +} + // TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves // reconciliation is not a one-shot startup-only pass: it must also run // on the periodic ticker, so a variant file that lands on disk with no From 33045a24b5cfea58ede6916d60158e513d87ab6d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 23:56:34 +0000 Subject: [PATCH 115/131] Keep a variant's accounting row when its file cannot be read (closes #340) The reconciliation pass dropped a variant's accounting row on any error reading its file's size, and logged every such file as missing. A file still on disk then had no row, so eviction no longer counted it against cache_max_bytes. Now only a missing file (ErrNotFound, which FileStore.Size returns for it) loses its row, with the same info line. Any other error keeps the row and logs a warning naming the variant key and the error, and the pass goes on to the next key. README.md says so under Storage. Model: opus-5-5 --- README.md | 4 ++++ TODO.md | 7 +++++++ internal/imgcache/eviction.go | 11 ++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a643476..49cafcb 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,10 @@ less than ten times as long after the last look ended as that look took. After a look that took one minute, the next one starts 10 to 15 minutes after that look ended. +Each look also drops the rows of variants whose files are missing. When pixa +cannot read a variant file's size for another reason, it keeps the variant's +row. It logs a warning naming the variant key and the error. + pixa's target is 1-5k r/s, which has not been measured at that rate (see Load Test). Toward it, a cache hit opens the variant's file and nothing else, and makes no database call. The content type comes from the file's first bytes, diff --git a/TODO.md b/TODO.md index f00e81f..4b6920f 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,13 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-26 the reconciliation pass drops a variant's accounting row only when + its file is missing (closes #340). Any other error reading the file's size + keeps the row. It dropped the row on any error and logged the file as missing. + - The kept row gets the warning "kept accounting row for unreadable variant + file", with the variant key and the error. + - The pass checks for `ErrNotFound`, which `FileStore.Size` returns for a + missing file. - 2026-10-26 on both image routes, a request is answered 499 and logged at info level when its client goes away before its image is ready, unless it is the request making the image (closes #339). It was answered 500 and logged at diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index ef5f346..2312a14 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -613,7 +613,9 @@ func (c *Cache) adoptVariantFile( } // reconcileVariantRows drops accounting rows whose variant files are -// missing, so the database never references deleted content. +// missing, so the database never references deleted content. A file whose +// size cannot be read for another reason may still be on disk, so its row +// stays, with a warning naming its key and the error. func (c *Cache) reconcileVariantRows(ctx context.Context) error { var after imagerequest.VariantKey @@ -641,6 +643,13 @@ func (c *Cache) reconcileVariantRows(ctx context.Context) error { continue } + if !errors.Is(err, ErrNotFound) { + c.log.Warn("kept accounting row for unreadable variant file", + "cache_key", key, "error", err) + + continue + } + _, err = c.db.ExecContext(ctx, `DELETE FROM variant_content WHERE cache_key = ?`, string(key)) if err != nil { From 26b5160aa80727e88f255e6db508a9719438ef89 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 27 Oct 2026 00:14:45 +0000 Subject: [PATCH 116/131] Test that reconciliation goes on past entries it cannot read Each storage directory gets a directory the pass cannot list and a file whose info it cannot read. Their paths are longer than Linux accepts, even for root. The test wants a warning naming each path and the error. It also wants the rest of the pass done: a variant file with no row adopted, the row of a missing variant file dropped, a source file with no row removed, and a wrong usage total corrected. A second test puts a temp file whose info cannot be read in each storage directory and wants the same warning. On next the first test's pass stops at the first such directory and returns its error. The second test gets no warning. Model: opus-5-5 --- internal/imgcache/eviction_internal_test.go | 201 ++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index ed7b4eb..fe64af1 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -9,8 +9,10 @@ import ( "log/slog" "os" "path/filepath" + "slices" "strings" "sync" + "syscall" "testing" "time" @@ -785,6 +787,205 @@ func TestReconcileVariantRowsDropsOnlyRowsOfMissingFiles(t *testing.T) { } } +// openDirectoryNearPathLimit makes a chain of directories inside dir, each +// named with 255 zeros, the longest name Linux allows. Linux refuses a path +// of 4096 bytes or more, and the chain goes as deep as it can below that. +// It returns the chain's deepest directory as an os.Root, closed when the +// test ends. An entry made through it with a 255-byte name has a path even +// the root user cannot read. The os.Root can make that entry because it +// names the entry only by its name, not by the whole path. +func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root { + t.Helper() + + name := strings.Repeat("0", 255) + deepest := dir + + for len(filepath.Join(deepest, name)) < 4096 { + deepest = filepath.Join(deepest, name) + } + + err := os.MkdirAll(deepest, StorageDirPerm) + if err != nil { + t.Fatalf("failed to make a chain of directories: %v", err) + } + + root, err := os.OpenRoot(deepest) + if err != nil { + t.Fatalf("os.OpenRoot(%s) error = %v", deepest, err) + } + + t.Cleanup(func() { _ = root.Close() }) + + return root +} + +// makeEntriesTooLongToRead makes a new directory and a new file in the +// deepest directory of the chain that openDirectoryNearPathLimit makes +// inside dir. Even the root user can then neither list the new directory +// nor read the new file's info. It returns the paths of the new directory +// and the new file. +func makeEntriesTooLongToRead(t *testing.T, dir string) []string { + t.Helper() + + deepest := openDirectoryNearPathLimit(t, dir) + dirName := strings.Repeat("0", 255) + + err := deepest.Mkdir(dirName, StorageDirPerm) + if err != nil { + t.Fatalf("failed to make a directory too deep to list: %v", err) + } + + fileName := strings.Repeat("1", 255) + + err = deepest.WriteFile(fileName, []byte("unreadable"), StorageFilePerm) + if err != nil { + t.Fatalf("failed to make a file too deep to read: %v", err) + } + + return []string{ + filepath.Join(deepest.Name(), dirName), + filepath.Join(deepest.Name(), fileName), + } +} + +// makeTempFileTooLongToRead makes a temp file in the deepest directory of +// the chain that openDirectoryNearPathLimit makes inside dir. Even the +// root user then cannot read the temp file's info. It returns the temp +// file's path. +func makeTempFileTooLongToRead(t *testing.T, dir string) string { + t.Helper() + + deepest := openDirectoryNearPathLimit(t, dir) + name := tempFilePrefix + strings.Repeat("1", 255-len(tempFilePrefix)) + + err := deepest.WriteFile(name, []byte("partial"), StorageFilePerm) + if err != nil { + t.Fatalf("failed to make a temp file too deep to read: %v", err) + } + + return filepath.Join(deepest.Name(), name) +} + +// assertSkippedEntryWarning checks that logs hold the warning "skipped +// unreadable entry in cache directory" naming path, with the error +// ENAMETOOLONG for that path. +func assertSkippedEntryWarning(t *testing.T, logs, path string) { + t.Helper() + + for line := range strings.Lines(logs) { + if strings.Contains(line, `"level":"WARN"`) && + strings.Contains(line, `"msg":"skipped unreadable entry in cache directory"`) && + strings.Contains(line, `"path":"`+path+`"`) && + strings.Contains(line, path+": "+syscall.ENAMETOOLONG.Error()) { + return + } + } + + t.Errorf("no warning %q names %s with the error %q", + "skipped unreadable entry in cache directory", path, syscall.ENAMETOOLONG) +} + +// TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage +// directory, a directory the pass cannot list and a file whose info it +// cannot read. The pass walks each directory in name order, so it reaches +// them before the cache's files, whose keys start with a letter. It must +// log a warning naming each one's path and the error, and still do the +// rest of its work: +// - adopt a variant file that has no row +// - drop the row of a variant whose file is missing +// - remove a source file that has no row +// - correct a wrong total cache usage +func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + ctx := t.Context() + + untrackedVariant := []byte("variant with no row") + + err := cache.variants.Write(string(testVariantKeyOne), untrackedVariant) + if err != nil { + t.Fatalf("failed to write a variant file with no row: %v", err) + } + + storeEvictionTestVariant(t, cache, testVariantKeyTwo, []byte("variant two")) + + err = os.Remove(cache.variants.keyToPath(string(testVariantKeyTwo))) + if err != nil { + t.Fatalf("failed to remove the second variant's file: %v", err) + } + + untrackedSourceKey := "ffeeddcc0001" + + err = cache.srcContent.Write(untrackedSourceKey, []byte("source with no row")) + if err != nil { + t.Fatalf("failed to write a source file with no row: %v", err) + } + + _, err = cache.db.ExecContext(ctx, + `UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`) + if err != nil { + t.Fatalf("failed to set a wrong total: %v", err) + } + + unreadable := slices.Concat( + makeEntriesTooLongToRead(t, cache.variants.baseDir), + makeEntriesTooLongToRead(t, cache.srcContent.baseDir), + ) + + logs := reconcileAsAtStartup(t, cache) + + for _, path := range unreadable { + assertSkippedEntryWarning(t, logs, path) + } + + rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?` + + if n := countRows(t, cache, rowQuery, string(testVariantKeyOne)); n != 1 { + t.Errorf("the variant file with no row got %d rows, want 1", n) + } + + if n := countRows(t, cache, rowQuery, string(testVariantKeyTwo)); n != 0 { + t.Errorf("the variant with no file kept %d rows, want 0", n) + } + + _, err = os.Stat(cache.srcContent.keyToPath(untrackedSourceKey)) + if err == nil { + t.Error("the source file with no row is still on disk") + } + + usage, err := cache.UsageBytes(ctx) + if err != nil { + t.Fatalf("UsageBytes() error = %v", err) + } + + if usage != int64(len(untrackedVariant)) { + t.Errorf("UsageBytes() = %d, want %d, the size of the adopted variant", + usage, len(untrackedVariant)) + } +} + +// TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead puts, in each +// storage directory, a temp file whose info the pass cannot read. The +// pass must log a warning naming each one's path and the error, as it +// does for any other file whose info it cannot read. +func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + unreadable := []string{ + makeTempFileTooLongToRead(t, cache.variants.baseDir), + makeTempFileTooLongToRead(t, cache.srcContent.baseDir), + } + + logs := reconcileAsAtStartup(t, cache) + + for _, path := range unreadable { + assertSkippedEntryWarning(t, logs, path) + } +} + // TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves // reconciliation is not a one-shot startup-only pass: it must also run // on the periodic ticker, so a variant file that lands on disk with no From 2c90144e7b4df7ca59e41a5505740b5df7de75c7 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 27 Oct 2026 00:16:54 +0000 Subject: [PATCH 117/131] Go on past cache entries the reconciliation pass cannot read (closes #343) The walk of each storage directory returned the error of a directory it could not list, or of a source or variant file whose info it could not read. The pass then stopped before its later checks, every time it ran. Now reconcileFiles skips a directory it cannot list and a file whose info it cannot read, temp files included. It logs a warning naming the path and the error, and the walk goes on. It reads each file's info once and hands it to sweepStaleTempFile, reconcileVariantFile or reconcileSourceFile. A file renamed or removed after its directory was listed is skipped with no warning. A cancelled context still stops the walk with its error. README.md says so under Storage. Model: opus-5-5 --- README.md | 5 +++ TODO.md | 11 ++++++ internal/imgcache/eviction.go | 73 +++++++++++++++++++---------------- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 49cafcb..c6593a2 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,11 @@ Each look also drops the rows of variants whose files are missing. When pixa cannot read a variant file's size for another reason, it keeps the variant's row. It logs a warning naming the variant key and the error. +During a look, pixa skips a directory it cannot list and a file whose size it +cannot read. It logs a warning naming the path and the error, and goes on with +the rest of the look. A file renamed or removed after pixa listed its directory +gets no warning. + pixa's target is 1-5k r/s, which has not been measured at that rate (see Load Test). Toward it, a cache hit opens the variant's file and nothing else, and makes no database call. The content type comes from the file's first bytes, diff --git a/TODO.md b/TODO.md index 4b6920f..8416fe5 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,17 @@ P2: security: per-origin rate limiting # Completed Steps +- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file + whose info it cannot read, and goes on (closes #343). It stopped before its + later checks at a directory it could not list, or at a source or variant file + whose info it could not read. + - It logs the warning "skipped unreadable entry in cache directory", with + the path and the error. + - A file renamed or removed after its directory was listed gets no warning. + - `reconcileFiles` reads each file's info once, before the temp-file check, + and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or + `reconcileSourceFile`. + - A cancelled context still stops the walk with its error. - 2026-10-26 the reconciliation pass drops a variant's accounting row only when its file is missing (closes #340). Any other error reading the file's size keeps the row. It dropped the row on any error and logged the file as missing. diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 2312a14..6746411 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -515,12 +515,15 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error { } // reconcileFiles walks baseDir, one of the cache's storage directories, -// sweeping stale temp files and passing each other file, with its name, to -// reconcileFile. Once ctx is cancelled it stops at the next file and -// returns ctx's error. +// sweeping stale temp files and passing each other file, with its info and +// name, to reconcileFile. It skips a directory it cannot list and a file +// whose info it cannot read, so one such entry does not stop the pass. It +// logs a warning naming the path and the error. A file renamed or removed +// after its directory was listed is skipped with no warning. Once ctx is +// cancelled it stops at the next file and returns ctx's error. func (c *Cache) reconcileFiles( ctx context.Context, baseDir string, - reconcileFile func(path string, entry fs.DirEntry, name string) error, + reconcileFile func(path string, info fs.FileInfo, name string) error, ) error { return filepath.WalkDir( baseDir, @@ -529,19 +532,38 @@ func (c *Cache) reconcileFiles( return ctx.Err() } - if err != nil || entry.IsDir() { - return err + if err != nil { + c.log.Warn("skipped unreadable entry in cache directory", + "path", path, "error", err) + + return nil + } + + if entry.IsDir() { + return nil + } + + info, err := entry.Info() + if errors.Is(err, fs.ErrNotExist) { + return nil + } + + if err != nil { + c.log.Warn("skipped unreadable entry in cache directory", + "path", path, "error", err) + + return nil } name := entry.Name() if strings.HasPrefix(name, tempFilePrefix) { - c.sweepStaleTempFile(path, entry) + c.sweepStaleTempFile(path, info) return nil } - return reconcileFile(path, entry, name) + return reconcileFile(path, info, name) }, ) } @@ -551,8 +573,8 @@ func (c *Cache) reconcileFiles( // and sweeping stale temp files. func (c *Cache) reconcileVariantFiles(ctx context.Context) error { return c.reconcileFiles(ctx, c.variants.baseDir, - func(path string, entry fs.DirEntry, name string) error { - return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name)) + func(path string, info fs.FileInfo, name string) error { + return c.reconcileVariantFile(ctx, path, info, imagerequest.VariantKey(name)) }) } @@ -562,16 +584,11 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error { // crash or a restored snapshot can leave it, is removed with its row and // a warning naming it, so the variant is converted again. func (c *Cache) reconcileVariantFile( - ctx context.Context, path string, entry fs.DirEntry, cacheKey imagerequest.VariantKey, + ctx context.Context, path string, info fs.FileInfo, cacheKey imagerequest.VariantKey, ) error { - info, err := entry.Info() - if err != nil { - return fmt.Errorf("failed to stat variant file: %w", err) - } - var recordedSize int64 - err = c.db.QueryRowContext(ctx, + err := c.db.QueryRowContext(ctx, `SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey), ).Scan(&recordedSize) @@ -714,8 +731,8 @@ func queryStringColumn[T ~string]( // and sweeping stale temp files. func (c *Cache) reconcileSourceFiles(ctx context.Context) error { return c.reconcileFiles(ctx, c.srcContent.baseDir, - func(path string, entry fs.DirEntry, name string) error { - return c.reconcileSourceFile(ctx, path, entry, imagerequest.SourceKey(name)) + func(path string, info fs.FileInfo, name string) error { + return c.reconcileSourceFile(ctx, path, info, imagerequest.SourceKey(name)) }) } @@ -726,16 +743,11 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error { // is removed with its row and a warning naming it, so the source is // fetched again. func (c *Cache) reconcileSourceFile( - ctx context.Context, path string, entry fs.DirEntry, key imagerequest.SourceKey, + ctx context.Context, path string, info fs.FileInfo, key imagerequest.SourceKey, ) error { - info, err := entry.Info() - if err != nil { - return fmt.Errorf("failed to stat source content file: %w", err) - } - var recordedSize int64 - err = c.db.QueryRowContext(ctx, + err := c.db.QueryRowContext(ctx, `SELECT size_bytes FROM source_content WHERE source_key = ?`, string(key), ).Scan(&recordedSize) @@ -900,17 +912,12 @@ func (c *Cache) correctUsageTotal( // sweepStaleTempFile removes a temp file left behind by a crashed // write once it is old enough that no in-flight store can own it. -func (c *Cache) sweepStaleTempFile(path string, entry fs.DirEntry) { - info, err := entry.Info() - if err != nil { - return - } - +func (c *Cache) sweepStaleTempFile(path string, info fs.FileInfo) { if time.Since(info.ModTime()) < staleTempFileAge { return } - err = os.Remove(path) + err := os.Remove(path) if err != nil && !os.IsNotExist(err) { c.log.Warn("failed to remove stale temp file", "path", path, "error", err) From 9f1e9f52530bb242672a35169a1757c09027a254 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 27 Oct 2026 05:06:36 +0100 Subject: [PATCH 118/131] Rewrite AGENTS.md in short plain sentences (#327) Every rule keeps its meaning and emphasis. Sentences are split so each holds one idea, with the main rule before its exception and every command, file name and identifier in backticks. Three statements were false against the repo or the standing rulings on record and are corrected: - the format check is `make fmt-check`, not `make check-fmt` - a feature branch reaches `next` through a PR that passes review, is merged only when the feature is completely finished and the code is up to standards, and is deleted after the merge; only the user merges `next` into `main` - the naming example uses the code's term: `al` is bad, `SourceHostAllowlist` is good The `Last Updated` line takes today's date. Model: fable-5-1 --- AGENTS.md | 103 +++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3933bc0..d50c321 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,75 +1,76 @@ # Repository Rules -Last Updated 2026-01-08 +Last Updated 2026-10-27 -These rules MUST be followed at all times, it is very important. +These rules MUST be followed at all times. This is very important. -- Never use `git add -A` - add specific changes to a deliberate commit. A commit - should contain one change. After each change, make a commit with a good - one-line summary. +- Never use `git add -A`. Stage specific changes for a deliberate commit. A + commit holds one change. After each change, make a commit with a good one-line + summary. - NEVER modify the linter config without asking first. -- NEVER modify tests to exclude special cases or otherwise get them to pass - without asking first. In almost all cases, the code should be changed, NOT the - tests. If you think the test needs to be changed, make your case for that and - ask for permission to proceed, then stop. You need explicit user approval to - modify existing tests. A bug fix may change an existing test without asking - when that test pins exactly the behavior the filed bug calls wrong, and the - change moves it to the fixed behavior and nothing else; any other change to an - existing test still needs explicit user approval. (You do not need user - approval for writing NEW tests.) +- NEVER modify a test to exclude special cases or otherwise make it pass without + asking first. In almost all cases, change the code, NOT the test. You need + explicit user approval to modify an existing test. If you think a test needs + to change, make your case and ask for permission to proceed. Then stop. A bug + fix may change an existing test without asking only when that test pins + exactly the behavior the filed bug calls wrong. The change must then move the + test to the fixed behavior and nothing else. Any other change to an existing + test needs explicit user approval. Writing NEW tests needs no approval. -- When linting, assume the linter config is CORRECT, and that each item output - by the linter is something that legitimately needs fixing in the code. +- When linting, assume the linter config is CORRECT. Treat each item the linter + reports as something that legitimately needs fixing in the code. - When running tests, use `make test`. -- Before commits, run `make check`. This runs `make lint` and `make test` and - `make check-fmt`. Any issues discovered MUST be resolved before committing - unless explicitly told otherwise. +- Before committing, run `make check`. It runs `make test`, `make lint` and + `make fmt-check`. Any issue it finds MUST be resolved before committing, + unless you are explicitly told otherwise. - When fixing a bug, write a failing test for the bug FIRST. Add appropriate - logging to the test to ensure it is written correctly. Commit that. Then go - about fixing the bug until the test passes (without modifying the test - further). Then commit that. + logging to the test, to make sure the test is written correctly. Commit that. + Then work on the fix until the test passes, without changing the test further. + Then commit the fix. -- When adding a new feature, do the same - implement a test first (TDD). It - doesn't have to be super complex. Commit the test, then commit the feature. +- When adding a new feature, do the same: write a test first (test-driven + development, TDD). The test does not need to be very complex. Commit the test, + then commit the feature. -- When adding a new feature, use a feature branch. When the feature is - completely finished and the code is up to standards (passes `make check`) then - and only then can the feature branch be merged into `main` and the branch - deleted. +- When adding a new feature, use a feature branch. The feature branch reaches + `next` through a PR that passes review. It is merged only when the feature is + completely finished and the code is up to standards. The code is up to + standards when it passes `make check`. The branch is deleted after the merge. + Only the user merges `next` into `main`. -- Write godoc documentation comments for all exported types and functions as you - go along. +- Write `godoc` comments for all exported types and functions as you go along. -- ALWAYS be consistent in naming. If you name something one thing in one place, - name it the EXACT SAME THING in another place. +- ALWAYS be consistent in naming. If you name something one way in one place, + use the EXACT SAME name in every other place. -- Be descriptive and specific in naming. `wl` is bad; `SourceHostWhitelist` is +- Be descriptive and specific in naming. `al` is bad; `SourceHostAllowlist` is good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good. -- This is not prototype or teaching code - this is designed for production. Any - security issues (such as denial of service) or other web vulnerabilities are - P1 bugs and must be added to TODO.md at the top. +- This is not prototype or teaching code. It is designed for production. Any + security issue (such as denial of service) or other web vulnerability is a P1 + bug. It must be added at the top of `TODO.md`. -- As this is production code, no stubbing of implementations unless specifically - instructed. We need working implementations. +- Do not stub an implementation unless you are specifically instructed to. This + is production code, and we need working implementations. - NEVER silently fall back to a different setting when a user's parameter - explicitly specifies a value. If a user requests format=webp and WebP encoding - is not supported, return an error - do NOT silently output PNG instead. If a - user specifies fit=invalid and that fit mode doesn't exist, return an error - - do NOT silently default to "cover". Silent fallbacks violate the principle of - least surprise and mask bugs. The only acceptable defaults are for OMITTED - parameters, never for INVALID explicit values. + explicitly specifies a value. For example, if a user requests `format=webp` + and WebP encoding is not supported, return an error. Do NOT silently output + PNG instead. If a user specifies `fit=invalid` and that fit mode does not + exist, return an error. Do NOT silently default to `cover`. A silent fallback + violates the principle of least surprise and masks bugs. Defaults are + acceptable only for OMITTED parameters, never for INVALID explicit values. -- Avoid vendoring deps unless specifically instructed to. NEVER commit the - vendor directory, NEVER commit compiled binaries. If these directories or - files exist, add them to .gitignore (and commit the .gitignore) if they are - not already in there. Keep the entire git repository (with history) small - - under 20MiB, unless you specifically must commit larger files (e.g. test - fixture example media files). Only OUR source code and immediately supporting - files (such as test examples) goes into the repo/history. +- Avoid vendoring dependencies unless specifically instructed to. NEVER commit + the `vendor/` directory. NEVER commit compiled binaries. If such a directory + or file exists and `.gitignore` does not list it, add it to `.gitignore`. + Commit that `.gitignore` change. Keep the entire git repository, history + included, under 20 MiB. Go over 20 MiB only when you specifically must commit + a large file, such as a media file used as a test fixture. Only OUR source + code and its immediately supporting files, such as test examples, go into the + repository and its history. From b3b8408d1896fb272eb7a29abe2732f4a0f4517d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 11:37:27 +0100 Subject: [PATCH 119/131] Rewrite README.md sentence by sentence (closes #327) (#345) --- README.md | 1796 +++++++++++++++++++++++++++++------------------------ 1 file changed, 977 insertions(+), 819 deletions(-) diff --git a/README.md b/README.md index c6593a2..c72daae 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ # pixa pixa is a GPL-3.0-licensed Go web server by [@sneak](https://sneak.berlin) that -proxies images from upstream sources, optionally resizing or transforming them, -and serves the results. Both source and transformed images are cached to disk so -that subsequent requests are served without origin fetches or additional -processing. +fetches images from other servers, resizes and converts them, and serves the +results. + +A web page asks pixa for an image at a size and in a format. pixa fetches the +original image, makes the requested image from it, and serves that. It keeps +both on disk. Later requests for the same image are served from disk, without a +fetch or a conversion. + +In this document, the upstream host is the server that holds an original image. ## Getting Started @@ -14,167 +19,200 @@ git clone https://git.eeqj.de/sneak/pixa.git cd pixa make build -# run with a config file: copy the example and set a real signing key -# (the example placeholder is refused at startup), e.g. with -# openssl rand -base64 32 +# make a config file from the example and put a real signing key in it cp configs/config.example.yml config.yml -$EDITOR config.yml # replace the signing_key placeholder +openssl rand -base64 32 # prints a new signing key +$EDITOR config.yml # replace the signing_key placeholder with it ./bin/pixad --config config.yml -# or build and run via Docker +# or build and run the Docker image make docker docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest ``` -The example config allowlists no host (`allowlist_hosts`), so pixa serves an -image only for a signed or encrypted URL. Every image on an allowlisted host is -served to anyone without a signature, at any size and in any format, so +`make build` compiles pixa on your machine into `bin/pixad`. `pixad` is the +server program. Compiling needs the libraries `script/bootstrap --cgo` installs +(see Deployment). + +The signing key is pixa's one required secret. It must be 32 characters or +longer. The placeholder in the example file is refused at startup. + +Once pixa runs, it serves images through URLs of this form: + +``` +/v1/image///. +``` + +For example, this URL serves `https://cdn.example.com/photos/cat.jpg` at 800x600 +as WebP: + +``` +/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp +``` + +A URL for a host in `allowlist_hosts` needs nothing more. A URL for any other +host needs a signature (see Signed URLs). Someone who knows the signing key can +also make an encrypted URL on pixa's own web page (see Encrypted URLs). An +encrypted URL works for any host and hides the original image's URL. + +The example config allowlists no host. Every image on an allowlisted host is +served to anyone, without a signature, at any size and in any format. So allowlist only hosts you control, where nobody else can put images. -A container takes its settings from environment variables (see Configuration -below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the -container exits at startup naming the variable. Everything else has a built-in -default. A config file mounted at `/etc/pixa/config.yml` is optional: it is read -when present, and an environment variable wins over the same setting in it. +A container takes its settings from environment variables (see Configuration). +Only `PIXA_SIGNING_KEY` is required. Without it, the container exits at startup +and names the variable. Every other setting has a built-in default. A config +file mounted at `/etc/pixa/config.yml` is read when present. An environment +variable wins over the same setting in the file. ## Deployment -pixa listens on plain HTTP and runs behind a reverse proxy that terminates TLS. -[`configs/Caddyfile`](configs/Caddyfile) is an example for Caddy, chosen because -it is the smallest correct one: Caddy gets the TLS certificate itself and does -everything in this list without further settings. The reverse proxy must: +pixa listens on plain HTTP. It runs behind a reverse proxy: a server in front of +it that receives the public requests and passes them on. The reverse proxy +terminates TLS, the encryption behind `https`. -- terminate TLS, as the login and generator pages work only over HTTPS (see - Routes); -- pass the `Host`, `Origin` and `Referer` headers on unchanged, as pixa refuses - a form from those pages unless `Origin` or `Referer` names the host in `Host`, - builds encrypted URLs from `Host`, and checks `Referer` against - `referer_blocklist`; -- set `X-Forwarded-For` to the client's address, with `trusted_proxies` set to - the address pixa sees the proxy's requests come from, so the login limit and - `image_requests_per_minute` count each client by its own address (see - `trusted_proxies` under Configuration); -- wait for pixa's answer for at least `downstream_timeout` (default `60s`), the - longest pixa takes to fetch, convert and send an image. +The reverse proxy must: -It may also refuse `/metrics`, as the example does, so that only a scraper that -reaches pixa directly can read it; pixa itself asks for the metrics username and -password there. +- terminate TLS. The login and generator pages work only over HTTPS (see + Routes). +- pass the `Host`, `Origin` and `Referer` headers on unchanged. pixa refuses a + form from its pages unless `Origin` or `Referer` names the host in `Host`. It + builds encrypted URLs from `Host`. It checks `Referer` against + `referer_blocklist`. +- set `X-Forwarded-For` to the client's address. `trusted_proxies` must hold the + proxy's address as pixa sees it. The login limit and + `image_requests_per_minute` then count each client by its own address. +- wait for pixa's answer for at least `downstream_timeout` (default `60s`). That + is the longest pixa takes to fetch, convert and send an image. -pixa does the rest itself: it checks signatures and encrypted URLs, applies the -allowlist, refuses upstream hosts with private or local addresses, limits login -attempts, each client's image requests, upstream response size and image -dimensions, and sends the security headers, `Strict-Transport-Security` -included, with every response. +[`configs/Caddyfile`](configs/Caddyfile) is an example for Caddy, a web server. +Caddy does everything in this list without further settings, and gets its own +TLS certificate. + +The reverse proxy may also refuse `/metrics`, as the example does. Only a +scraper that reaches pixa directly can then read it. pixa itself asks for +`metrics.username` and `metrics.password` there. + +pixa does the rest itself: + +- it checks signatures and encrypted URLs; +- it applies the allowlist; +- it refuses upstream hosts with private or local addresses; +- it limits login attempts and each client's image requests; +- it limits the size of upstream responses and the dimensions of images; +- it sends the security headers, `Strict-Transport-Security` included, with + every response. The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the database and the disk cache: -- It needs a persistent volume: without one, every restart starts with an empty - cache. Losing the directory also ends every login session (see Encrypted - URLs). In the container, the startup script gives the directory to the user - pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user - must be able to write the directory. -- `cache_max_bytes` limits the source and transformed images together. The - database and files still being written come on top, and eviction runs in the - background, so the cache can pass the limit for a while: leave room on the - volume beyond it. -- Set `cache_max_bytes` for a lasting deployment. Its default, worked out each - time pixa starts, is 75% of the sum of the space free on the volume and the - space the cached images already take, so a restart keeps the limit the cache - had, but anything else that fills or frees space on the volume moves it. +- It needs a persistent volume. Without one, each new container starts with an + empty cache, as after every redeploy. Losing the directory also ends every + login session (see Encrypted URLs). +- In the container, the startup script gives the directory to the user pixa runs + as, uid 65532, and sets its mode to `750`. Outside the container, the user + running pixa must be able to write it. +- `cache_max_bytes` limits the cached images. The limit does not count the + database, or files still being written. Eviction runs in the background, so + the cache can pass the limit for a while. Leave room on the volume beyond it. +- Set `cache_max_bytes` for a lasting deployment. Its default is worked out at + each start from the free space on the volume (see Configuration). Anything + else that fills or frees space on the volume moves that default. -A load balancer's health check can request `/.well-known/healthcheck.json`, -which answers 200 whenever pixa is running, in maintenance mode too (see +A load balancer's health check can request `/.well-known/healthcheck.json`. It +answers 200 whenever pixa is running, in maintenance mode too (see `maintenance_mode`). -On SIGTERM or SIGINT pixa stops accepting connections. The requests in progress -and the images being processed get 5 seconds to finish. A request not finished -by then is cut off. pixa then writes the waiting last-used times of cached -images to the database (see Storage) and exits. It exits with 1 when: +On `SIGTERM` or `SIGINT`, pixa stops accepting connections. The requests in +progress and the images being processed get 5 seconds to finish. A request not +finished by then is cut off. pixa then saves the last-used times it has not yet +written (see Storage), and exits. It exits with status 1 when: -- images were still being processed after those 5 seconds, -- some of those times could not be written, or +- images were still being processed after those 5 seconds; +- some of those times could not be saved; - another part of pixa failed to stop. Otherwise it exits with 0. `docker stop` waits 10 seconds before it kills the container. -Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as -it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL -support, which on Alpine is the `vips-jxl` package and which the nix and brew -packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04 -on. Building pixa also needs the development files of libvips and libheif, -`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these; -the `Dockerfile` installs its own packages. Plain `script/bootstrap`, which -`script/setup` and `script/cibuild` run, installs git, make and Go, and Node, -Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but -none of the C libraries: the checks compile pixa in Docker. Docker itself must -already be installed. +Outside Docker, pixa needs these at run time: + +- libvips, the image library pixa converts images with, including its JPEG XL + support. pixa does not start without that support. On Alpine it is the + `vips-jxl` package. The Nix and Homebrew packages of libvips include it. So do + the Debian and Ubuntu packages, from Debian 12 and Ubuntu 24.04 on. +- libheif, which libvips needs for AVIF. +- the system's CA certificates, which HTTPS fetches need. + +Compiling pixa needs the development files of libvips and libheif, `pkg-config` +and a C compiler. `script/bootstrap --cgo` installs them, and libvips and +libheif with them. The `Dockerfile` installs its own packages. ## Running under upaas -What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs: +[upaas](https://git.eeqj.de/sneak/upaas) is the deployment tool pixa is meant to +run under. Its app for pixa needs: - **Port:** pixa listens on container port `8080`. - **Volume:** container path `/var/lib/pixa`, where pixa keeps its database and - cache. Creating the host directory when it is missing is upaas's job, tracked - in https://git.eeqj.de/sneak/upaas/issues/235. + cache. - **Environment variables:** - - `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and - login, 32+ characters, for example from `openssl rand -base64 32` + - `PIXA_SIGNING_KEY` (required): the secret for signed URLs, encrypted URLs + and login. It is 32 characters or more, for example from + `openssl rand -base64 32`. - `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature, comma-separated; default none. Every image on a listed host is served to - anyone without a signature, so list only hosts you control, where nobody - else can put images. - - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it. Set - it: the volume is a directory on the host's filesystem, shared with the - host and every other app. The default is 75% of the sum of that - filesystem's free space and what the cache holds, and at least 500 MiB, so - the cache can grow to three quarters or more of the space they all have - left, and to all of it when that is 500 MiB or less. - - the rest are in the list under Configuration below + anyone, so list only hosts you control, where nobody else can put images. + - `PIXA_CACHE_MAX_BYTES`: the disk cache limit in bytes; `0` turns the disk + cache off. Set it. The volume is a directory on the host's filesystem. The + host and every other app use that filesystem too. The default is 75% of + the sum of that filesystem's free space and what the cache already holds. + So by default the cache may grow to three quarters of the space the host + and the other apps have left. The default is also at least 500 MiB. So + when that space is 500 MiB or less, the cache may take all of it. + - the rest are in the list under Configuration. - **Memory:** pixa can use up to `max_concurrent_processing` times 4.2 GiB for - the images it processes at once, plus `upstream_connections` times - `upstream_max_response_size` for the source images it has fetched that wait - for processing. 4.2 GiB is the most resident memory was measured to grow while - pixa, with pixad's libvips settings, processed one 8192x8192 source, the - largest it accepts, saving it at full size to each output format in turn, one - at a time. The sources were a JPEG, a GIF, a WebP, an 8-bit and a 16-bit PNG, - an 8-bit and a 12-bit AVIF, and an 8-bit, a 16-bit and a 32-bit floating-point - JPEG XL; memory grew most for the 16-bit JPEG XL, of random pixels, saved as - WebP. At the defaults that is half the CPUs pixa sees times 4.2 GiB, plus 64 - times 50 MiB: on a host where pixa sees 48 CPUs, 24 times 4.2 GiB plus 3.1 - GiB, about 104 GiB. Set `PIXA_MAX_CONCURRENT_PROCESSING` and - `PIXA_UPSTREAM_CONNECTIONS` so that this fits the memory the container may - use. -- **Temporary files:** while pixa processes a large source, libvips writes its - decoded pixels to a file in `/tmp`. The file disappears from `/tmp` once it is - written, so a listing of `/tmp` does not show it, but it keeps its space, on - disk or, where `/tmp` is a tmpfs, in memory, until the image is done. For a - GIF, WebP, AVIF or JPEG XL source the decoded pixels are held in memory as - well as written to that file, and the 4.2 GiB above includes that memory. For - an 8192x8192 source the file was measured on pixad at 192 MiB for an 8-bit RGB - JPEG, 512 MiB for a 16-bit RGBA PNG or JPEG XL, and 1 GiB for a JPEG XL of - 32-bit floating-point RGBA, the largest measured: up to 1 GiB per image - processed at once, 24 GiB at the defaults on a host where pixa sees 48 CPUs. - This `/tmp` is in the container's own filesystem, not on the volume at - `/var/lib/pixa`, and `cache_max_bytes` does not count it. The 4.2 GiB above - does not include these files; where `/tmp` is a tmpfs, they are memory on top - of it. + the images it processes at once. On top of that, the fetched source images + waiting for processing can take `upstream_connections` times + `upstream_max_response_size`. 4.2 GiB is a measured value. It is the largest + rise in resident memory measured while pixa saved one 8192x8192 source, the + largest it accepts, to each output format at full size. The rise was largest + for a 16-bit JPEG XL of random pixels saved as WebP. At the defaults the total + is half the CPUs pixa sees times 4.2 GiB, plus 64 times 50 MiB. On a host + where pixa sees 48 CPUs, that is 24 times 4.2 GiB plus 3.1 GiB, about 104 GiB. + Set `PIXA_MAX_CONCURRENT_PROCESSING` and `PIXA_UPSTREAM_CONNECTIONS` so that + this fits the memory the container may use. +- **Temporary files:** while pixa processes a large source, libvips writes the + decoded pixels to a file in `/tmp`. The file is removed from the directory + once it is written, so a listing of `/tmp` does not show it. It keeps its + space until the image is done: on disk, or in memory where `/tmp` is a tmpfs. + For a GIF, WebP, AVIF or JPEG XL source the decoded pixels are held in memory + as well, and the 4.2 GiB above includes that memory. For an 8192x8192 source + the file was measured at: + - 192 MiB for an 8-bit RGB JPEG; + - 512 MiB for a 16-bit RGBA PNG or JPEG XL; + - 1 GiB for a 32-bit floating-point RGBA JPEG XL. + + So count up to 1 GiB per image processed at once: 24 GiB at the defaults on + a host where pixa sees 48 CPUs. This `/tmp` is in the container's own + filesystem, not on the volume at `/var/lib/pixa`. `cache_max_bytes` does not + count these files, and the 4.2 GiB above does not include them. + - **Health check:** the image's `HEALTHCHECK` requests `/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds after a deploy and marks the deploy failed unless it is `healthy`. The probe - uses the port from `PORT` (default `8080`), so a port changed only in a - mounted config file is not seen by it: change the port with `PORT`. + takes the port from `PORT` (default `8080`). It does not see a port changed + only in a mounted config file, so change the port with `PORT`. ## Rationale -Image-heavy web applications need a fast, caching reverse proxy that can resize -and transcode images on the fly. pixa fills that role as a single, -self-contained binary with no external runtime dependencies beyond libvips. It -supports HMAC-SHA256 signed URLs with expiration to prevent abuse, and -allowlisted source hosts for open access. +A web application that shows images needs each one at several sizes and in +several formats. pixa makes them on request from one original, and keeps what it +made. The application stores one image per picture and hands out URLs. pixa is +one binary. At run time it needs only the image libraries and the CA +certificates named under Deployment. Signed URLs with an expiry keep strangers +from using it to resize anything they like. Allowlisted hosts give open access +where that is wanted. ## Design @@ -183,30 +221,30 @@ allowlisted source hosts for open access. These terms are the names the code uses: - **Source**: the image at one upstream URL, named by its host, path and query. -- **Source key**: the SHA-256 of a source's host, path and query, which names - the source's file and its row in the database. -- **Transform**: the output size, format, quality and fit an image request asks - for; with the format `auto`, pixa chooses the output format from the `Accept` - header. -- **Image request**: a source and a transform, which each image route reads from +- **Source key**: the SHA-256 hash of a source's host, path and query. SHA-256 + is a hash function: it turns any text into a fixed-length fingerprint. The + source key names the source's file and its row in the database. +- **Transform**: the output size, format, quality and fit that an image request + asks for. With the format `auto`, pixa chooses the output format from the + `Accept` header. +- **Image request**: a source and a transform. Each image route reads one from its URL. - **Variant**: the image pixa serves for an image request, made from its source with its transform. -- **Variant key**: the SHA-256 of a variant's source, size, format, quality and - fit, which names the variant's file and its row in the database, and gives its - `ETag`. -- **Signed URL**: a `/v1/image/` URL whose `sig` signs its image request and its - expiry `exp` with `signing_key`. +- **Variant key**: the SHA-256 hash of a variant's source, size, format, quality + and fit. It names the variant's file and its row in the database, and gives + its `ETag`. +- **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image + request and its expiry `exp` with `signing_key`. - **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any expiry, encrypted with a key derived from `signing_key`. - **Login session**: an encrypted cookie that keeps a browser logged in to the - generator page for 30 days, or until a logout replaces the random value pixa - keeps in the `logout` table (see Encrypted URLs). + generator page. It lasts 30 days, or until a logout (see Encrypted URLs). - **Negative cache**: the source URLs pixa does not fetch again for 5 minutes, - after one of these failed fetches: - - the upstream answered with an error status + after a fetch failed in one of these ways: + - the upstream answered with an error status; - the body's first bytes were not those of the format its `Content-Type` - names + names. - **Image cache**: the part of pixa that keeps the sources, the variants and the negative cache (see Storage). - **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached @@ -214,95 +252,31 @@ These terms are the names the code uses: - **Image proxy**: the part of pixa that serves a variant from the image cache when it can, and otherwise makes it from the source. -### Storage - -- **Sources**: `/cache/sources///`, one file for - each source URL -- **Database**: `/state.sqlite3` (SQLite) -- **Variants**: `/cache/variants///` - -`` and `` are the first and second pairs of characters of the file's -name. - -The image cache keeps these tables in the database: - -- `source_content`: a row for each source, by source key, with its size and when - it was last used -- `variant_content`: a row for each variant, by variant key (column - `cache_key`), with its size and when it was last used -- `cache_usage`: the total size of the cached sources and variants, which - eviction reads -- `negative_cache`: a row for each source URL in the negative cache, with when - it expires - -pixa reads a cached source image's format from its file's first bytes. When they -match none of the source formats, pixa fetches the source again. Two source URLs -that serve the same bytes are stored twice, once under each URL. A source image -is stored by writing its file and then its row; when the row cannot be written, -the file is removed again. - -Each source and variant file is written under a temporary name and synced to -disk before it is renamed into place. So a crash never leaves part of a file -under its own name. When pixa starts, and every five minutes after, it removes -each source or variant whose file is empty, or not the size its row records. It -logs a warning naming the file. A later request for that image fetches the -source or makes the variant again. Looking for those files reads the database -once for every cached file, so pixa skips each five-minute look that would start -less than ten times as long after the last look ended as that look took. After a -look that took one minute, the next one starts 10 to 15 minutes after that look -ended. - -Each look also drops the rows of variants whose files are missing. When pixa -cannot read a variant file's size for another reason, it keeps the variant's -row. It logs a warning naming the variant key and the error. - -During a look, pixa skips a directory it cannot list and a file whose size it -cannot read. It logs a warning naming the path and the error, and goes on with -the rest of the look. A file renamed or removed after pixa listed its directory -gets no warning. - -pixa's target is 1-5k r/s, which has not been measured at that rate (see Load -Test). Toward it, a cache hit opens the variant's file and nothing else, and -makes no database call. The content type comes from the file's first bytes, -which are those of the format pixa wrote it in; a file whose first bytes are not -those of a format pixa writes is not served, and the image is converted again. A -hit hands the time it was served to a background writer and does not wait for -the writer. Eviction orders by that time. The hit is counted in memory. A miss -that uses a cached source image also hands that writer the time it used the -image. Once a second that writer writes the waiting times in one transaction. Up -to 10,000 access times of variants, and as many of sources, can wait to be -written. A time handed over beyond that is dropped. Its image then only looks -less recently used to eviction. `script/bench` measures how many hits pixa -serves per second (see Load Test). - ### Routes -pixa answers these routes; any other path answers 404. A path in this list asked -with a method the list does not give answers 405, except `/static/`, which -answers any method as it answers `GET`. A browser's CORS preflight request -(`OPTIONS` with `Origin` and `Access-Control-Request-Method` headers) to any -path under `/v1/` answers 200, in maintenance mode too. +pixa answers these routes. Any other path answers 404. A listed path answers 405 +to any method the list does not give for it. -- `GET /` — the login page, or the URL generator page with a login session (see - Encrypted URLs). Needs: nothing. Answers: 200. -- `POST /` — log in with the signing key typed into the login page. Needs: the - login page's form (below). Answers: 303 to `/` with a login session cookie - that lasts 30 days for the right key; 200 with the login page and an error for - a wrong key; 429 over the login limit (below). -- `POST /generate` — make an encrypted URL from the generator page's form. - Needs: a login session and the generator page's form (below); without a login - session it answers 303 to `/`. Answers: 200 with the page showing the URL; 400 - with the page naming a field that is not valid; 500 when the URL cannot be - made. -- `POST /logout` — log out with the generator page's Logout button: end every - login session, in every browser (see Encrypted URLs), and remove the session - cookie. Needs: a login session and the generator page's form (below); without - a login session it only removes the cookie. Answers: 303 to `/`; 500 when the +- `GET /` — the login page, or the generator page with a login session (see + Encrypted URLs). Needs nothing. Answers 200. +- `POST /` — log in with the signing key typed into the login page. Needs the + login page's form (below). For the right key it answers 303 to `/` and sets + the login session cookie. For a wrong key it answers 200 with the login page + and an error. Answers 429 over the login limit (see Limits). +- `POST /generate` — make an encrypted URL from the generator page's form. Needs + a login session and the generator page's form. Without a login session it + answers 303 to `/`. Answers 200 with the page showing the URL. Answers 400 + with the page naming the field it refused (see Encrypted URLs). Answers 500 + when the URL cannot be made. +- `POST /logout` — log out with the generator page's Logout button. That ends + every login session, in every browser (see Encrypted URLs), and removes the + cookie. Needs a login session and the generator page's form. Without a login + session it only removes the cookie. Answers 303 to `/`. Answers 500 when the logout cannot be saved in the database. - `GET` or `HEAD` `/v1/image///.`, or `/v1/image///` with no format — an image, fetched, resized - and converted (below). Needs: a signature, unless the host is allowlisted (see - Source Hosts). Answers: + and converted (see Image URLs). Needs a signature, unless the host is + allowlisted (see Source Hosts). Answers: - 200 with the image. - 304 when `If-None-Match` matches the image's `ETag`. - 400 for a URL or parameter that is not valid. @@ -312,34 +286,34 @@ path under `/v1/` answers 200, in maintenance mode too. - 401 for a missing or wrong signature. - 401 for a missing `exp`. - 401 for an `exp` in the past. - - 403 when the request's `Referer` names a host in `referer_blocklist`, - checked before the signature, the cache and the upstream fetch. + - 403 when the request's `Referer` names a host in `referer_blocklist`. This + is checked before the signature, the cache and the upstream fetch. - 403 when the upstream host, or a host it redirects to, is `localhost` or ends in `.localhost` or `.local`. - 403 when the upstream host, or a host it redirects to, has an address in a blocked network (see `blocked_networks`). - - 429 over `image_requests_per_minute` (below). + - 429 over `image_requests_per_minute` (see Limits). - 499, with no body, when the client goes away before its image is ready, - unless its request is the one making the image (below). + unless its request is the one making the image (see Limits). - 415 when the upstream answers with a `Content-Type` that is not one of the - source formats (below). + source formats (see Image URLs). - 415 when the upstream sends a body whose first bytes are not those of the format its `Content-Type` names. - 415 when the upstream sends an image whose first bytes libvips reads as another format. - 502 when the upstream answers with an error status. - 502 for 5 minutes after that for the same source URL, when pixa has - neither the image nor its source image in its cache. + neither the image nor its source in its cache. - 502 likewise for 5 minutes after a body whose first bytes are not those of its `Content-Type`. - - 503 when pixa is busy. - - 503 when pixa is in maintenance mode. + - 503 when pixa is busy (see Limits). + - 503 in maintenance mode. - 504 when the upstream has not sent its response headers within `upstream_fetch_timeout`. - 500 when that time runs out while the image itself is still arriving. - 500 for any other failure. - `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL - (see Encrypted URLs). Needs: nothing but the URL. Answers: + (see Encrypted URLs). Needs nothing but the URL. Answers: - 200 with the image. - 304 when `If-None-Match` matches the image's `ETag`. - 400 for a token that does not decrypt. @@ -357,395 +331,581 @@ path under `/v1/` answers 200, in maintenance mode too. - 503 - 504 - 500 -- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: - nothing. Answers: 200. +- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs + nothing. Answers 200. - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, `uptime_seconds`, `uptime_human`, `version`, `appname` and `maintenance_mode`. - Needs: nothing. Answers: 200, always. + Needs nothing. Answers 200, always. - `GET /static/` — the stylesheet and script the login and generator pages - load. Needs: nothing. Answers: 200, or 404 for a file that does not exist. -- `GET /metrics` — Prometheus metrics (see Architecture). Needs: HTTP basic - authentication with `metrics.username` and `metrics.password`. Answers: 200; - 401 without them; 404 when they are not set, as the route then does not exist. + load. Needs nothing. Answers 200, or 404 for a file that does not exist. It + answers any other method as it answers `GET`. +- `GET /metrics` — Prometheus metrics (see Architecture). Needs HTTP basic + authentication with `metrics.username` and `metrics.password`. Answers 200. + Answers 401 without them. Answers 404 when they are not set, as the route then + does not exist. -Every response carries an `X-Request-ID` header holding the request's ID, which -a client can quote when reporting a problem: the request's own `X-Request-ID`, -as a reverse proxy in front of pixa may send, when it is at most 64 letters, -digits, `-`, `_` or `.`; otherwise a random one pixa makes for the request, -which tells nothing about the machine or the other requests. pixa's log line for -the request carries the same ID as `request_id`, and so do the lines it logs -when it fetches, converts and serves an image; the fetch sends it to the -upstream host as `X-Request-ID`. +CORS is the browser rule that lets a page on one site read responses from +another. The preflight is the `OPTIONS` request the browser sends first. It +carries an `Origin` header and an `Access-Control-Request-Method` header. A +preflight to any path under `/v1/` answers 200, in maintenance mode too. -The `POST` routes accept only a form that pixa's own page served: the page puts -a token in the form and sets a cookie to match, and a request without both is -refused with 403, so another site cannot submit the form from a visitor's -browser. The login and generator pages are meant to be opened over HTTPS: while -`debug` is off, a form sent from a page opened over plain HTTP is refused with -403, and while it is on, so is one sent from a page opened over HTTPS. Plain -HTTP is for development on the browser's own machine: the login session cookie -is always marked `Secure`, and over plain HTTP a browser keeps such a cookie -only for its own machine (`localhost`), if at all. A form is also refused with -403 when the page's host is not the `Host` header pixa receives, so a reverse -proxy in front of pixa must pass that header on unchanged. A form body over 1 -MiB is refused with 413. The image routes answer the errors listed for them with -JSON holding `error`, `status` and `timestamp`. +Every response carries an `X-Request-Id` header with the request's ID. A client +can quote it when reporting a problem. A reverse proxy may send an +`X-Request-Id` header with the request. pixa keeps that ID when it is 1 to 64 +letters, digits, `-`, `_` or `.`. Otherwise pixa makes a random ID, which tells +nothing about the machine or the other requests. pixa's log line for the request +carries the same ID as `request_id`. So do the lines it logs when it fetches, +converts and serves an image. The fetch sends it to the upstream host as +`X-Request-Id`. -An image URL has one of these forms, the second with no format: +The `POST` routes accept only a form that pixa's own page served. The page puts +a token in the form and sets a cookie to match. A request without both is +refused with 403. So another site cannot submit the form from a visitor's +browser. A form is also refused with 403 when the page's host is not the `Host` +header pixa receives. So a reverse proxy in front of pixa must pass that header +on unchanged. A form body over 1 MiB is refused with 413. + +The login and generator pages are meant to be opened over HTTPS. While `debug` +is off, a form sent from a page opened over plain HTTP is refused with 403. +While `debug` is on, a form sent from a page opened over HTTPS is refused +instead. Plain HTTP is for development on the browser's own machine. The login +session cookie is always marked `Secure`. Over plain HTTP a browser keeps such a +cookie only for its own machine (`localhost`), if at all. + +The image routes answer the errors listed for them with a JSON body holding +`error`, `status` and `timestamp`. + +### Image URLs + +An image URL has one of these forms. The second has no format: ``` /v1/image///.?sig=&exp=&q=&fit= /v1/image///?sig=&exp=&q=&fit= ``` -Images are only fetched from origins using TLS with valid certificates, unless -`allow_http` is set: then pixa fetches every image over plain HTTP, which is for -testing only. - -A request whose query string cannot be decoded, or gives any parameter more than -once, is refused with 400. - -- ``: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, - `avif`, `jxl` (JPEG XL), `gif`, or `auto` (below). A URL with no format (the - second form, with no dot after the size) is served as JPEG XL, the default, as - with `jxl` -- ``: `orig` or `x` (e.g. `800x600`) +- ``: the upstream host, with a port when the source URL has one (see + Source Hosts). +- ``: the source URL's path, and its query after `%3F` (below). +- ``: `orig` or `x`, for example `800x600`. A side of `0` + is worked out from the other side and the source's proportions (below). `orig` + is `0x0`: the source's own size. +- ``: `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, `avif`, + `jxl` (JPEG XL), `gif`, or `auto` (below). `orig` is the source's own format. + A URL with no format is served as JPEG XL, the default, as with `jxl`. - `sig` and `exp`: the signature and its expiry, needed unless the host is - allowlisted (see Signature Specification) -- `q` and `fit`: the output quality and how the image is fitted to ``, - both optional (values under Signature Specification). Both are part of what is - cached, so each value of either is a separate cached image. + allowlisted (see Signed URLs). +- `q`: the output quality, a whole number from 1 to 100. It is `85` when the URL + has no `q`. +- `fit`: how the image is fitted to ``: `cover`, `contain`, `fill`, + `inside` or `outside` (below). It is `cover` when the URL has no `fit`. -A source URL's query follows its path after `%3F`, the encoded `?`: -`/v1/image//%3F/.`. pixa decodes the -percent-escapes in the image URL's path once, then reads the host up to the -first `/`, the size and format after the last `/`, and between them the source -path up to the first `?` and the source query after it. So the `?` before the -query, any `?` in the query, and every `%` in the source path or query must be -percent-encoded, as `%3F` and `%25`. Any other character may be written as -itself or percent-encoded: `/`, `;`, `,`, `=` and `&` work either way. Neither -the source path nor the source query may hold a `#`, and an image URL that -writes one as `%23` is refused with 400. For example, +`q` and `fit` are part of what is cached, so each value of either is a separate +cached image. A request is refused with 400 when its query string cannot be +decoded, or when it gives any parameter more than once. A request with a `q`, +`fit` or `exp` that is empty or not valid is refused with 400 too. + +pixa fetches sources over HTTPS only, and only from hosts with valid +certificates. With `allow_http` set, it fetches every source over plain HTTP +instead. That is for testing only. + +**The source path and query.** A source URL's query follows its path after +`%3F`, the encoded `?`: + +``` +/v1/image//%3F/. +``` + +pixa decodes the percent-escapes in the image URL's path once. It then reads the +host up to the first `/`, and the size and format after the last `/`. Between +them it reads the source path up to the first `?`, and the source query after +it. So these must be percent-encoded, as `%3F` and `%25`: + +- the `?` before the query; +- any `?` in the query; +- every `%` in the source path or query. + +Any other character may be written as itself or percent-encoded. `/`, `;`, `,`, +`=` and `&` work either way. Neither the source path nor the source query may +hold a `#`. An image URL that writes one as `%23` is refused with 400. For +example, the image `https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg` at -800x600 WebP is -`/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp`, -signed as shown under Signature Specification. +800x600 as WebP is: -The source path is the path with its percent-escapes decoded, and pixa fetches -it percent-encoded again: every character other than letters, digits, `-`, `.`, -`_`, `~`, `/`, `$`, `&`, `+`, `,`, `:`, `;`, `=` and `@` is sent as a -percent-escape. So `my%20cat.jpg` in the image URL is the source path -`/my cat.jpg`, fetched as `/my%20cat.jpg`; `my%2520cat.jpg` is the source path -`/my%20cat.jpg`, fetched as `/my%2520cat.jpg`; and `100%25.png` is fetched as -`/100%25.png`. The source query is the source URL's query as that URL writes it, -its percent-escapes included, and is sent as it is, except that the characters a -browser percent-encodes in a query (a space, `"`, `'`, `<`, `>`, control -characters and characters outside ASCII) and a `%` that does not begin a -percent-escape are percent-encoded. pixa signs and caches the query in that -form, so `img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are -the same source query, `name=my%20cat`. +``` +/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp +``` -A size with one side `0` has that side scaled to keep the source's proportions, -and is refused with 400 when that side would come out above 8192, the most -either side may be; so is a size with `fit=outside`, which scales the image to -cover the size and so past it on one side, when that side would come out -above 8192. +Signed URLs shows this URL with its signature. -An AVIF image of more than 16,777,216 pixels, the number 4096x4096 and 8192x2048 -have, is refused with 400 before anything is decoded, as its encode could take -more than 40 seconds (see below). That includes `orig` of an AVIF source, and -`auto` when it chooses AVIF, so a client whose `Accept` names `image/avif` but -not `image/jxl` is refused such a size. +pixa keeps the source path decoded and fetches it percent-encoded again. Every +character other than these is sent as a percent-escape: letters, digits, `-`, +`.`, `_`, `~`, `/`, `$`, `&`, `+`, `,`, `:`, `;`, `=` and `@`. So: -The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's -`Content-Type` must name its format, and the image's first bytes must match it. -An upstream answering with any other `Content-Type`, SVG included, is refused -with 415. So is a body whose first bytes are not those of the format its -`Content-Type` names, such as a JPEG sent as `image/png`, text sent as an image, -or an empty body. pixa then does not fetch that source URL again for 5 minutes, -as after an upstream error status, and meanwhile answers 502 for an image that -would need that fetch. An image whose first bytes libvips reads as another -format, such as an SVG whose first bytes pass the check for AVIF, is refused -with 415 too. +- `my%20cat.jpg` in the image URL is the source path `/my cat.jpg`, fetched as + `/my%20cat.jpg`; +- `my%2520cat.jpg` is the source path `/my%20cat.jpg`, fetched as + `/my%2520cat.jpg`; +- `100%25.png` is fetched as `/100%25.png`. -With the format `auto`, pixa chooses the format for each request from its -`Accept` header, in this order: +The source query is sent as the source URL writes it, its percent-escapes +included. Two kinds of character are percent-encoded first: those a browser +percent-encodes in a query, and a `%` that does not begin a percent-escape. The +first kind is a space, `"`, `'`, `<`, `>`, control characters and characters +outside ASCII. pixa signs and caches the query in that form. So +`img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are the same +source query, `name=my%20cat`. + +**Sizes.** The largest width or height is 8192, for the source and for the +output. A size with one side `0` has that side worked out from the other side +and the source's proportions. The side is rounded down, but is at least 1. When +it would come out above 8192, the request is refused with 400. The fit modes: + +- `cover`: scale the image to cover the size, keeping its proportions, and cut + out the centre. The output is exactly the size. +- `contain`: scale the image to fit inside the size, keeping its proportions. + The output can be smaller than the size on one side. +- `fill`: stretch the image to the size, without keeping its proportions. +- `inside`: as `contain`, but never enlarge the image. +- `outside`: scale the image to cover the size, keeping its proportions, without + cutting. The output can be larger than the size on one side. An 8192x1 source + at `100x0` is 100x1 by its proportions, but comes out at 8192x1. A side that + would come out above 8192 is refused with 400. + +An AVIF output of more than 16,777,216 pixels is refused with 400 before +anything is decoded. That is the number of pixels in 4096x4096 or 8192x2048. A +larger AVIF encode could take more than 40 seconds (see Limits). The limit also +applies to the format `orig` when the source is AVIF, and to `auto` when it +chooses AVIF. So a client whose `Accept` names `image/avif` but not `image/jxl` +is refused such a size. + +**Source formats.** The source may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The +upstream's `Content-Type` must name its format, and the image's first bytes must +match it. An upstream answering with any other `Content-Type`, SVG included, is +refused with 415. So is a body whose first bytes are not those of the format its +`Content-Type` names: a JPEG sent as `image/png`, text sent as an image, or an +empty body. pixa then does not fetch that source URL again for 5 minutes, as +after an upstream error status. Meanwhile it answers 502 for an image that would +need that fetch. An image whose first bytes libvips reads as another format is +refused with 415 too. An SVG whose first bytes pass the check for AVIF is one +example. + +**The format `auto`.** With `auto`, pixa chooses the format for each request +from its `Accept` header, in this order: 1. JPEG XL, when the header names `image/jxl`; 2. AVIF, when it names `image/avif`; 3. WebP, when it names `image/webp`; -4. JPEG, when the first of `image/jpeg`, `image/*` and `*/*` that it names - allows it, or when there is no `Accept` header or it is empty. +4. JPEG, when the most specific of `image/jpeg`, `image/*` and `*/*` that the + header names has a `q` above 0, or when there is no `Accept` header or it is + empty. -An entry with `q=0` refuses its format; other `q` values do not change the +An entry with `q=0` refuses its format. Other `q` values do not change the order. JPEG XL, AVIF and WebP must be named, as clients that cannot show them -also send `image/*` and `*/*`. pixa never sends a format the client refused: -when the header allows none of the four, the answer is 406, and a header that -does not parse, or has a `q` that is not a number from 0 to 1, is refused -with 400. The signature, or the token of an encrypted URL, covers `auto` itself, -so one URL serves every client. Each format chosen is cached as a separate -image, and every answer that depends on `Accept` (the image, a 304, and the 400 -and 406 above) carries `Vary: Accept`, so a shared cache keeps the formats apart -too. +also send `image/*` and `*/*`. pixa never sends a format the client refused. +When the header allows none of the four, the answer is 406. A header that does +not parse, or has a `q` that is not a number from 0 to 1, is refused with 400. +The signature, or the token of an encrypted URL, covers `auto` itself, so one +URL serves every client. Each format chosen is cached as a separate image. Every +answer that depends on `Accept` carries `Vary: Accept`: the image, a 304, and +the 400 and 406 above. So a shared cache keeps the formats apart too. -An image is served with `Cache-Control: public, max-age=, immutable`. -When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), `max-age` -is the whole seconds left until then, at most one year, so no browser or proxy -cache keeps the image after pixa would refuse the URL. A URL with no expiry gets -one year. `immutable` only stops a client revalidating while its copy is fresh. +**Response headers.** An image is served with +`Cache-Control: public, max-age=, immutable`. When the URL has an +expiry (an `exp`, or the expiry of an encrypted URL), `max-age` is the whole +seconds left until then, at most one year. So no browser or proxy cache keeps +the image after pixa would refuse the URL. A URL with no expiry gets one year. +`immutable` only stops a client revalidating while its copy is fresh. The +response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache, +and `MISS` for one made for this request. The `ETag` is the first 16 characters +of the variant key, in quotes. -When several requests for the same image, size, format, quality and fit miss the -cache at once, they share one upstream fetch (or one read of the cached source) -and one transcode: the first request does the work, and the others wait for its -image or its error, holding no upstream connection or processing slot of their -own. A waiting request stops waiting when its own client goes away. The work -goes on for the others even if the first request's client goes away. The shared -fetch sends the first request's ID upstream, and the lines logged for the fetch -and the transcode carry that ID. +### Signed URLs -A transcode whose request's `downstream_timeout` has ended (for a shared one, -the first request's) stops waiting for a processing slot, or, once it has one, -gives it back without beginning to encode. A request waiting for a shared -transcode that stops this way is answered 503 with the error -`server busy, try again later`, though its own `downstream_timeout` may not have -ended. govips gives pixa no way to stop libvips once an encode has begun, so an -encode keeps its slot until it finishes, which can be after -`downstream_timeout`. The limit of 8192 on each side of the source and of the -output, and for AVIF the limit of 16,777,216 pixels, bound how long that takes. -With one libvips thread, as pixad runs it, encoding 8192x8192 random pixels -takes up to about 1 second as JPEG, 15 as PNG and 16 as GIF, the slowest of 8 or -16 bits per sample, with or without an alpha channel. The WebP and JPEG XL -encoders run below libvips' default effort, and AVIF at the lowest effort govips -can set, so that no encode takes more than 40 seconds of the default -`downstream_timeout` of 60: at any quality and 8 or 16 bits per sample, -8192x8192 random pixels with an alpha channel take up to about 23 seconds as -WebP and 27 as JPEG XL, and 4096x4096, 8192x2048 or 2048x8192 up to about 34 as -AVIF, which at quality 85 would take about 96 at 8192x8192. Without an alpha -channel they take up to about 16, 13 and 22 seconds. Decoding such a source adds -up to about 11 seconds. +A signed URL is a `/v1/image/` URL with two query parameters: `exp`, the Unix +time (seconds since 1 January 1970) when it expires, and `sig`, its signature. +The signature is an HMAC-SHA256: a hash of the request's details, keyed with +`signing_key`, that only someone who knows the key can make. Signatures match +exactly: every part of the signed data must be exactly what was signed. There is +no suffix, wildcard or partial matching. -The login form (`POST /`) is limited to 5 attempts per minute per client -address, counting an IPv6 client by its /64; an attempt over the limit is -refused with 429 and a `Retry-After` header. Behind a reverse proxy the client -address comes from `X-Forwarded-For` only when the address pixa sees for -requests that come through the proxy is in `trusted_proxies`; otherwise all -users behind the proxy are counted as one client. That address is not always the -proxy's own: a proxy on the Docker host that connects to pixa over `127.0.0.1` -is seen as the gateway of the container's Docker network, such as `172.17.0.1` -on the default bridge, and one that connects through another of the host's -addresses is seen with that address. To be sure, read it as `remoteIP` in pixa's -request log while it is not in `trusted_proxies` (see `trusted_proxies` under -Configuration). With the default `trusted_proxies` (the RFC 1918 ranges), a -client with a private address can choose the address it is counted by through -its own `X-Forwarded-For`, whether it connects directly or through the proxy, -because its own address is trusted too. Setting `trusted_proxies` to only the -address pixa sees for requests that come through the proxy closes this. - -One client may send `image_requests_per_minute` requests per minute -(default 600) to the image routes, `/v1/image/` and `/v1/e/` together, `GET` and -`HEAD`, cache hits included; a request over the limit is refused with 429 and a -`Retry-After` header. Clients are counted as for the login limit. A browser's -CORS preflight is not counted. Behind a CDN or proxy whose addresses are not in -`trusted_proxies`, all requests arriving from one proxy or CDN address count as -one client, so such a deployment needs a higher limit, or `0`, which turns the -limit off. - -### Encrypted URLs - -An encrypted URL is an image URL made on pixa's own web page by someone who -knows the signing key. It works for any upstream host, allowlisted or not, -without a signature, and whoever gets it can neither read the source URL from it -nor change what it asks for. - -1. Open `/` in a browser over HTTPS (or over plain HTTP while `debug` is on, see - Routes) and log in with the signing key (`signing_key`). The login session - lasts 30 days, or until someone logs out with the generator page's Logout - button. Every login is made with the one signing key, so logging out ends - every login session, in every browser, a copy of the session cookie included. - pixa keeps a random value in its database, which it makes when the database - has none and replaces at each logout. A login session carries the value - current at its login and is refused once the value has changed, also after a - restart (see Deployment). The check for a logout compares no time, so a login - after a logout works whatever the host's clock says; the 30 days are still - measured by the host's clock. -2. On the generator page, give the source image's URL, the width and height, the - format, quality and fit, and how long the URL lasts, then submit the form - (`POST /generate`). The format is JPEG XL unless another is chosen; a form - sent with an empty format, or none, also makes a JPEG XL URL. Width and - height both empty or `0` keep the original size; if only one of them is empty - or `0`, that side is scaled to keep the image's proportions. A URL whose - scaled side would come out above 8192 answers 400, and so does an AVIF URL of - more than 16,777,216 pixels or a source URL whose path holds a `..` segment. - So does a source URL that the image URL would fetch written another way, and - the page then shows the URL pixa would fetch: one with a user name and - password, which pixa does not send, or one whose path holds `%2F`, fetched as - `/`, or `(`, fetched as `%28`, as pixa keeps the source path decoded and - fetches it percent-encoded again (see Routes). -3. The page shows the URL, `https:///v1/e//img.`, and when - it expires. `` is the host the page was opened on, and the URL starts - with `http` instead while `debug` is on. The name after the token is ignored - and only gives the URL a file extension, `jpg` for `orig` and `auto`, and - `jxl` for a form with no format. - -The token holds the source's host, path and query and the size, format, quality, -fit and expiry, encrypted with a key derived from `signing_key`. The host is -kept in lower case and an empty path as `/`, as for `/v1/image/`, so -`https://CDN.Example.com` and `https://cdn.example.com/` are one source image, -fetched and cached once. The source URL's scheme is not kept: the image is -fetched like any other (see Routes), and the blocked networks still apply. - -How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never. -The expiry is fixed in the token when the URL is made and cannot be changed or -revoked afterwards. Until then the image is served with a `max-age` that ends at -the expiry (see Routes); after it the URL answers 410 `URL has expired`. A URL -made to last forever stops working only when `signing_key` changes: changing it -makes every encrypted URL already handed out answer 400, and ends every login -session. - -### Image Metadata - -pixa decodes and re-encodes every image it serves, and removes all metadata from -the output: EXIF (GPS position, camera make, model and serial number, capture -time, embedded thumbnail), XMP, IPTC and the ICC colour profile. This cannot be -turned off. - -- The `orig` format means the source's own format, not the source's bytes: an - `orig` image is re-encoded and stripped like any other. -- An image with an EXIF orientation is turned upright first, so it displays the - same without the tag; a requested size applies to the upright image. -- An image with an ICC profile is converted to sRGB first, since clients show an - image with no profile as sRGB. Colours outside sRGB, such as the most - saturated ones in a Display P3 photo, are clipped. -- The one exception is JPEG XL with libvips 8.16 and later: libvips then writes - an EXIF block of its own into the image, as govips cannot ask libvips to leave - it out. It holds the image's size and otherwise fixed values, such as an - orientation of 1 and a resolution of 72 dpi, and nothing from the source. - -### Source Hosts - -Source hosts may be allowlisted in the configuration. Non-allowlisted hosts -require an HMAC-SHA256 signature. - -#### Signature Specification - -Signatures use HMAC-SHA256 and include an expiration timestamp to prevent replay -attacks. Signatures are **exact match only**: every component (host, path, -query, dimensions, format, expiration, quality, fit) must match exactly what was -signed. No suffix matching, wildcard matching, or partial matching is supported. - -**Signed data format** (colon-separated): +The signed data is one line of nine parts, joined with `:`: ``` HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit") ``` -Where: +- `host`: the source host as the URL writes it up to the first `/`, in lower + case. An IPv6 address keeps its brackets, and a port is kept after its `:`. + Examples: `cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`. +- `path`: the source path with its percent-escapes decoded, as under Image URLs, + then with each `%` written as `%25` and each `:` as `%3A`. So + `/photos/my%20cat.jpg` in the URL is signed as `/photos/my cat.jpg`, and + `/photos/a:b.jpg` as `/photos/a%3Ab.jpg`. +- `query`: the source query as under Image URLs, with `%` and `:` written as in + `path`. It is empty when there is no query. Neither part then holds a `:`, so + path `/a:b` with no query and path `/a` with query `b:` are signed + differently. +- `width` and `height`: the requested size in pixels. A side of `0` is signed as + `0`, and `orig` as `0` and `0`. +- `format`: the output format as listed under Image URLs, with `original` signed + as `orig`, `jpg` as `jpeg`, and no format as `jxl`. So a URL with no format + has the signature of the same URL ending in `.jxl`. `auto` is signed as + `auto`, not as the format chosen for the request. +- `expiration`: the URL's `exp`, the Unix time in seconds when the signature + expires. A request whose `exp` is not a whole number is refused with 400. An + empty `exp=` is one example. +- `quality`: the URL's `q`, or `85` when the URL has no `q`. +- `fit`: the URL's `fit`, or `cover` when the URL has no `fit`. -- `host` — source host as the URL writes it up to the first `/`, in lower case, - keeping an IPv6 address's brackets and any `:` and port after it (e.g. - `cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`) -- `path` — source path, with its percent-escapes decoded as under Routes (e.g. - `/photos/cat.jpg`, or `/photos/my cat.jpg` for `/photos/my%20cat.jpg`), with - each `%` written as `%25` and each `:` as `%3A`, so `/photos/a:b.jpg` is - signed as `/photos/a%3Ab.jpg` -- `query` — source query string as under Routes, empty string if none, with `%` - and `:` written as in `path`; neither then holds a `:`, so path `/a:b` with no - query and path `/a` with query `b:` are signed differently -- `width` — requested width in pixels, or `0`: the width is then worked out from - the source's proportions and the requested height, rounded down but at least - 1, or is the source's own width when the height is `0` too (the `orig` size) -- `height` — requested height in pixels, or `0`: the height is then worked out - from the source's proportions and the requested width, rounded down but at - least 1, or is the source's own height when the width is `0` too (the `orig` - size) -- `format` — output format, one of those listed under Routes, with `original` - signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no - format has the signature of the same URL ending in `.jxl`; `auto` is signed as - `auto`, not as the format chosen for the request -- `expiration` — the URL's `exp` query parameter, the Unix timestamp when the - signature expires; a request whose `exp` is not a whole number, an empty - `exp=` included, is refused with 400 -- `quality` — the URL's `q` query parameter, a whole number from 1 to 100, or - `85` when the URL has no `q`; a request whose `q` is anything else is refused - with 400 -- `fit` — the URL's `fit` query parameter (cover, contain, fill, inside, - outside), or `cover` when the URL has no `fit`; a request whose `fit` is - anything else, an empty `fit=` included, is refused with 400. `outside` scales - the image to cover the requested size, keeping its proportions, so it can come - out larger than that size on one side: an 8192x1 source at `100x0`, which is - 100x1 by its proportions, comes out at 8192x1 +The URL's `sig` is the HMAC-SHA256 result in base64url, the URL-safe alphabet of +RFC 4648, with the trailing `=` padding kept: 44 characters in all. pixa +compares it exactly. A signature encoded without padding is refused with 401. +Node's `base64url` and Go's `base64.RawURLEncoding` encode without padding. -The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of -RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa -compares it exactly, so a signature encoded without padding, as Node's -`base64url` and Go's `base64.RawURLEncoding` do, is refused with 401. +**Example.** With the signing key `example-signing-key-for-documentation`, +resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP, expiring at +1704067200, with the default quality and fit: -**Example:** with the signing key `example-signing-key-for-documentation`, -resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP with expiration -1704067200, default quality and fit: - -1. Build input: +1. Build the signed data: `cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover` -2. Compute HMAC-SHA256 of it with the signing key -3. Base64URL-encode the result, keeping the `=` padding: +2. Compute its HMAC-SHA256 with the signing key. +3. Encode the result as base64url, keeping the `=` padding: `-ay7KHpfqmtIGbibDGbUuBDkymi-Ymdn0NkC6j5EJag=` -4. URL: +4. The URL: `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=-ay7KHpfqmtIGbibDGbUuBDkymi-Ymdn0NkC6j5EJag=&exp=1704067200` -For the same image at quality 40 with fit `contain`, the input ends in -`:40:contain`, the signature is `5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=`, +For the same image at quality 40 with fit `contain`, the signed data ends in +`:40:contain`. The signature is `5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=`, and the URL is `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`. For a source URL with a query, `https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg`, at -800x600 WebP with the same key and expiration and the default quality and fit: +800x600 WebP with the same key, expiry, quality and fit: -1. Build input, with the query's `:` written as `%3A` and its `%` as `%25`: +1. The signed data, with the query's `:` written as `%3A` and its `%` as `%25`: `cdn.example.com:/img:url=https%3A//origin.example.com/my%2520cat.jpg:800:600:webp:1704067200:85:cover` -2. The signature is `37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=` -3. URL, with the `?` before the query written as `%3F` and the query's `%` as - `%25` (see Routes): +2. The signature: `37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=` +3. The URL, with the `?` before the query written as `%3F` and the query's `%` + as `%25`: `/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp?sig=37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=&exp=1704067200` -**Allowlist patterns:** +### Encrypted URLs -- **Exact match**: `cdn.example.com` — matches only that host -- **Suffix match**: `.example.com` — matches `cdn.example.com`, - `images.example.com`, and `example.com` +An encrypted URL is an image URL made on pixa's own web page by someone who +knows the signing key. It works for any upstream host, allowlisted or not, +without a signature. Whoever gets it can neither read the source URL from it nor +change what it asks for. -An IP address is matched exactly; write an IPv6 address without brackets. An -entry that is neither a host name (letters, digits, hyphens, underscores and -dots, with at most one leading dot) nor an IP address, such as one with a port -or a `*.` wildcard, aborts startup. +1. Open `/` in a browser over HTTPS and log in with the signing key. The login + session lasts 30 days, or until someone logs out with the generator page's + Logout button. +2. On the generator page, fill in the form. Its fields are listed below. Then + submit it (`POST /generate`). +3. The page shows the URL, `https:///v1/e//img.`, and when + it expires. `` is the host the page was opened on. While `debug` is on, + the URL starts with `http` instead. The name after the token is ignored. It + only gives the URL a file extension: `jpg` for `orig` and `auto`, and `jxl` + for a form with no format. -pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so -`CDN.Example.com` is matched, fetched, cached and counted against -`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has -a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature -even when the host is allowlisted. The host must be a host name or an IPv4 -address (letters, digits, hyphens, underscores and dots) or an IPv6 address in -brackets, either one optionally followed by `:` and a port. Any other host, such -as one holding `@`, or a `?`, `#`, `%`, `\` or space written percent-encoded, is -refused with 400. +The form's fields: + +- The source image's URL. Its scheme is not kept (below). +- The width and height. Both empty or `0` keep the source's size. When only one + of them is empty or `0`, that side is worked out from the source's proportions + when the image is made (see Image URLs). +- The format: `orig`, `auto`, `jpeg`, `png`, `webp`, `avif`, `jxl` or `gif`. + `jxl` is selected unless another is chosen. A form sent with an empty format, + or none, also makes a JPEG XL URL. +- The quality, a whole number from 1 to 100. The page offers five values: `25`, + `50`, `70`, `85` and `100`. `85` is selected unless another is chosen. +- The fit, one of the five fit modes (see Image URLs). `cover` is selected + unless another is chosen. A form sent with an empty fit, or none, makes a + `cover` URL. +- How long the URL lasts, in seconds, or `0` for never. The page offers seven + choices: never, 1 minute, 1 hour, 1 day, 1 week, 30 days and 1 year. Never is + selected unless another is chosen. + +The page refuses a form with 400 when: + +- the source URL does not parse, or has no host; +- the source URL's path holds a `..` segment, as `/v1/image/` refuses one; +- a width, height, quality or expiry is not a number, or is out of range; +- the fit is not one of the five fit modes; +- pixa would fetch the source URL in another form (below). + +The page refuses a source URL that pixa would fetch in another form, and shows +the URL pixa would fetch. A URL with a user name and password is one example: +pixa does not send them. A URL ending in `?` with no query is another. A path +holding `%2F` is another: pixa would fetch it as `/`. So is a path holding `(`: +pixa would fetch it as `%28` (see Image URLs). + +The image is made only when the URL is requested. A side that would come out +above 8192, or an AVIF output of more than 16,777,216 pixels, is refused with +400 then (see Image URLs). + +Every login is made with the one signing key. So logging out ends every login +session, in every browser, and a copied session cookie too. pixa keeps a random +value in its database. It makes the value when the database has none, and +replaces it at each logout. A login session carries the value current at its +login. Once the value has changed, the session is refused, after a restart too. +The check compares no time, so a login after a logout works whatever the host's +clock says. The 30 days are still measured by the host's clock. + +The token holds the source's host, path and query, and the size, format, +quality, fit and expiry. It is encrypted with a key derived from `signing_key`. +The host is kept in lower case and an empty path as `/`, as for `/v1/image/`. So +`https://CDN.Example.com` and `https://cdn.example.com/` are one source, fetched +and cached once. The source URL's scheme is not kept. The image is fetched like +any other (see Image URLs), and the blocked networks still apply. + +The expiry is fixed in the token when the URL is made. It cannot be changed or +revoked afterwards. Until then the image is served with a `max-age` that ends at +the expiry (see Image URLs). After it, the URL answers 410 `URL has expired`. A +URL made to last forever holds no expiry. It stops working only when +`signing_key` changes. Changing the key makes every encrypted URL already handed +out answer 400, and ends every login session. + +### Source Hosts + +Hosts in `allowlist_hosts` are served without a signature. Any other host needs +one. An entry is written in one of two ways: + +- `cdn.example.com`: an exact match, only that host; +- `.example.com`: a suffix match, which matches `cdn.example.com`, + `images.example.com` and `example.com` itself. + +An IP address is matched exactly. Write an IPv6 address without brackets. An +entry that is neither a host name nor an IP address aborts startup. A host name +here is letters, digits, hyphens, underscores and dots, with at most one leading +dot. So an entry with a port, or a `*.` wildcard, aborts startup. + +pixa lower-cases the host of a `/v1/image/` URL before anything else reads it. +So `CDN.Example.com` is matched, fetched, cached and counted against +`upstream_connections_per_host` as `cdn.example.com`. A URL whose host has a +port needs a signature even when the host is allowlisted. So does one whose host +ends in `:` with no port. `cdn.example.com:8443` is a host with a port. The host +must be one of: + +- a host name or an IPv4 address: letters, digits, hyphens, underscores and + dots; +- an IPv6 address in brackets. + +Either may be followed by `:` and a port. Any other host is refused with 400. +One holding `@` is an example. So is one with a `?`, `#`, `%`, `\` or space +written percent-encoded. + +### Image Metadata + +pixa decodes and re-encodes every image it serves, and removes all metadata from +the output. This cannot be turned off. The metadata removed is: + +- EXIF, which can hold the GPS position, the camera make, model and serial + number, the capture time and an embedded thumbnail; +- XMP and IPTC, two other metadata formats; +- the ICC colour profile. + +The rest of the rules: + +- The `orig` format means the source's own format, not the source's bytes. An + `orig` image is re-encoded and stripped like any other. +- An image with an EXIF orientation is turned upright first, so it displays the + same without the tag. A requested size applies to the upright image. +- An image with an ICC profile is first converted to sRGB, the standard colour + space, since clients show an image with no profile as sRGB. Colours outside + sRGB are clipped: the most saturated ones in a Display P3 photo, for example. +- JPEG XL output is the one exception, with libvips 8.16 and later. libvips then + writes an EXIF block of its own, which pixa cannot stop. The block holds the + image's size and fixed values: an orientation of 1 and a resolution of 72 dpi. + Nothing in it comes from the source. The Docker image has libvips 8.16, so its + JPEG XL output carries this block. + +### Limits + +**Shared work.** Several requests for one variant can miss the cache at the same +time. They then share one upstream fetch, or one read of the cached source, and +one conversion. The first request does the work. The others wait for its image +or its error, and hold no upstream connection or processing slot of their own. A +waiting request stops waiting when its own client goes away. It is then +answered 499. The work goes on for the others even when the first request's +client goes away. The shared fetch sends the first request's ID upstream. The +lines logged for the fetch and the conversion carry that ID. + +**Timeouts.** `downstream_timeout` (default `60s`) bounds the whole answer to +one request. The upstream fetch counts toward it. So do the waits for a +connection slot and for a processing slot. The wait for a shared +`upstream_connections` slot ends after 10 seconds, and so does the wait for a +processing slot. The wait for an `upstream_connections_per_host` slot ends only +with `downstream_timeout`. Keep `downstream_timeout` longer than +`upstream_fetch_timeout` plus 20 seconds. + +A conversion whose request's `downstream_timeout` has ended stops waiting for a +processing slot. If its `downstream_timeout` ends after it has a slot but before +it begins to encode, it gives the slot back. For a shared conversion, the first +request's deadline counts. When a shared conversion stops this way, each request +waiting for it is answered 503 `server busy, try again later`, even if its own +`downstream_timeout` has not ended. `govips`, the library pixa drives libvips +through, gives no way to stop an encode once it has begun. So an encode keeps +its slot until it finishes, which can be after `downstream_timeout`. + +**Encode times.** The limits on image size bound how long a conversion can take. +They are 8192 on each side of the source and of the output, and 16,777,216 +pixels for an AVIF output. Decoding an 8192x8192 source takes up to about 11 +seconds. The WebP and JPEG XL encoders run below libvips' default effort. AVIF +runs at the lowest effort `govips` can set. With one libvips thread, as pixa +runs it, encoding 8192x8192 random pixels takes up to about: + +- 1 second as JPEG; +- 15 seconds as PNG; +- 16 seconds as GIF; +- 23 seconds as WebP with an alpha channel, and 16 without; +- 27 seconds as JPEG XL with an alpha channel, and 13 without. + +An AVIF output of 4096x4096, 8192x2048 or 2048x8192 random pixels takes up to +about 34 seconds with an alpha channel, and 22 without. At 8192x8192 and quality +85 it would take about 96 seconds with an alpha channel, and 52 without. The 34 +seconds hold at any quality and at 8 or 16 bits per sample. No encode within the +limits takes more than about 40 seconds. The default `downstream_timeout` is 60 +seconds. + +**Processing and connections.** `max_concurrent_processing` images are decoded +and encoded at once. A request that finds every slot in use waits up to 10 +seconds for one. If none frees up, it is answered 503 +`server busy, try again later`. `upstream_connections` bounds the fetches in +progress to all upstream hosts together, and `upstream_connections_per_host` +those to one host. A fetch holds its connection until its image has been +processed. A fetch that finds every shared connection in use waits up to 10 +seconds for one. If none frees up, the request is answered 503 +`server busy, try again later`. A fetch that finds every connection to its host +in use waits for one until `downstream_timeout` ends. + +**Client limits.** The login form (`POST /`) is limited to 5 attempts per minute +per client address. An attempt over the limit is refused with 429 and a +`Retry-After` header. One client may send `image_requests_per_minute` requests +per minute (default 600) to `/v1/image/` and `/v1/e/` together, `GET` and +`HEAD`, cache hits included. A request over the limit is refused with 429 and a +`Retry-After` header. A browser's CORS preflight is not counted. `0` turns the +limit off. Both limits count an IPv6 client by its `/64`. + +**The client address.** Behind a reverse proxy, pixa takes the client address +from `X-Forwarded-For` only when the request arrives from an address in +`trusted_proxies`. Otherwise all users behind the proxy count as one client. The +address the request arrives from is not always the proxy's own. A proxy on the +Docker host may connect to pixa over `127.0.0.1`. pixa then sees the gateway of +the container's Docker network, such as `172.17.0.1` on the default bridge. A +proxy that connects through another of the host's addresses is seen with that +address. To be sure, find the address as described under `trusted_proxies` (see +Configuration). The default `trusted_proxies` is the RFC 1918 ranges, the +private address ranges. With that default, a client with a private address is +itself trusted. It can then choose the address it is counted by, through its own +`X-Forwarded-For`. Setting `trusted_proxies` to only the proxy's address closes +this. Behind a content delivery network (CDN) or proxy whose addresses are not +in `trusted_proxies`, all requests arriving from one address count as one +client. Such a deployment needs a higher `image_requests_per_minute`, or `0`. + +**Request limits.** pixa reads a request's headers within 10 seconds and the +whole request within 30 seconds. Headers are limited to 8 KiB. An idle +connection is closed after 120 seconds. A form body over 1 MiB is refused +with 413. + +### Storage + +- **Sources**: `/cache/sources///`, one file for + each source URL +- **Database**: `/state.sqlite3` +- **Variants**: `/cache/variants///` + +`` and `` are the first and second pairs of characters of the file's +name. The database is one SQLite file. SQLite is the database library pixa +embeds; nothing else needs to run. By default the database is in WAL mode, +SQLite's write-ahead log (see `db_url`). SQLite then keeps `state.sqlite3-wal` +and `state.sqlite3-shm` beside the file while pixa runs. + +The image cache keeps these tables in the database: + +- `source_content`: a row for each source, by source key, with its size and when + it was last used +- `variant_content`: a row for each variant, by variant key (column + `cache_key`), with its size and when it was last used +- `cache_usage`: the total size of the cached sources and variants, which + eviction reads +- `negative_cache`: a row for each source URL in the negative cache, with when + it expires + +The `logout` table holds the random value that a logout replaces (see Encrypted +URLs). + +pixa reads a cached source's format from its file's first bytes. When they match +none of the source formats, pixa fetches the source again. Two source URLs that +serve the same bytes are stored twice, once under each URL. A source is stored +by writing its file and then its row. When the row cannot be written, the file +is removed again. + +Each source and variant file is written under a temporary name and synced to +disk before it is renamed into place. So a crash never leaves part of a file +under its own name. + +When pixa starts, and then at the spacing below, it checks the cached files +against the database. It removes each source or variant whose file is empty, or +not the size its row records. It logs a warning naming the file. A later request +for that image fetches the source or makes the variant again. The check also +drops the rows of variants whose files are missing. When pixa cannot read a +variant file's size for another reason, it keeps the variant's row and logs a +warning naming the variant key and the error. During a check, pixa skips a +directory it cannot list and a file whose size it cannot read. It logs a warning +naming the path and the error, and goes on with the rest of the check. A file +renamed or removed after pixa listed its directory gets no warning. + +A check reads the database once for every cached file, so the checks are spaced +out. After a check ends, pixa waits at least ten times as long as that check +took. Eviction runs every five minutes (below). The next check starts at the +first of those runs after that wait. After a check that took one minute, the +next one starts 10 to 15 minutes after that check ended. + +A cache hit opens the variant's file and nothing else. It makes no database +call. The content type comes from the file's first bytes, which are those of the +format pixa wrote it in. A file whose first bytes are not those of a format pixa +writes is not served, and the image is converted again. A hit hands the time it +was served to a background writer and does not wait for the writer. Eviction +orders by that time. The hit is counted in memory, for the +`pixa_cache_hits_total` metric. A miss that uses a cached source also hands that +writer the time it used the source. Once a second the writer writes the waiting +times in one transaction. Up to 10,000 access times of variants, and as many of +sources, can wait to be written. A time handed over beyond that is dropped. Its +image then only looks less recently used to eviction. `script/bench` measures +how many hits pixa serves per second (see Load Test). + +Eviction removes the least recently used sources and variants until the cache is +within `cache_max_bytes`. It runs after each store, and every five minutes. It +also removes the expired rows of the negative cache every five minutes, at most +1000 per statement, so a request waits behind 1000 deletions at most. ### Configuration -Every setting can be given as an environment variable, in a YAML config file -(`--config`), or both. A variable present in the environment wins over the file, -even when it is empty, and the file wins over the built-in default. The one -exception is a variable set in the file's `env:` section. pixa sets that -variable while loading the file, so the variable wins over the environment pixa -started with. It also wins over the file's key for the same setting. A -variable's value is parsed as the same text in the file would be. A setting that -holds a list takes comma-separated entries, with the spaces around each trimmed. -An empty variable is an empty list. A value that does not parse or is invalid -aborts startup, naming the variable. pixa refuses to start when a variable -starts with `PIXA_` but is not in the list below, such as a misspelled one or -`PIXA_PORT`. The error names the variable, as it names an unknown config key. +Every setting can be given as an environment variable, in a config file +(`--config`), or both. The file is YAML, a plain-text format. A variable present +in the environment wins over the file, even when it is empty. The file wins over +the built-in default. pixa reads a variable's value as a quoted string in the +file. So `PIXA_DEBUG=1` turns debug on, but `debug: 1` in the file aborts +startup. A setting that holds a list takes comma-separated entries, with the +spaces around each trimmed. An empty variable is an empty list. + +The file's `env:` section is the one exception to that order. pixa sets each +variable named there while it loads the file. So that variable wins over the +environment pixa started with, and over the file's key for the same setting. + +pixa refuses to start when a value does not parse or is invalid. The error names +the setting. It also refuses to start when a key in the file is set to null. A +key written as `key:` with nothing after it is null. Omit the key to use the +default. It refuses to start when the file has a key that is not in the list +below, and the error names the key. It refuses to start when a variable starts +with `PIXA_` but is not in the list below, such as a misspelled one or +`PIXA_PORT`. The error names the variable. The variables set by the file's +`env:` section are checked the same way. + pixa also accepts `PIXA_CONFIG_PATH`, which names the config file, like -`--config`. The variables set by the file's `env:` section are checked the same -way. +`--config`. pixa reads at most one config file. It reads the file given with `--config` (or `-c`), or else the one `PIXA_CONFIG_PATH` names. pixa refuses to start when that @@ -758,191 +918,174 @@ parse, and the error names the file. It also refuses to start when it may not enter one of these directories, whether or not the file is there, since it cannot tell. With no file, pixa uses the environment and the defaults. +Durations are written as Go durations, such as `30s` or `2m`, and must be +positive. A bare number has no unit and aborts startup. Sizes are a whole number +of bytes. A CIDR range is an address range written as an address and a prefix +length, such as `10.0.0.0/8`. An address matches a range that contains it. + +`blocked_networks` and `trusted_proxies` hold CIDR ranges. For both, an IPv6 +address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 +address it holds. So an IPv6 range such as `::/0` matches neither it nor a plain +IPv4 address. A range written in that form is read as the IPv4 range it covers: +`::ffff:10.0.0.0/104` is `10.0.0.0/8`. Such a range shorter than `/96` covers +more than IPv4 addresses and aborts startup. + Each item below is one setting, named by its config key: - `signing_key`: variable `PIXA_SIGNING_KEY`, required. The secret for signed - URLs, encrypted URLs and login. It must be 32 characters or longer. -- `port`: variable `PORT`, default `8080`. The port to listen on. + URLs, encrypted URLs and login. It must be 32 characters or longer. The + placeholder in `configs/config.example.yml` is refused. pixa uses the key as + it is for two things: as the HMAC key of signed URLs (see Signed URLs), and as + the password of the login page. From it, pixa derives four more keys at + startup, with HKDF-SHA256, a key derivation function. One signs the session + cookie and one encrypts it. One encrypts the token of an encrypted URL. One + signs the token cookie of the login and generator pages' forms. +- `port`: variable `PORT`, default `8080`. The port to listen on, from 1 + to 65535. - `state_dir`: variable `PIXA_STATE_DIR`, default `/var/lib/pixa`. The directory - for the database and the disk cache. -- `db_url`: variable `PIXA_DB_URL`, default `state.sqlite3` in the state - directory. The URL of the SQLite database. -- `cache_max_bytes`: variable `PIXA_CACHE_MAX_BYTES`, default 75% of the sum of - the free space and what the cache holds. The disk cache limit in bytes. `0` - turns the disk cache off. + for the database and the disk cache. pixa creates it at startup when it is + missing. It refuses to start when it cannot create or write the directory. +- `db_url`: variable `PIXA_DB_URL`, default + `file:/state.sqlite3?_pragma=journal_mode(WAL)`. The SQLite + database to open. The default keeps the database in WAL mode, SQLite's + write-ahead log, which lets reads go on during a write. pixa adds + `_pragma=busy_timeout(5000)` to any `db_url`. A write that finds another + program writing to the file then waits up to five seconds for it instead of + failing. pixa opens one connection, so its own reads and writes run one at a + time. WAL mode comes only from the URL: keep `_pragma=journal_mode(WAL)` in + one you set. An empty `db_url` aborts startup: omit the key to use the + default. In the default, `state_dir` is percent-encoded: a `%`, `?` or `#` in + the name is written as `%25`, `%3F` or `%23`. For a `state_dir` of `/`, or one + starting with `//`, the default starts with `file:////`, so SQLite does not + read the first name in the path as a host. +- `cache_max_bytes`: variable `PIXA_CACHE_MAX_BYTES`, default below. The disk + cache limit in bytes. When the key is omitted, pixa works the limit out at + startup. It takes the free space on the filesystem holding + `/cache/`, adds the bytes the cache already holds, and takes 75% of + the sum. The default is at least 500 MiB. A value you set is used exactly as + given. `0` turns the disk cache off: every request then fetches and converts + its image, and only the negative cache is used. A negative value aborts + startup. - `allowlist_hosts`: variable `PIXA_ALLOWLIST_HOSTS`, default none. The upstream - hosts whose images pixa serves without a signature. -- `referer_blocklist`: variable `PIXA_REFERER_BLOCKLIST`, default none. The - image routes refuse a request with 403 when its `Referer` header names one of - these hosts. -- `blocked_networks`: variable `PIXA_BLOCKED_NETWORKS`, default none. The CIDR - ranges pixa never fetches from, on top of the built-in ones. + hosts whose images pixa serves without a signature, written as under Source + Hosts. Every image on a listed host is served to anyone, so list only hosts + you control. An entry that is neither a host name nor an IP address aborts + startup. +- `referer_blocklist`: variable `PIXA_REFERER_BLOCKLIST`, default none. Hosts + whose pages may not show pixa's images, to stop other sites embedding them. + Entries are written and matched as for `allowlist_hosts`. A request whose + `Referer` header names a listed host is refused with 403. That applies to + `/v1/image/` and `/v1/e/`. It happens before the signature or token is + checked, and before the cache or the upstream host is used. The request is + refused even when the image is cached. A request with no `Referer` is served, + as many clients send none. So is one whose `Referer` does not parse as a URL + with a host. So a site whose pages send no `Referer`, for example with + `Referrer-Policy: no-referrer`, is not stopped. The login and generator pages + are not covered. +- `blocked_networks`: variable `PIXA_BLOCKED_NETWORKS`, default none. CIDR + ranges pixa never fetches from, on top of the built-in ones. An invalid CIDR + aborts startup. The built-in ranges always apply. They are: + - `0.0.0.0/8`, this network, `0.0.0.0` included; + - `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`, the private ranges; + - `100.64.0.0/10`, carrier-grade NAT; + - `127.0.0.0/8`, loopback; + - `169.254.0.0/16`, link-local; + - `192.0.0.0/24`, IETF protocol assignments; + - `198.18.0.0/15`, benchmarking; + - `224.0.0.0/4`, multicast; + - `::`, the unspecified address; + - `::1`, loopback; + - `64:ff9b::/96` and `64:ff9b:1::/48`, NAT64; + - `fc00::/7`, the unique local (private) range; + - `fe80::/10`, link-local; + - `ff00::/8`, multicast. - `trusted_proxies`: variable `PIXA_TRUSTED_PROXIES`, default the RFC 1918 - ranges. The CIDR ranges of the proxies whose `X-Forwarded-For` header pixa - believes. -- `allow_http`: variable `PIXA_ALLOW_HTTP`, default `false`. Allows plain-HTTP - upstream hosts. It is for testing only. + private ranges: `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`. CIDR + ranges of the reverse proxies in front of pixa. `X-Forwarded-For` is believed + only when the request reaches pixa from an address in one of these ranges. The + client address is then the rightmost entry of the header that is not itself a + trusted proxy. Otherwise the header is ignored and the address the request + came from is used. So a client connecting from outside these ranges cannot + choose its address. An explicitly empty list (`[]`, or an empty variable) + trusts no one. A list you set replaces the default. An invalid CIDR aborts + startup. Set this to the address pixa sees for requests that come through your + proxy, such as `172.17.0.1/32`. Do so when the default does not cover it, or + to trust nothing else (see Limits). To find that address, set this to `[]`, + send a request through the proxy, and read `remoteIP` in pixa's log line for + it. +- `allow_http`: variable `PIXA_ALLOW_HTTP`, default `false`. While `true`, pixa + fetches every source over plain HTTP, and follows redirects to plain HTTP. It + is for testing only. - `upstream_connections_per_host`: variable `PIXA_UPSTREAM_CONNECTIONS_PER_HOST`, default `20`. The most connections to - one upstream host at once. + one upstream host at once. It must be at least 1. A fetch that finds every + connection to its host in use waits for one until `downstream_timeout` ends + (see Limits). - `upstream_connections`: variable `PIXA_UPSTREAM_CONNECTIONS`, default `64`. - The most connections to all upstream hosts together at once. + The most connections to all upstream hosts together at once, on top of the + limit per host. It must be at least 1. A fetch holds its connection until its + image has been processed (see Limits). - `max_concurrent_processing`: variable `PIXA_MAX_CONCURRENT_PROCESSING`, - default half the CPUs, at least 1. The most images processed at once. + default half the CPUs Go uses (`GOMAXPROCS`), rounded down, and at least 1. + The most images decoded and encoded at once. `GOMAXPROCS` follows a + container's CPU limit. A value that is not a whole number of at least 1 aborts + startup. Each image can take a lot of memory (see Running under upaas). - `image_requests_per_minute`: variable `PIXA_IMAGE_REQUESTS_PER_MINUTE`, - default `600`. How many image requests one client may send per minute. `0` - turns the limit off. + default `600`. How many requests one client may send per minute to + `/v1/image/` and `/v1/e/` together (see Limits). `0` turns the limit off. A + negative or non-integer value aborts startup. - `upstream_fetch_timeout`: variable `PIXA_UPSTREAM_FETCH_TIMEOUT`, default - `30s`. The time allowed for one fetch from an upstream host. + `30s`. The time allowed for one fetch from an upstream host. A fetch whose + response headers have not arrived by then answers 504. - `upstream_max_response_size`: variable `PIXA_UPSTREAM_MAX_RESPONSE_SIZE`, - default 50 MiB. The largest upstream response accepted, in bytes. + default `52428800` (50 MiB). The largest upstream response accepted, in bytes, + from 1 to `1073741824` (1 GiB). It also limits the image data pixa decodes. A + cached source larger than it is fetched again. - `downstream_timeout`: variable `PIXA_DOWNSTREAM_TIMEOUT`, default `60s`. The - time allowed for answering one client request. + time allowed for answering one client request. Keep it longer than + `upstream_fetch_timeout` plus 20 seconds (see Limits). - `access_control_allow_origin`: variable `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN`, - default `*`. The CORS origin allowed to read image responses. It is `*` or one - origin. + default `*`. The origin whose pages may read the responses of `/v1/image/` and + `/v1/e/` in a browser. pixa sends it as the CORS `Access-Control-Allow-Origin` + header. No other route sends it. `*` is any site. Otherwise it is one `http` + or `https` origin such as `https://example.com`. Its host is a lowercase host + name (letters, digits, hyphens and dots, with a letter in its last part) or an + IP address (IPv6 in brackets, in its shortest form). It may have a port from 1 + to 65535 that has no leading zero and is not the scheme's default. Any other + value aborts startup, including an origin with another scheme, such as a + browser extension's. - `metrics.username`: variable `PIXA_METRICS_USERNAME`, default empty. The - username for `/metrics`. pixa serves `/metrics` only when the username and the - password are both set. + username for `/metrics`. pixa serves `/metrics`, and measures every HTTP + request, only when the username and the password are both set. Setting one + without the other aborts startup. - `metrics.password`: variable `PIXA_METRICS_PASSWORD`, default empty. The password for `/metrics`. Set it together with the username. -- `sentry_dsn`: variable `PIXA_SENTRY_DSN`, default empty. The Sentry DSN for - error reporting. An empty DSN turns error reporting off. +- `sentry_dsn`: variable `PIXA_SENTRY_DSN`, default empty. Sentry is an error + reporting service, and the DSN is the URL it gives you. A DSN turns error + reporting on. It must be a URL with a scheme and a host, or empty. - `debug`: variable `PIXA_DEBUG`, default `false`. Turns on debug logging and - plain-HTTP local development. + plain-HTTP local development (see Routes). - `maintenance_mode`: variable `PIXA_MAINTENANCE_MODE`, default `false`. While - `true`, pixa answers image requests with 503. The health check still - answers 200. + `true`, `/v1/image/` and `/v1/e/` answer every `GET` and `HEAD` request with + 503, a `Retry-After` header of 300 seconds and a JSON error body. The health + check still answers 200 and reports `"maintenance_mode": true`. It stays 200 + because the image's Docker `HEALTHCHECK` requests it, and upaas marks a deploy + failed when its container is unhealthy. The login and generator pages and + `/metrics` keep working. -Key settings in more detail: - -- `access_control_allow_origin` — the origin a browser lets read the responses - of the image routes, `/v1/image/` and `/v1/e/`, sent as the CORS - `Access-Control-Allow-Origin` header; no other route sends it. `*`, the - default, is any site; otherwise one `http` or `https` origin such as - `https://example.com`, whose host is a lowercase host name (letters, digits, - hyphens and dots, with a letter in its last part) or an IP address (IPv6 in - brackets, in its shortest form), with an optional port 1-65535 that has no - leading zero and is not the scheme's default. Any other value, including - another scheme such as a browser extension's, aborts startup -- `allowlist_hosts` — list of allowed upstream hosts -- `referer_blocklist` — list of hosts whose pages may not show pixa's images, to - stop other sites hotlinking them. Entries are written and matched as for - `allowlist_hosts` (see Allowlist patterns), and an entry that is neither a - host name nor an IP address aborts startup. A request to `/v1/image/` or - `/v1/e/` whose `Referer` header names a listed host is refused with 403 before - its signature or token is checked and before the cache or the upstream host is - used, so it fetches nothing, and it is refused even when the image is cached. - A request with no `Referer`, or one that does not parse as a URL with a host, - is served, as many clients send none. So this is easily got around: a site - whose pages send no `Referer` (for example with - `Referrer-Policy: no-referrer`) is not stopped. It does not apply to the login - and generator pages. Default: empty -- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added - to the always-enforced built-in ranges (loopback, private, link-local, CGNAT, - benchmark, NAT64, and the like); an invalid CIDR aborts startup. An IPv6 - address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 - address it holds, so an IPv6 range such as `::/0` refuses neither it nor a - plain IPv4 address. An entry written in that form is read as the IPv4 range it - covers, so `::ffff:10.0.0.0/104` refuses what `10.0.0.0/8` refuses; such an - entry shorter than `/96` covers more than IPv4 addresses and aborts startup -- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of - pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one - of these ranges; the logged and login-recorded client address is then the - rightmost forwarded entry that is not itself a trusted proxy. Otherwise the - direct peer address is used and the header is ignored, so a client connecting - directly from an address outside these ranges cannot spoof its address. An - omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`, - `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a - private network; an explicitly empty list (`[]`) trusts no one, and an - explicit list replaces the default. An invalid CIDR aborts startup. An IPv6 - address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 - address it holds, so an IPv6 range such as `::/0` trusts neither it nor a - plain IPv4 address. An entry written in that form is read as the IPv4 range it - covers, so `::ffff:10.0.0.0/104` trusts what `10.0.0.0/8` trusts; such an - entry shorter than `/96` covers more than IPv4 addresses and aborts startup. - Set this to the address pixa sees for requests that come through your proxy, - such as `172.17.0.1/32`, when the defaults do not cover it, or to trust - nothing else (see the login limit and `image_requests_per_minute` under - Routes). For a proxy on the Docker host that connects to pixa over - `127.0.0.1`, that address is the gateway of the container's Docker network - (`172.17.0.1` on the default bridge), not the proxy's own address; a proxy - that connects through another of the host's addresses is seen with that - address. To be sure which address it is, set this to `[]` (or - `PIXA_TRUSTED_PROXIES` to empty), send a request through the proxy, and read - `remoteIP` in pixa's request log line for it -- `upstream_fetch_timeout` — time allowed for one fetch from an upstream host, - as a duration such as `30s` (the default) or `2m` -- `upstream_max_response_size` — largest upstream response accepted, in bytes; - default `52428800` (50 MiB). It also limits the image data pixa decodes -- `downstream_timeout` — time allowed for answering one client request, as a - duration; default `60s`. The upstream fetch counts toward it, and so do the - waits for an upstream connection and for a processing slot (up to 10 seconds - each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds -- `signing_key` — the secret that protects signed URLs, encrypted URLs and - logins. pixa uses it as it is for two things: - - the HMAC key of signed URLs (see Signature Specification) - - the password of the login page - - pixa derives every other key it uses from it at startup, with HKDF-SHA256: - - a key that signs the session cookie - - a key that encrypts the session cookie - - a key that encrypts the token of an encrypted URL - - a key that signs the CSRF token cookie of the login and generator pages - -- `db_url` — the SQLite database to open; omitted, it is - `file:/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the - database in WAL mode. `state_dir` is percent-encoded in it, so a `%`, `?` or - `#` in the name is written as `%25`, `%3F` or `%23`. For a `state_dir` of `/` - or one starting with `//` it starts `file:////`, so SQLite does not read the - first name in the path as a host. pixa opens one connection to it, so its own - reads and writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to - any `db_url`, so a write that finds another program writing to the file waits - up to five seconds for it instead of failing. WAL mode comes only from the - URL: keep `_pragma=journal_mode(WAL)` in one you set -- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk - cache entirely; omitted defaults to 75% of the sum of the free space on the - filesystem containing `/cache/` and the bytes of source and - transformed images the cache already holds, worked out at startup (minimum 500 - MiB) -- `upstream_connections` — the most connections to upstream hosts at once, all - hosts together, on top of `upstream_connections_per_host`; default `64`. A - fetch holds its connection until its image has been processed. A fetch that - finds all of them in use waits up to 10 seconds for one to free up; if none - does, and `downstream_timeout` has not ended first, the request is answered - 503 with the error `server busy, try again later` -- `max_concurrent_processing` — the most images decoded and encoded at once; - default half the CPUs pixa can use (`GOMAXPROCS`, which follows a container's - CPU limit), rounded down, and at least 1. A value that is not a whole number - of at least 1 aborts startup. A request that finds all of them in use waits up - to 10 seconds for one to free up; if none does, and `downstream_timeout` has - not ended first, it is answered 503 the same way. Each one can take a lot of - memory: see "Running under upaas" -- `image_requests_per_minute` — how many requests one client may send to the - image routes per minute (see the limit on them under Routes); default `600`. - `0` turns the limit off, and a negative or non-integer value aborts startup. - Behind a CDN or proxy whose addresses are not in `trusted_proxies`, all - requests arriving from one proxy or CDN address count as one client, so raise - the limit there, or set `0` -- `maintenance_mode` — while `true`, the image routes (`/v1/image/` and - `/v1/e/`) answer every request for an image with 503, a `Retry-After` header - and a JSON error body. The health check (`/.well-known/healthcheck.json`) - still answers 200 and reports `"maintenance_mode": true`. It stays 200 because - the image's Docker `HEALTHCHECK` requests it: a 503 there would make the - container unhealthy, and upaas marks a deploy failed when its container is - unhealthy. The login and URL generator pages and `/metrics` keep working - -See `configs/config.example.yml` for all options with defaults. +`configs/config.example.yml` names every setting with its default. ### Architecture -- **Dependency injection**: Uber fx. `internal/app` puts every part of pixad in - one fx graph. `internal/providers` builds the image cache, the image proxy and - the other parts that serve images and logins. Each setting's default is - declared once, in `internal/config`. +- **Dependency injection**: `fx` (`go.uber.org/fx`), a library that builds the + program's parts in dependency order and starts and stops them. `internal/app` + puts every part of `pixad` in one `fx` graph. `internal/providers` builds the + image cache, the image proxy and the other parts that serve images and logins. + Each setting's default is declared once, in `internal/config`. - **Packages**: these packages under `internal/` work with the concepts above: - - `handlers`: the routes, which read the image request from the URL, check - the signed or encrypted URL, and make the login session + - `handlers`: the functions the routes call, which read the image request + from the URL, check the signed or encrypted URL, and make the login + session - `imagerequest`: the image request, with the source key and the variant key - `imageproxy`: the image proxy - `imgcache`: the image cache, which keeps its sources in one `FileStore` @@ -955,12 +1098,13 @@ See `configs/config.example.yml` for all options with defaults. - `seal`: the keys derived from `signing_key`, and the encryption of those tokens - `session`: the login session -- **HTTP router**: go-chi -- **Image processing**: govips (CGO wrapper for libvips) -- **Database**: SQLite via modernc.org/sqlite -- **Static assets**: embedded via `//go:embed` -- **Metrics**: Prometheus, at `/metrics`. pixa serves `/metrics` and measures - HTTP requests only when `metrics.username` and `metrics.password` are set. +- **HTTP router**: `chi` (`github.com/go-chi/chi/v5`) +- **Image processing**: `govips`, a Go wrapper for libvips +- **Database**: SQLite via `modernc.org/sqlite` +- **Static assets**: embedded in the binary via `//go:embed` +- **Metrics**: Prometheus, at `/metrics`. Prometheus is a monitoring system that + reads metrics from that route. pixa serves `/metrics` and measures HTTP + requests only when `metrics.username` and `metrics.password` are set. `/metrics` holds: - the duration of HTTP requests - the size of HTTP responses @@ -976,75 +1120,91 @@ See `configs/config.example.yml` for all options with defaults. A request's `handler` label is the pattern of the route it matches, such as `/v1/image/*`, never its path. Requests that match no route share the label `unmatched`: those the router answers 404 or 405, and CORS preflights. A - request's `method` label is its method when that is `GET`, `HEAD`, `POST`, - `PUT`, `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` or `TRACE`, and `other` for - any other method. + request's `method` label is its method, for these nine: `GET`, `HEAD`, + `POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` and `TRACE`. Any + other method is labelled `other`. -- **Logging**: stdlib slog +- **Logging**: the standard library's `slog`, as JSON on standard output, or as + text when standard output is a terminal. ## Entrypoints This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) -standard: normalized scripts in `script/` are the entrypoints for the -development workflow, and the Makefile targets are thin shims that call them. We -provide: +standard. The scripts in `script/` are the entrypoints for the development +workflow, and most `Makefile` targets are thin shims that call them. We provide: -- `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and - download the Go modules (idempotent); with `--cgo`, the C compiler and the - libvips (with its JPEG XL support) and libheif libraries that compiling and - testing pixa need instead of Node, Yarn and prettier -- `script/setup` — make a fresh clone ready for development (bootstrap, then - install-precommit) -- `script/projectname` — output the project name ("pixa") +- `script/bootstrap` — install `git`, `make`, Go, Node, Yarn and `prettier`, and + download the Go modules. It can be run again at any time. With `--cgo`, it + installs a C compiler, `pkg-config`, libvips (with its JPEG XL support) and + libheif instead of Node, Yarn and `prettier`. Compiling pixa outside Docker, + as `make build` and `script/bench` do, needs those. Docker itself must already + be installed. +- `script/setup` — make a fresh clone ready for development: `script/bootstrap`, + then `script/install-precommit`. +- `script/projectname` — output the project name (`pixa`). - `script/test` — run the test suite: build the `test` phase of the - `Dockerfile`, writing no image -- `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`, - writing no image; the linter never runs on the host -- `script/fmt` — format the Go code with gofmt and the markdown with prettier - (writes) -- `script/fmt-check` — check the same formatting (read-only), on the host -- `script/check` — run test, lint, and fmt-check + `Dockerfile`, writing no image. +- `script/lint` — run `golangci-lint`: build the `lint` phase of the + `Dockerfile`, writing no image. The linter never runs on the host. +- `script/fmt` — format the Go code with `gofmt` and the markdown with + `prettier` (writes). +- `script/fmt-check` — check the same formatting (read-only), on the host. +- `script/check` — run `script/test`, `script/lint` and `script/fmt-check`. - `script/docker` — build the Docker image tagged via `script/projectname`, with - the version from `git describe`; the image's build stage depends on the `lint` - and `test` phases, so this runs them too -- `script/docker-smoke` — build the image, start it, wait for it to be healthy -- `script/loadtest` — measure pixad's throughput, latency and peak memory; a - benchmark, not part of `script/check` (see Load Test) + the version from `git describe`. The image's build stage depends on the `lint` + and `test` phases, so this runs them too. +- `script/docker-smoke` — build the image, start it, and wait for it to be + healthy. +- `script/loadtest` — measure `pixad`'s throughput, latency and peak memory. It + is a benchmark, not part of `script/check` (see Load Test). - `script/bench` — run the Go benchmark that serves cache hits in parallel - through the image handler, with Go on the host after `script/bootstrap --cgo`; - a benchmark, not part of `script/check` (see Load Test) -- `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`), - then `script/check`, then build the image as `script/docker` does -- `script/precommit` — pre-commit checks (`go mod tidy` guard, then - `script/check`) + through the image handler, with Go on the host after `script/bootstrap --cgo`. + It is a benchmark, not part of `script/check` (see Load Test). +- `script/cibuild` — the CI entrypoint: run `script/bootstrap` (without + `--cgo`), then `script/check`, then build the image as `script/docker` does. +- `script/precommit` — run `go mod tidy` and fail when it changes `go.mod` or + `go.sum`, then run `script/check`. - `script/install-precommit` — install the git pre-commit hook that runs - `script/precommit` + `script/precommit`. Every `docker build` in these scripts passes a new value of the `CACHEBUST` -build argument, which every stage of the `Dockerfile` declares, so every `RUN` -step after that line, the lint and test steps included, runs on every build -instead of coming from the build cache. Above the line are the steps that -install system packages and download the Go modules; Docker keeps their layers -until the base image, the step itself, `go.mod` or `go.sum` changes. Go's build -cache is kept from build to build: the steps that compile mount it, so a package -whose inputs are unchanged is not compiled again, and both `go test` runs pass -`-count=1` so that no stored test result is replayed. The lint step also mounts -golangci-lint's cache, which reuses a package's findings only when its files, -those of the packages it imports, `go.mod`, the linters and their settings in -`.golangci.yml`, and the golangci-lint version are all unchanged. The `test` -phase runs on the Debian Go image, with Debian's libvips and libheif; the image -pixa ships is built and runs on Alpine. `script/check`, `script/cibuild`, -`script/docker`, `script/lint`, `script/test`, `script/setup` and -`script/install-precommit` are the standard copies from `sneak/prompts`, kept -identical to them. `script/fmt` and `script/fmt-check` are the standard copies -with pixa's `gofmt` step kept before prettier. prettier formats the markdown -only, not the HTML templates, as it cannot parse a Go template action inside a -tag. +build argument. Every stage of the `Dockerfile` declares it with an +`ARG CACHEBUST` line. So every `RUN` step after that line, the lint and test +steps included, runs on every build instead of coming from the build cache. The +steps above that line install system packages and download the Go modules. +Docker keeps their layers until the base image, the step itself, `go.mod` or +`go.sum` changes. + +Go's build cache is kept from build to build: the steps that compile mount it, +so a package whose inputs are unchanged is not compiled again. The test step +runs `go test`, and reruns it with `-v` after a failure. Both runs pass +`-count=1`, so no stored test result is replayed. The lint step also mounts +`golangci-lint`'s cache. That cache reuses a package's findings only while its +files, its imports, `go.mod`, `.golangci.yml` and the `golangci-lint` version +are unchanged. + +The `test` phase runs on the Debian Go image, with Debian's libvips and libheif. +The image pixa ships is built and runs on Alpine. + +These scripts are the standard copies from `sneak/prompts`: + +- `script/check` +- `script/cibuild` +- `script/docker` +- `script/lint` +- `script/test` +- `script/setup` +- `script/install-precommit` + +`script/fmt` and `script/fmt-check` are the standard copies with pixa's `gofmt` +step kept before `prettier`. They also run `yarn` under the Node version that +`script/bootstrap` pins. `prettier` formats the markdown only, not the HTML +templates, as it cannot parse a Go template action inside a tag. ## Load Test -`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how +`script/loadtest` (or `make loadtest`) measures how fast `pixad` answers and how much memory it uses. It is a benchmark, not a check: `script/check` does not run it. It needs Docker and Go. @@ -1053,65 +1213,63 @@ script/loadtest # 10 seconds per scenario, 4 clients script/loadtest 30s 32 # 30 seconds per scenario, 32 clients ``` -It builds the image with `script/docker` and the load tool, -[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario -starts a new pixad container and a new origin container, `cmd/loadtest-origin`: -an upstream host that answers every path with the same generated 1600x1200 JPEG. -vegeta then sends requests from the given number of clients, each sending its -next request as soon as its last one is answered, all for an image resized to -400x300 WebP: +It builds the image with `script/docker`. It also builds the load tool, +[`vegeta`](https://github.com/tsenart/vegeta), from a pinned commit. Each +scenario starts a new `pixad` container and a new origin container, +`cmd/loadtest-origin`. The origin is an upstream host that answers every path +with the same generated 1600x1200 JPEG. `vegeta` then sends requests from the +given number of clients. Each client sends its next request as soon as its last +one is answered. Every request asks for an image resized to 400x300 WebP: - `hit`: the same image every time, put in the cache first; -- `miss`: a new source image every time, so pixad fetches and converts each one; -- `herd`: each new source image once per client in a row, so that all clients - ask for it at the same time and share one fetch and one conversion (see - Routes). +- `miss`: a new source image every time, so `pixad` fetches and converts each + one; +- `herd`: every client asks for each new source image at the same moment. The + clients then share one fetch and one conversion (see Limits). -pixad refuses upstream hosts with private or local addresses, so the containers -share a Docker network in `203.0.113.0/24`, a range set aside for documentation. -A second run on the same Docker host while one is going fails, as it cannot -create that network. +`pixad` refuses upstream hosts with private or local addresses, so the +containers share a Docker network in `203.0.113.0/24`, a range set aside for +documentation. A second run on the same Docker host while one is going fails, as +it cannot create that network. -For each scenario the script prints vegeta's report and two lines of its own: +For each scenario the script prints `vegeta`'s report and two lines of its own: - `Requests [total, rate, throughput]`: the requests sent, how many were sent - per second, and how many were answered successfully per second; the last is - the number to compare with the target under Storage; + per second, and how many were answered successfully per second; - `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request - to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th - percentiles; + to the end of its answer. `50`, `90`, `95` and `99` are the 50th, 90th, 95th + and 99th percentiles; - `Status Codes` and `Error Set`: anything other than `200` means the other - numbers are not for the scenario described, such as `503` when pixad was busy; -- `Bytes In`: `0`, as vegeta is told not to keep the images it receives; -- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since - its container started, in kB; for `hit` it includes the request that put the - image in the cache; -- `requests to the origin`: the fetches pixad made: one for `hit`, one per - request for `miss`, and one per image for `herd`, that is the requests sent + numbers are not for the scenario described, such as `503` when `pixad` was + busy; +- `Bytes In`: `0`, as `vegeta` is told not to keep the images it receives; +- `pixad peak memory (VmHWM)`: the peak resident memory of the `pixad` process + since its container started, in kB; for `hit` it includes the request that put + the image in the cache; +- `requests to the origin`: the fetches `pixad` made. That is one for `hit`, one + per request for `miss`, and one per image for `herd`: the requests sent divided by the number of clients. -The numbers depend on the machine and on whatever else runs on it. The first -measurement, made on a shared machine with few clients, is in `TODO.md`; it says -nothing about the target. +The numbers depend on the machine and on whatever else runs on it. `script/bench` (or `make bench`) measures cache hits alone, without the network -and with no pixad process: a Go benchmark in `internal/handlers` serves one -cached image through the `/v1/image/` handler from as many goroutines at once as -the machine has CPUs, and reports the hits served per second as `hits/s`. Each -run starts with an empty cache and an empty database, a SQLite file of its own, -and puts the image in the cache before it measures. The benchmark fails if -serving the hits ran any statement on the database. Like the load test it is not -part of `script/check`. It runs Go on the host, as the load test does, and -compiles pixa there, so it needs the C compiler and libraries that +and with no `pixad` process. A Go benchmark in `internal/handlers` serves one +cached image through the `/v1/image/` handler from several goroutines at once, +one per CPU. A goroutine is a Go thread. The benchmark reports the hits served +per second as `hits/s`. Each run starts with an empty cache and an empty +database, a SQLite file of its own, and puts the image in the cache before it +measures. The benchmark fails if serving the hits ran any statement on the +database. Like the load test it is not part of `script/check`. It runs Go on the +host and compiles pixa there. So it needs the C compiler and libraries that `script/bootstrap --cgo` installs, but not Docker. ## TODO -See [TODO.md](TODO.md) for the full prioritized task list. +See [`TODO.md`](TODO.md) for the full prioritized task list. ## License -GPL-3.0. See [LICENSE](LICENSE). +GPL-3.0. See [`LICENSE`](LICENSE). ## Author From 1379497897510e858a38a0ac72250bae578784c7 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 13:07:16 +0100 Subject: [PATCH 120/131] Keep only open work in TODO.md (closes #348) (#349) --- TODO.md | 1389 +------------------------------------------------------ 1 file changed, 9 insertions(+), 1380 deletions(-) diff --git a/TODO.md b/TODO.md index 8416fe5..9c99a32 100644 --- a/TODO.md +++ b/TODO.md @@ -2,7 +2,7 @@ - branch per issue from `next` - do the work in Next Step -- move Next Step to the top of Completed Steps +- delete Next Step from `TODO.md` - `TODO.md` merges with git's union merge (`.gitattributes`), which never reports a conflict: read the merged entries after every merge or rebase - move the top item of Future Steps into Next Step @@ -16,1394 +16,23 @@ # Status -pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work lands on -`next`, and `main` receives only the milestone PR that the owner merges. `next` -is at the canonical `golangci-lint` v2.14.0 config and is green. Recent work -extracted the internal/magic, internal/allowlist, internal/httpfetcher, and -internal/signature packages. The gosec findings from the 2026-07-06 survey are -resolved. The disk cache is now size-bounded with LRU eviction -(`cache_max_bytes`), closing the unbounded disk growth DoS vector. +pre-1.0. The `1.0.0` milestone is in progress. # Next Step P2: security: per-origin rate limiting -# Completed Steps - -- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file - whose info it cannot read, and goes on (closes #343). It stopped before its - later checks at a directory it could not list, or at a source or variant file - whose info it could not read. - - It logs the warning "skipped unreadable entry in cache directory", with - the path and the error. - - A file renamed or removed after its directory was listed gets no warning. - - `reconcileFiles` reads each file's info once, before the temp-file check, - and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or - `reconcileSourceFile`. - - A cancelled context still stops the walk with its error. -- 2026-10-26 the reconciliation pass drops a variant's accounting row only when - its file is missing (closes #340). Any other error reading the file's size - keeps the row. It dropped the row on any error and logged the file as missing. - - The kept row gets the warning "kept accounting row for unreadable variant - file", with the variant key and the error. - - The pass checks for `ErrNotFound`, which `FileStore.Size` returns for a - missing file. -- 2026-10-26 on both image routes, a request is answered 499 and logged at info - level when its client goes away before its image is ready, unless it is the - request making the image (closes #339). It was answered 500 and logged at - error level, as pixa's own failure. - - The line is "client went away before the image was ready". It carries the - `request_id`, the host and the path. - - The 499 has no body, since nobody reads it. - - `serveImage` answers it before calling `respondImageError`, which cannot - see the request's context. - - An error wrapping `context.Canceled` while the request's context is live - is still answered 500 and logged at error level. -- 2026-10-26 `README.md`'s Design section starts with a Concepts list (closes - #321). Each concept gets one sentence, under the name the code gives it. - - Storage calls the transformed images variants, in shorter sentences. - - Storage's paths end in the source key and the variant key. - - Storage lists the image cache's tables in the database. - - Architecture names `internal/providers`, and lists the packages that work - with the concepts. -- 2026-10-26 one table in `internal/config` lists every setting once (closes - #320). `Config.settings` gives each setting's key, its environment variable - and the function that loads it. - - Most rows load through `field`, whose arguments give the setting's type, - its default and its check. - - `db_url` and `cache_max_bytes` have their own load functions. The default - of `db_url` comes from `state_dir`, and `cache_max_bytes` records whether - it is set. - - Loading, the unknown-key check, the unknown-variable check and the names - in error messages all read the table. - - `allowlist_hosts` loads as `referer_blocklist` does, through - `parseHostList`. - - The metrics username and password are checked together once the table has - loaded. - - When a config holds more than one invalid value, the error names the one - that comes first in the table. - - `README.md` lists the settings one item each, where it had a table. A test - checks that this list and `configs/config.example.yml` name exactly the - keys of the table. - - The key constants moved to the tests, their only users. - - Removed: - - `isKnownConfigKey`, `envVarNames`, `strictLoader` and `validate` - - `validateAllowlistHostsValue` and `getStringSlice`, with the three - tests of `getStringSlice` -- 2026-10-26 a fetched image's format is detected from its bytes once, as an - `imagerequest.Format` (closes #319). The upstream `Content-Type` check in the - fetcher stays as the filter before downloading. - - `magic.DetectFormat` returns an `imagerequest.Format`, and - `magic.ValidateMagicBytes` returns the format it checked against the - `Content-Type`. - - `Format.MIMEType` gives each format's MIME type. The fetcher, the mock - fetcher, the cache's read of a variant's type and the handler's `Accept` - matching use it. - - `Cache.Source` returns a source's format, read from its file's first bytes - as `Cache.Variant` reads a variant's. A file in no source format is an - error, and the proxy fetches the source again. - - The proxy hands the format to `ImageProcessor.Process`. The processor - refuses input that libvips would load as another format, as before, with - `ErrUnsupportedInputFormat`. It refuses a format that is not one of the - six source formats the same way. - - `internal/magic` no longer detects SVG, which pixa does not read. SVG data - is `ErrUnknownFormat`. - - Removed: - - `magic.MIMEType`, `magic.ImageFormat` and `magic.MIMEToImageFormat` - - `detectFormat`, `formatFromString`, `FormatToMIME` and - `Result.InputFormat` of the processor - - the fetcher's MIME constants of the six formats - - `errNotAnOutputFormat` of the cache -- 2026-10-26 the cache keeps its source images in one `imgcache.FileStore` and - its variants in another (closes #318). A `FileStore` is a directory of files, - each named by a hex key, at `//`. - - `ContentStorage` and `VariantStorage` are gone. - - `FileStore` has these methods: - - `Write`, which writes a file under a temp name, syncs it and renames - it into place - - `Open`, which returns the file and its size - - `Size` - - `Delete` - - `FileStore` has no `Exists`. The reconciliation pass checks that a - variant's file is there with `Size`. The tests look for the file on disk. - - `Cache.Variant` reads the content type from the variant file's first - bytes, as `VariantStorage.LoadWithContentType` did. A cache hit still - makes no database call. - - `storage.go` is now `filestore.go`. `internal/imgcache/testdata/red.avif`, - which no test used, is gone. -- 2026-10-26 `internal/imgcache` holds the cache only (closes #317). The new - package `internal/imageproxy` fetches, transforms, caches and serves images. - It imports `imgcache`, and `imgcache` does not import it. - - `imgcache.Service` is `imageproxy.Proxy`, built by `imageproxy.New` from - an `imageproxy.Config`. - - `ImageResponse`, `CacheStatus`, `ErrNegativeCached` and - `ErrProcessingTimedOut` moved with it, under their names. - - `imgcache.CacheConfig` is `imgcache.Config`. - - The cache has one read and one write for each kind it keeps. Each takes - the place of an earlier method: - - `Variant`, of `GetVariant` - - `PutVariant`, of `StoreVariant` - - `Source`, of `LookupSource` followed by `GetSourceContent` - - `PutSource`, of `StoreSource` - - `Failure`, of `checkNegativeCache` - - `PutFailure`, of `StoreNegative` - - `Variant` and `Source` each open the file and hand its access time to the - background writer. A cache hit still makes no database call. - - `imgcache.go` and `module.go` are gone. - - `imageproxy.New` makes the counters of upstream fetches and transforms, - and registers them on the registry it is given. The cache keeps only the - hit and miss counters. - - A cached source whose row cannot be read is logged as "failed to load - cached source, fetching". It was logged as "source lookup failed". - - The proxy's tests moved to `internal/imageproxy`. They reach the cache - through its database, its state directory and its metrics registry. - - The tests of an empty or cut cached file are split in two: - - `internal/imgcache` checks that the reconciliation pass removes the - file and its row, with a warning naming the file. - - `internal/imageproxy` removes the file and its row as the pass does, - and checks that `Get` fetches or converts again. -- 2026-10-26 both image routes serve an image with one function, `serveImage`, - and answer its errors with one, `respondImageError` (closes #316). - `HandleImage` parses the path and verifies the signature, and `HandleImageEnc` - decrypts the token. Each then calls `serveImage`. - - `handleImageError`, the encrypted route's own mapping, is gone. - - `/v1/image/` answers 504 when the upstream has not sent its response - headers within `upstream_fetch_timeout`, as `/v1/e/` did. It answered 500. - - `respondImageError` logs the image service's errors as "failed to get - image". Each answer gets the same level on `/v1/image/` and on `/v1/e/`: - - 500: error, as pixa's own failure - - 400: warn - - 403: warn - - 415: warn - - 502: warn - - 503: warn - - 504: warn - - Before, `/v1/image/` logged all of these answers at error level. `/v1/e/` - logged only its 500, as "image request failed", without the host and the - path. - - `respondImageError` does not log the 415 for a body refused for its first - bytes, on either route. The image service logs it as "source refused", at - warn level. - - The "image served" line carries the host, the path and `fetched_bytes` on - both routes. The debug line "encrypted image request" is gone. - - The debug line "failed to decrypt URL" is now "invalid encrypted URL". A - token that decrypts to a payload that does not decode also gets it. - - A failed write is logged once, as "failed to write response" at warn - level, since most often the client has gone away. It carries the - `request_id`, the host, the path, `served_bytes` and the error, and is not - also logged as "image served". -- 2026-10-26 every key derived from the signing key is derived in one place - (closes #315): `seal.DeriveKeys` returns them as one `seal.Keys`, with their - salts declared beside it. `internal/providers` calls it once at startup. Each - key is derived as before, with `seal.DeriveKey` and the same salt. - - The session manager, the encrypted URL generator and the CSRF protection - each take their own key instead of the signing key. - - `session.NewManager`, `encurl.NewGenerator`, `newCSRFProtect` and - `handlers.New` return no error, as nothing in them can fail now. - - The signing key itself stays the HMAC key of signed URLs and the login - password. - - `README.md` says under `signing_key` what it protects. -- 2026-10-26 the image request holds only what names the cached object (closes - #314): its source and its transform, which the variant key covers. The signed - URL format, the signing algorithm and every route's answers are unchanged. - - The `/v1/image/` handler checks `allowlist_hosts` and verifies the - signature itself, with `Signer.Verify(req, sig, expires)`. - - Each image handler keeps its URL's expiry as a value of its own, for - `Cache-Control`. `encurl`'s `Generate` takes the expiry and its `Parse` - returns it. - - The fetcher takes the source and builds its URL with `Source.URL`, over - http only when the fetcher's own `AllowHTTP` is set. `validateURL` takes - that URL, so no URL string is built and parsed again. - - `chooseAutoFormat` returns the format it chooses, and the handler puts it - into the request it parsed. - - The image service serves images through `Get`, and has `WaitForProcessing` - for shutdown. - - Removed: - - `Signature`, `Expires`, `AllowHTTP` and `SourceURL` of the image - request - - `Service.ValidateRequest`, and the service's signer, allowlist and - `AllowHTTP` - - `extractHost` of the fetcher and `urlToFSPath` of the mock fetcher -- 2026-10-26 pixad's fx graph builds the image pipeline (closes #313). - `database.New` now opens the database, so the parts below can read it while - they are built; its stop hook still runs after the cache's. The cache's - `MaxBytes` is the one value for the disk cache limit, and 0 turns the disk - cache off. For an omitted `cache_max_bytes`, `internal/providers` works out - `config.DefaultCacheMaxBytes` before it builds the cache. `internal/app` and - the server tests build the parts below with `providers.Constructors()`; the - image proxy test replaces only the fetcher's dial function. - - Built by the providers in `internal/providers`, and given to - `handlers.New`: - - the cache - - the fetcher - - the processor - - the image service - - the session manager - - the encrypted URL generator - - Removed: - - `initImageService` and `handlers.Params.Fetcher` - - `CacheConfig.UseDefaultMaxBytes`, `CacheConfig.DisableDiskCache` and - `Cache.disabled` - - `httpfetcher.DefaultConfig()` and the fetcher's copies of setting - defaults - - `imageprocessor.DefaultMaxInputBytes`, and the processor's defaults - for a zero `MaxInputBytes` or `MaxConcurrentProcessing` -- 2026-10-26 the code nothing in production called is gone (closes #312). The - handlers tests sign their image URLs with `generateSignedURL`, a test helper - that writes them as the removed generator did. Removed: - - `Service.Warm` and the `ImageCache` interface - - `Service.GenerateSignedURL` and `Signer.GenerateSignedURL` - - `signature.ParseParams` - - `imagerequest.ParseImagePath` - - `PeekAndValidate`, `IsSupportedMIMEType`, `ImageFormatToMIME` and - `FormatOriginal` of `internal/magic` - - `SupportedInputFormats` and `SupportedOutputFormats` of the image - processor - - `IsEmpty` and `Count` of the allowlist - - `ssrfSafeDialer` - - `CacheStale`, `ErrCacheMiss`, `ErrNegativeCache` and - `ImageResponse.LastModified` - - `ContentStorage.Load` and `VariantStorage.Load` -- 2026-10-26 cache hits, misses, upstream fetches, the bytes those fetches read - and transforms are counted as Prometheus counters in memory (closes #311). - They are `pixa_cache_hits_total`, `pixa_cache_misses_total`, - `pixa_upstream_fetches_total`, `pixa_upstream_fetch_bytes_total` and - `pixa_transforms_total`, fields of the cache. `NewCache` makes them and - registers them on the registry it is given (`internal/imgcache/counters.go`). - pixad gives it Prometheus' default registry, which `/metrics` serves; each - test gives it a new one. The `Increment` methods add to their cache's - counters. `cache_stats`, `Cache.Stats`, `Service.Stats`, `CacheStats` and the - pending counts are gone. The background goroutine writes only access times - now: `StartPendingAccessTimeWrites` starts it and - `StopPendingAccessTimeWrites` stops it, in `pendingaccesstimes.go`. -- 2026-10-21 the schema file declares only the tables and columns pixa writes - and reads (closes #310): `output_content` and `request_cache` are gone, and so - are the columns nothing read: the host, path, query, `content_type` and - `fetched_at` of `source_content`, the `content_type` and `created_at` of - `variant_content`, and the `status_code`, `error_message` and `fetched_at` of - `negative_cache`. `StoreSource` no longer takes the fetch result, - `LookupSource` returns only the source key, `StoreVariant` no longer takes a - content type, `StoreNegative` no longer takes a status code or message, and - reconciliation adopts a variant file without reading its first bytes. -- 2026-10-21 a cache hit makes no database call (closes #309): `Service.Get` - opens the variant file with `Cache.GetVariant` and nothing else, with no - existence check first. The content type comes from the file's first bytes; a - file whose first bytes are not those of an output format, SVG included, is a - miss, and the variant is made again. A hit hands its access time to a channel - that holds up to 10,000 and drops the rest, and its hit count to the pending - counts, without waiting; every count is now a pending count. A miss that finds - a cached source hands that source's access time to a second such channel. The - goroutine `StartPendingCountWrites` starts writes, once a second, the access - times in one transaction and the counts in one `UPDATE`; - `StopPendingCountWrites` writes what is left once at shutdown and logs - separately the access times and the counts it could not write. - `checkNegativeCache` no longer deletes an expired row; `CleanExpired` in the - eviction goroutine does. The `.meta` files, `VariantMeta`, `LoadWithMeta`, - `variantContentTypeFromSidecar`, `metaCache`, `Lookup` and `golang-lru` are - gone. `make bench` (`script/bench`) runs `BenchmarkHandleImage_CacheHits` with - Go on the host after `script/bootstrap --cgo`, outside `make check`. -- 2026-10-21 a source is stored the way a variant is (closes #308): one - `source_content` row and one file per source URL, the file named by the - source's `SourceKey`, the SHA-256 of its host, path and query made as the - variant key is, under `cache/sources/`. A store writes the file, then the row, - and removes the file when the row cannot be written; an eviction deletes the - row, then the file, with no lock and no transaction. `source_metadata`, the - JSON files under `cache/metadata/`, `MetadataStorage` and the content lock are - gone. Two source URLs that serve the same bytes now store them twice. -- 2026-10-21 one image request type instead of five (closes #307): the new - package `internal/imagerequest`, which imports no other package of pixa, holds - `imagerequest.Request` with its two parts, `Source` (host, path and query) and - `Transform` (size, format, quality and fit), the `Format` and `Fit` types, the - checks, the parsing of the `/v1/image/` path and the variant key. - `imgcache.ParsedURL`, `imgcache.ImageRequest`, `signature.Request`, - `imageprocessor.Request`, the second `Size`, format and fit types and the - conversions between them are gone. The image processor takes a `Transform` - only, and `encurl` keeps a private struct with the token's short CBOR field - names, so tokens made before still decode. Every URL of both image routes - answers as before. -- 2026-10-21 the Gitea workflow checks out the repo and runs `script/cibuild`, - nothing else (closes #203): its `script/docker-smoke` step is removed, as the - canonical `REPO_POLICIES.md` says, since `make check` covers what CI needs to - check. CI no longer starts the image and waits for its healthcheck. - `script/docker-smoke` and `make docker-smoke` stay, to run by hand. -- 2026-10-20 a WebP, AVIF or JPEG XL encode at the limits takes at most about 34 - seconds (closes #272): with one libvips thread, 8192x8192 random pixels with - an alpha channel took about 61 seconds to encode as WebP, 81 as JPEG XL and, - at quality 85, 96 as AVIF, so with the default `downstream_timeout` of 60 - seconds such an image was never delivered, yet its encode held a processing - slot to the end. WebP is now saved at effort 2 and JPEG XL at effort 4, which - take up to about 23 and 27 seconds at any quality. AVIF was already at the - lowest effort govips can set, so an AVIF output of more than 16,777,216 pixels - is refused with 400 before anything is decoded; 4096x4096, 8192x2048 or - 2048x8192 takes up to about 34 seconds at any quality. Resizing with - `fit=cover` already decodes the input, so the output's dimensions are worked - out before resizing. `README.md` gives the limit and the new times. -- 2026-10-20 a source whose first bytes do not match its `Content-Type` answers - 415 (closes #253): a body whose first bytes are not those of the format its - `Content-Type` names, such as a JPEG sent as `image/png`, text, or a body - under 12 bytes, answered 500 on both image routes and was logged as an error - of pixa's, and each request for it fetched the whole body again. Both routes - now answer 415, and the image service logs it at warning level as a refused - source. It also gets a negative cache entry, as an upstream error status does, - so the source URL is not fetched again for 5 minutes, and meanwhile an image - that would need that fetch answers 502. -- 2026-10-20 `AGENTS.md`: a bug fix may change an existing test without asking - when that test pins exactly the behavior the filed bug calls wrong, and the - change moves it to the fixed behavior and nothing else (closes #280). -- 2026-10-18 re-vendored from `sneak/prompts` commit `0653579` (closes #221): - each `Dockerfile` stage installs its system packages and downloads the Go - modules above `ARG CACHEBUST`, so check builds keep them, and the lint step - keeps golangci-lint's cache. -- 2026-10-19 a `trusted_proxies` entry in the IPv4-mapped form trusted nothing; - it is now read as the IPv4 range it covers by the same code that reads - `blocked_networks`, and such an entry shorter than `/96` aborts startup - (closes #303). -- 2026-10-18 the next reconciliation pass waits at least ten times as long as - the last one took, so passes on a large cache no longer run back to back - (closes #250). -- 2026-10-18 a `blocked_networks` entry in the IPv4-mapped form refused nothing; - `::ffff:10.0.0.0/104` is now read as `10.0.0.0/8`, and such an entry shorter - than `/96` aborts startup (closes #301). -- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal - addresses refuses the local-use NAT64 range (`64:ff9b:1::/48`, RFC 8215) as it - refuses `64:ff9b::/96`, and refuses an IPv6 address in the IPv4-compatible - form (`::/96`) when `blocked_networks` contains either the address as written - or the IPv4 address it holds (closes #299). A source host such as - `[64:ff9b:1::a00:1]` was fetched, which on a network that runs NAT64 on that - range reaches the IPv4 address it holds, here `10.0.0.1`; and with - `203.0.113.0/24` in `blocked_networks`, `[::ffff:203.0.113.5]` was refused but - `[::203.0.113.5]` was fetched. Both now answer 403. The mapped form - (`::ffff:a.b.c.d`) is still matched only as the IPv4 address it holds: Go - keeps a plain IPv4 address in that form, and an IPv6 entry such as `::/0` must - not refuse every IPv4 address. -- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal - addresses judges an IPv6 address in the IPv4-compatible form (`::/96`), such - as `::127.0.0.1` or `::10.0.0.1`, by the IPv4 address in its last four bytes - (closes #297). It read only the mapped form (`::ffff:a.b.c.d`) as IPv4, so a - source host such as `[::127.0.0.1]` was checked as a public IPv6 address and - fetched, which on a network that routes it reaches the internal IPv4 host; it - is now refused with 403. The NAT64 form (`64:ff9b::/96`) was already refused - as a whole, as one of the built-in blocked ranges, and still is. -- 2026-10-16 a source host that is an IPv6 address in brackets with no port, - such as `[2001:db8::a]`, is fetched like one with a port, through `/v1/image/` - and through an encrypted URL (closes #295). The fetcher's check of the source - URL kept the brackets when no port followed them, could not look the host up, - and the request answered 500. It now checks the address without brackets, so a - private one is refused with 403, as with a port. -- 2026-10-16 a `state_dir` whose name holds `%`, `?` or `#` keeps its database - at `state.sqlite3` in that directory (closes #290): the `db_url` derived from - it put the name into a `file:` URL as it was, so SQLite read `%41` as `A`, a - `?` or `#` ended the path early, and the journal mode parameter was lost. The - derived `db_url` is now built with Go's `net/url`, which writes those - characters as `%25`, `%3F` and `%23`. For a `state_dir` of `/` or one starting - with `//` it starts `file:////`, so SQLite no longer reads the first name in - the path as a host. -- 2026-10-16 a size with one side `0` keeps the side computed from the source's - proportions at 1 or more (closes #283): it was rounded down, so for a thin - source it came out at 0, and libvips then made the size square. An 8192x1 - source at `100x0` came out at 100x100, and at `1300x0` with `fit=cover` made - libvips ask for about 39 GB at once; an 8192x2 source at `100x0` answered 500. - `fit=contain` and `fit=inside` work both sides out again from the smaller of - the two scales and `fit=outside` from the larger, and each keeps both sides at - 1 or more too, so a 1x8192 source at `0x100` with `fit=contain`, and a 49x8192 - source at `0x100` or a 49x49 source at `1x1` with `fit=outside`, no longer - answer 500. An 8192x1 source at `1300x0` now comes out at 1300x1, and the - process running it alone peaked at about 34 MiB resident. With `fit=outside` - the image is scaled to cover the size, so that source at `100x0` comes out at - 8192x1. `README.md` now says this under Signature Specification, and says how - a `0` side is worked out where it said `0` meant the original. -- 2026-10-16 the generator page lower-cases the source host and writes an empty - source path as `/` before it makes the encrypted URL, as `/v1/image/` does - (closes #293). A host written in upper and in lower case, or a source URL with - no path and with the path `/`, made two cache keys and two upstream requests - for one image. -- 2026-10-16 the source URL a request fetches is built with `net/url` from the - source host, path and query, not joined as text (closes #288). The source - path, decoded, is fetched percent-encoded again, so `100%25.png` fetches - `/100%25.png` where it answered 401, and `my%20cat.png` and `my%2520cat.png` - fetch different paths where both fetched `/my%20cat.png` under two cache keys. - The source query stays as the source URL writes it; the characters a browser - percent-encodes in a query, such as `'` or a space, which was sent raw and - answered 502, are percent-encoded when the image URL is read, so the query is - signed and cached as it is sent. The generator page answers 400, showing the - URL pixa would fetch, for a source URL its image URL would fetch written - another way, such as with `%2F` in the path, whatever else the path holds, or - with a user name and password. It also answers 400 for a source path holding a - `..` segment, as `/v1/image/` refuses one. -- 2026-10-15 a thin source requested with `fit=cover` no longer makes libvips - build a huge image (closes #282): libvips' thumbnail scaled the whole source - to cover the requested size and copied that into memory before it cut out the - centre, so an 8192x64 source at 8192x8192 made it allocate about 24 GiB, and - an 8192x1 source answered 500. The image processor now first cuts the centre - of the source, widened to whole source pixels, and thumbnail scales and cuts - only that, by the same amount on both sides as before. What it copies into - memory is then larger than the output by less than one scaled source pixel on - each side. With the runtime image's libvips and JPEG output at 8192x8192, - resident memory peaked at about 242 MiB for the 8192x64 source and 431 MiB for - the 8192x1 one. The output can sit a fraction of an output pixel from where it - was. -- 2026-10-15 a `/v1/image/` URL whose source path or source query holds a `#`, - written as `%23`, is refused with 400 (closes #286): the source URL built from - it ended at the `#` when fetched, so each text after it fetched the same - upstream image, without a signature for an allowlisted host, and cached it - under keys of its own. `README.md` says neither may hold a `#`. -- 2026-10-15 the host of a `/v1/image/` URL must be a host name or an IPv4 - address (letters, digits, hyphens, underscores and dots) or an IPv6 address in - brackets, either one optionally followed by `:` and a port, and any other host - is refused with 400 (closes #275): only `@` was refused, so a `?` written as - `%3F` gave the host `cdn.example.com?x=1`, which Go's URL parser reads as - `cdn.example.com` with a query. Such a URL passed the allowlist, and each text - after the `?` fetched the host's root path and cached the answer under keys of - its own. A `#`, `%`, `\` or space written percent-encoded is refused the same - way, and so is an IPv6 address without brackets, which needed a signature - before. Underscores are kept because `allowlist_hosts` accepts host names - holding them. -- 2026-10-15 the README says how a source URL with a query is written into an - image URL, and the image route accepts it (closes #255): the query follows the - source path after `%3F`. pixa now reads the image URL's path decoded once, so - the `?` before the query, any `?` in it and every `%` in the source path or - query must be percent-encoded, and any other character may be. The route used - to parse chi's wildcard, which holds the path as the client wrote it whenever - that is not Go's own encoding of it; a `/`, `;` or `,` in the query written as - `%2F`, `%3B` or `%2C`, as `GenerateSignedURL` wrote them, kept the `%3F` from - being read as `?`, and the request failed with 401. `GenerateSignedURL` now - writes Go's own encoding of the path. The README has an example with a query - and its signature. The path traversal check no longer reads the source query, - and keeps a `%` that does not start a percent-escape as it is instead of - refusing the URL, so a source path or query holding `%25` is accepted, while a - double-encoded `..` next to such a `%` is still refused. -- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees, - rounded down, and at least 1 (closes #242): it defaulted to one image per CPU, - and a container with no CPU limit sees every CPU of the host, so on a 48-CPU - host pixa processed 48 images at once, each of which can take about 4.2 GiB. - `config.DefaultMaxConcurrentProcessing` gives the default to the config and to - `imageprocessor.New`; `PIXA_MAX_CONCURRENT_PROCESSING` still overrides it. - "Running under upaas" in `README.md` gives the worst-case memory, measured, - and tells the operator to set `PIXA_MAX_CONCURRENT_PROCESSING` and - `PIXA_UPSTREAM_CONNECTIONS` to fit the container's memory. It also gives the - file libvips writes to `/tmp` for each large source, up to 1 GiB, measured, - which is outside `/var/lib/pixa` and `cache_max_bytes` and is memory where - `/tmp` is a tmpfs. -- 2026-10-15 the canonical files are re-vendored from `sneak/prompts` commit - `e3f79c4` (closes #221): `.dockerignore`, `.editorconfig`, - `.gitea/workflows/check.yml`, `.gitignore`, `.golangci.yml`, - `.prettierignore`, `.prettierrc`, `REPO_POLICIES.md` and the standard scripts - are its copies. pixa's own entries follow the standard part: `/bin`, `/data` - and the local config files in `.dockerignore`; Go's `*.log`, `*.out` and - `*.test`, `vendor/`, the binaries, `/data/`, `*.sqlite3` and the local config - files in `.gitignore`. The workflow keeps its `script/docker-smoke` step while - https://git.eeqj.de/sneak/pixa/issues/203 is open. The scripts pass a new - `CACHEBUST` build argument in place of `--no-cache`, and every `Dockerfile` - stage declares it first, so every `RUN` step still runs on every build while - Go's build cache, mounted on the steps that compile, is kept; both `go test` - runs keep `-count=1`. The lint and test builds write no image. The lint phase - runs golangci-lint v2.14.0, the test phase runs on the Debian Go image, and - the build stage marks `/src` safe for git. `script/bootstrap`, `script/fmt` - and `script/fmt-check` use the node on the `PATH` only when it has the pinned - major version. Of golangci-lint v2.14.0's two findings, an unused `nolint` - directive is removed, and gosec's new path check is marked as not applying to - the fixed config file locations. The agent guidance moves unchanged from - `CLAUDE.md` to a root `AGENTS.md`, where `REPO_POLICIES.md` keeps it. -- 2026-10-15 the request metrics record any method that is not a standard one as - `other` (closes #274): their `method` label was each request's method as sent, - and Go's HTTP server accepts any method name, so a client sending made-up - methods added series to the Prometheus registry that were never removed. The - label is now the request's method only for `GET`, `HEAD`, `POST`, `PUT`, - `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` and `TRACE`. The metrics library reads - the method from the request it is handed, so the metrics middleware hands it a - copy of the request carrying the label; the handlers still see the method the - client sent. -- 2026-10-15 logging out ends every login session (closes #256): a login session - was a cookie with nothing kept on the server, so `/logout` removed it from one - browser and a copy of it kept working for its 30 days. Every login is made - with the one signing key, so logging out now ends them all: the session - manager keeps a random value, made with `crypto/rand` when the new `logout` - table has none and replaced by each logout, saves it there so that it outlasts - a restart, puts the value at login in each session, and refuses a session - carrying another value. The check for a logout compares no time, so a login - after a logout works whatever the host's clock does; a session's 30 days are - still checked against the clock. Losing the state directory ends every login - session, and no logout can make an ended one work again. Logging out is now - `POST /logout` from a form on the generator page, behind the CSRF check the - other forms have, so a link or image elsewhere on the same site cannot end the - sessions. A logout without a valid login session ends none and only removes - the cookie, as anyone can take a CSRF token from the login page. A logout that - cannot be saved answers 500. `session.NewManager`, which the tests use, keeps - the value in memory only; pixad uses `session.NewManagerWithDatabase`. -- 2026-10-15 a cached file left empty or partial is no longer served, or fetched - again on every request (closes #252). Source and transformed image files, and - each transformed image's `.meta` file, are synced to disk before they are - renamed into place, and their directory after; the `.meta` file is written - before the image, and a failed directory sync removes the renamed file. The - reconciliation pass, at startup and every five minutes, removes a cached - source or transformed image whose file is empty, or not the size the database - records for it, with its rows and a warning naming the file, so the next - request fetches the source or converts the image again; requests compare no - sizes. Storing a source over a file of another size replaces that file. A - source that fails to decode is not removed for that alone: a damaged upstream - image would then be fetched on every request. -- 2026-10-15 a transcode stops once its request's `downstream_timeout` has - ended, unless it has begun encoding (closes #243): the image processor looked - at its context only while it waited for a processing slot, so an image was - decoded and encoded, holding the slot, however long after its request's - deadline that took. `Process` now returns the context's error, and gives the - slot back, when the context has ended by the time it has the slot or before it - encodes. A request that waited for a shared transcode stopped at the first - request's deadline is answered 503 `server busy, try again later`, not 500, - though its own `downstream_timeout` may not have ended. govips gives pixa no - way to stop libvips once an encode has begun, so that encode keeps its slot - until it finishes; the limit of 8192 on each side of the source and of the - output bounds how long that takes. With one libvips thread, 8192x8192 random - pixels take up to about 1 second to encode as JPEG, 15 as PNG, 16 as GIF, 63 - as WebP, 110 as AVIF and 96 as JPEG XL, the last three with an alpha channel, - plus up to about 11 seconds to decode the source; `README.md` gives these - times and no longer says the work stops when `downstream_timeout` ends. The - settings of the WebP, AVIF and JPEG XL encoders, which can hold a slot past - the default `downstream_timeout` of 60 seconds, are unchanged. -- 2026-10-15 "Running under upaas" in `README.md` tells the operator to set - `PIXA_CACHE_MAX_BYTES` (closes #251): it listed the variable as optional, but - under upaas the volume is a directory on the host's filesystem. The default, - 75% of the sum of that filesystem's free space and what the cache holds, and - at least 500 MiB, let one pixa take three quarters or more of the space the - host and every other app had left, and all of it when that was 500 MiB or - less. The section now says so. The default is unchanged. -- 2026-10-15 the request metrics are labelled by route, not by path (closes - #254): their `handler` label was each request's path, so every new path, a 404 - or an encrypted URL, added series to the Prometheus registry that were never - removed, and `/metrics` listed every encrypted URL's token. The label is now - the pattern of the chi route the request matches, such as `/v1/image/*`, or - `unmatched` for a request that matches none: one the router answers 404 or - 405, or a CORS preflight. The metrics library takes the label before the - request is routed, so the metrics middleware asks the router which route - matches. -- 2026-10-15 expired negative cache rows are removed on a schedule (closes - #244): the row of a failed fetch, which makes pixa answer 502 for that source - URL for 5 minutes, was deleted only when the same URL was requested again - after that, so a client asking for a new missing path each time added rows - that stayed for good. The eviction goroutine now calls `CleanExpired` at - startup and on each periodic tick, every 5 minutes in pixad, so a row stays at - most about 10 minutes. `CleanExpired` deletes at most 1000 rows per statement, - so a request waits behind 1000 deletions at most, however many rows have - expired. The goroutine runs with the disk cache disabled - (`cache_max_bytes: 0`) too, where it does nothing else, as the negative cache - is used either way. -- 2026-10-15 a cached variant is served while its source URL is in the negative - cache (closes #245): `Get` checked the negative cache before the variant - cache, so once a fetch of the source failed, as one does when another variant - misses after the source was evicted and the upstream is down, every variant of - that source answered 502 for 5 minutes, also one on disk or one that could be - made from the cached source. The negative cache is now checked only just - before a fetch from the upstream, so a cached variant is served and a variant - not in the cache is made from the cached source when there is one. `README.md` - says so under `/v1/image/`. A refusal from the negative cache now counts as a - miss, as any other request that misses the variant cache does. -- 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port - needs a signature (closes #246): with `cdn.example.com` allowlisted, - `CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and - each spelling was fetched and cached on its own and had connections of its own - under `upstream_connections_per_host`. The image URL parser now lower-cases - the host and refuses one holding `@`, which a URL reads as a user name and - password, with 400. A host with a port, or with a `:` and no port, needs a - signature even when it is allowlisted. That is checked in the image service, - not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a - `Referer` is blocked whatever its port. A URL is signed over its host as it - writes it, in lower case, with any IPv6 brackets, `:` and port kept. -- 2026-10-15 a `:` in the source path or query no longer moves where the path - ends and the query begins (closes #247): the signed data and the cache key - joined host, path and query with `:` and no escaping, so a URL signed for - `https://h/a:b` verified for `https://h/a?b:`; the two also shared a cache - key, and whichever was fetched first was served for both. In both, the path - and the query now have each `%` written as `%25` and each `:` as `%3A`; a path - and query holding neither are signed and keyed as before. Signature - Specification in `README.md` says so. -- 2026-10-15 the example config allowlists no host (closes #249): it allowlisted - `github.com` and `user-images.githubusercontent.com`, where anyone can put an - image, so a deployment set up as Getting Started says served those images to - anyone without a signature, at any size and in any format. - `configs/config.example.yml` now leaves `allowlist_hosts` out, with one - commented entry under `example.com`. Getting Started and "Running under upaas" - in `README.md` say that every image on an allowlisted host is served without a - signature, so an allowlisted host must be one the operator controls. -- 2026-10-15 a chunked upstream body of exactly `upstream_max_response_size` - bytes is accepted (closes #248): once the limit was used up, the fetcher's - next read refused the body without looking for its end. Go's HTTP client - returns the end of a body sent with a `Content-Length` together with its last - bytes, but the end of a chunked body can come in a later read, so such a body - was refused after it had been downloaded whole, and the image routes - answered 500. Once the limit is used up, the fetcher now reads one more byte: - if the body has ended it is accepted, and if there is more it is refused with - `ErrResponseTooLarge`, as before. -- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side - (closes #257): it scales the image to cover the requested size, and the side - scaled past that size was held to no maximum, so `1x8192?fit=outside` of a - 3000x600 source was resized to 40960x8192 and encoded. The image processor now - refuses such a size with `ErrOutputTooLarge` before resizing, and both image - routes answer it 400, as for a size with one side `0`. The other fit modes - make the requested size or one within it, so none can go above 8192. -- 2026-10-13 a size with one side `0` no longer makes an output above 8192 on a - side (closes #241): that side is computed from the source's proportions, and - nothing held it to the 8192 a requested side is held to, so `0x8192` of a - 3000x600 source was resized to 40960x8192 and encoded. The image processor now - refuses such a size with `ErrOutputTooLarge` once it knows the source's - dimensions, before resizing, and both image routes answer it 400 with the JSON - error body. The refusal is not cached as a failed fetch, so the same source is - still served at other sizes. `imgcache.MaxDimension` is now the image - processor's `MaxOutputDimension`, so the two cannot differ. -- 2026-10-13 the unused cache lifetime is removed (closes #69): - `CacheConfig.CacheTTL`, which pixad set to `DefaultCacheTTL` (24 hours), was - never read, so cached images never expired. The field and the constant are - removed. Cached source and transformed images stay until eviction removes them - to keep the cache within `cache_max_bytes`. -- 2026-10-13 SVG sources are refused, and `orig` is never saved as JPEG in place - of the source's own format (closes #68): the upstream fetch no longer accepts - `image/svg+xml`, which libvips would render. Both image routes answer 415 for - an upstream `Content-Type` that is not one of the source formats, SVG - included, where they answered 500. Before decoding, the image processor - refuses with `ErrUnsupportedInputFormat`, also answered 415, any input that - govips' `DetermineImageType`, which picks the libvips loader from the first - bytes, does not name as a source format. That catches an SVG whose first bytes - pass the first-bytes check for another type, as an XML comment can make them - do for AVIF; an SVG whose first bytes do not pass that check answers 500, as - before, and so does a damaged image in a source format, whose first bytes - `DetermineImageType` names. `orig` is always the source's own format, where a - source the processor could not name was saved as JPEG. `README.md` no longer - lists SVG. The SVG detection in `internal/magic` stays: the request path does - not call it, and it goes with the unused code of - https://git.eeqj.de/sneak/pixa/issues/168. -- 2026-10-12 per-client limit on the image routes (closes #236): the new setting - `image_requests_per_minute` (`PIXA_IMAGE_REQUESTS_PER_MINUTE`), default 600, - is how many `GET` and `HEAD` requests one client may send to `/v1/image/` and - `/v1/e/` together per minute, cache hits included; the next is answered 429 - with `Retry-After`. It uses the login limit's `RateLimit` middleware, so - clients are told apart the same way, and runs after the CORS middleware, so a - preflight is not counted. `0` turns the limit off; a negative or non-integer - value aborts startup. -- 2026-10-08 a request past its deadline no longer waits for the database to - count it (closes #224). On a busy host, `TestService_Get_ReturnsByItsDeadline` - failed because its request, past its deadline, still waited for its miss count - to be written, a write with no deadline, which in pixad waits for the one - database connection every request shares. Each count write now keeps the - request's deadline but not its cancellation. A count not written by then is - kept in memory, where `Stats` sees it, and one goroutine of the cache writes - those counts, one write at a time, and once more at shutdown before the - database closes. The test phase of the `Dockerfile` also runs at most 4 tests - of a package at once (`go test -parallel 4`), where it ran as many as the host - has CPUs: on a busy host they then wait so long to be scheduled that a test - that times a request can fail. `-p`, how many packages are tested at once, is - unchanged, as capping it also slows compiling. -- 2026-10-08 every output format is saved with settings pixa sets on purpose - (closes #232): each format has its own govips export, as JPEG XL does, in - place of govips' generic `Export`, which sent libvips a zero for some settings - it was not given. PNG gets libvips' default compression, 6, where it had none, - and WebP libvips' default effort, 4, where it had 0. GIF was already at - libvips' default effort, 7, and JPEG output is unchanged. AVIF is saved at - effort 1, the lowest govips can set, where it had libvips' default, 4. With - one libvips thread, an 8192x8192 image of random pixels, the worst case, takes - about 51 seconds as AVIF at effort 1 and 43 as WebP at effort 4, against the - default `downstream_timeout` of 60 seconds. On an image of milder noise, which - AVIF at effort 1 saves in about 12 seconds, effort 4 takes minutes. AVIF is - also saved with 8 bits per sample from a 16-bit source, which libvips would - save with 12: a 16-bit 8192x8192 image of milder noise takes about 54 seconds - at effort 1 with 12 bits, nearly all of the default `downstream_timeout`, and - about 12 with 8. -- 2026-10-08 JPEG XL is the default output (closes #222): a `/v1/image/` URL - whose last segment is a size with no format, such as `800x600` or `orig`, is - served as JPEG XL and signed as `jxl`, so it has the signature of the same URL - ending in `.jxl`. An encrypted URL whose token holds no format is served as - JPEG XL (`encurl.DefaultFormat`). The generator page selects JPEG XL until - another format is chosen, and a form with an empty format, or none, makes a - JPEG XL URL whose name ends in `.jxl`. The image processor no longer takes an - empty format as `orig`: both routes give every request a format, and it - refuses a request with none. `auto` still ends with JPEG. -- 2026-10-08 JPEG XL as an input and output format (part of #222): a source - whose bytes start with either JPEG XL signature, the bare codestream's `FF 0A` - or the container's, is detected as `image/jxl`, which the upstream fetch - accepts; `orig` of such a source is JPEG XL. The format `jxl` works in plain - and encrypted URLs and on the generator page, served as `image/jxl` and cached - like the other formats. `auto` chooses JPEG XL first when `Accept` names - `image/jxl`. govips sends libvips a JPEG XL distance with every save, so - libvips ignores the quality: pixa turns `q` into a distance with libvips' own - formula, keeping 100 lossy, and removes the metadata from the image before - saving, as govips cannot have libvips strip it from JPEG XL. libvips 8.16 and - later still write an EXIF block of their own into JPEG XL, from the image's - size, orientation and resolution, and fixed values; the image is upright and - is given 72 dpi before the save, so the block holds nothing from the source. - An image with an ICC profile is converted to sRGB before the JPEG XL save too, - and a CMYK image with none is converted to sRGB, as libvips cannot save CMYK - as JPEG XL. libvips' default effort, 7, is kept. JPEG XL is not yet the - default output. -- 2026-10-08 pixad runs on the Alpine release it is built on (closes #229): the - runtime stage of the `Dockerfile` uses `alpine:3.22`, the release of the - `golang:1.25.4-alpine3.22` image that the test phase and the build stage use, - so all three have libvips 8.16, where the runtime image had 8.15. -- 2026-10-08 requests no longer wait behind eviction queries that read a whole - table (closes #227): the new `cache_usage` table holds the total cache usage, - kept up to date by triggers on `source_content` and `variant_content` in the - statement that adds, removes or resizes a row, and `UsageBytes` reads it. The - reconciliation pass reads the content tables 1000 rows per query, sums them - and corrects the total when it differs, unless a row changed while it summed. - Source rows get `last_accessed_at` when added, so choosing source images to - evict reads that column's index instead of sorting the whole table. -- 2026-10-08 libvips' JPEG XL support is installed and required (part of #222): - `script/bootstrap --cgo` installs `vips-jxl` when its package manager is apk, - as Alpine's `vips` package lacks the support, and the runtime stage of the - `Dockerfile` installs it too. `imgcache.NewService` fails, naming the fix, - when `imageprocessor.CheckJPEGXLSupport` finds that libvips cannot load and - save JPEG XL, so pixad does not start without it. A test saves an image as - JPEG XL with govips and loads it back. JPEG XL is not yet a format pixa - serves. -- 2026-10-08 SQLite writes no longer fail with "database is locked" under load - (closes #223): `internal/database` opens the database with one connection, so - pixa's own reads and writes run on it one at a time instead of competing for - SQLite's lock, where a write that kept losing could wait past the five-second - busy timeout and be lost. The busy timeout stays, for another program writing - to the same file. No code in pixa keeps rows or a transaction open while it - runs another query, which with one connection would wait forever. -- 2026-10-05 the format `auto` (closes #88): a format in the `/v1/image/` path, - an encrypted URL's token and the generator page's format choice, chosen for - each request from `Accept` once the signature or token is checked: AVIF when - the header names `image/avif`, else WebP when it names `image/webp`, else JPEG - when the first of `image/jpeg`, `image/*` and `*/*` that it names allows it, - or when it names nothing; `q=0` refuses a format. AVIF and WebP must be named, - as clients that cannot show them send the wildcards too. A header that allows - none of the three answers 406, one that does not parse 400. The signature and - the token cover `auto` itself; the cache key and `ETag` use the format chosen. - Answers from the point the format is chosen carry `Vary: Accept`, next to the - CORS `Vary: Origin`; fixed-format answers do not. -- 2026-10-05 the markdown is formatted with prettier (closes #100): `script/fmt` - and `script/fmt-check` run prettier 3.8.1, pinned in `package.json` and - `yarn.lock`, on `**/*.md` after `gofmt`, with four-space tabs and - `proseWrap: always` as `.prettierrc` says; `.prettierignore` keeps it off - `REPO_POLICIES.md`, the copy from `sneak/prompts`, and `vendor/`. Plain - `script/bootstrap` installs Node and Yarn as the one in `sneak/prompts` does - and then prettier; `script/bootstrap --cgo` does not, as the `Dockerfile` - stages that run it format nothing. The HTML templates stay unformatted: - prettier cannot parse a Go template action inside a tag. The markdown was - reflowed in a commit of its own. -- 2026-10-05 lint and tests run as the `lint` and `test` phases of the - `Dockerfile`, built with `--no-cache` (closes #202): `script/check`, - `script/cibuild`, `script/docker`, `script/lint`, `script/test`, - `script/setup` and `script/install-precommit` are now the copies from - `sneak/prompts` `main`, unchanged. The `lint` phase runs golangci-lint from - the image `REPO_POLICIES.md` names, with `libvips-dev` from `apt-get`; the - `test` phase runs the tests with a 90-second timeout; the build stage depends - on both. `Dockerfile.lint` and the `CHECK_EPOCH` build argument are gone, the - formatting check runs on the host, and `make docker-versioned` and - `make docker-test` call the scripts. `script/bootstrap`, `script/fmt`, - `script/fmt-check`, `script/precommit` and `script/projectname` stay pixa's - own. `script/bootstrap` installs git, make and Go, refreshing apt's package - lists before its first apt install; with `--cgo`, which only the `test` phase - and the build stage pass, it also installs the C compiler and the libvips and - libheif libraries. The stage that compiles still takes the version from - `git describe` when no `VERSION` is given, per - https://git.eeqj.de/sneak/pixa/issues/166, so the copied scripts' comment that - `.dockerignore` leaves out `.git` does not hold for pixa. -- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]` - (`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check` - does not run, measures three scenarios, each against a new pixad container and - a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss` - (a new source image every request) and `herd` (each new source image asked for - by all clients at once). For each it prints vegeta's report (requests per - second, latency percentiles, status codes), pixad's peak resident memory and - the requests that reached the origin. `README.md` says how to run it and read - it, and keeps 1-5k r/s as a target not yet measured. First measurement, with - the defaults on a shared 48-CPU machine with other work running: a baseline - for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95 - 8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so - not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100 - MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms, - peak 60 MiB, 188 fetches for 749 requests. -- 2026-10-04 the CI checkout fetches the tags (closes #208): the checkout step - in `.gitea/workflows/check.yml` sets `fetch-depth: 0`, as `REPO_POLICIES.md` - asks of a repo that takes its version from the tags, so a CI build of a tagged - commit stamps the tag from `git describe` instead of a bare commit. -- 2026-10-04 `config.yml` stays out of git and the Docker build context (closes - #212): `.gitignore` now ignores `config.yml`, the config file Getting Started - creates with the signing key, and `.dockerignore` leaves it out in every - directory and in any letter case, as it already did `config.yaml` and - `config.dev.yml`. -- 2026-10-04 local config files stay out of the Docker build context (closes - #211): `.dockerignore` now leaves out `config.yaml` and `config.dev.yml` in - every directory and in any letter case, the local config files `.gitignore` - keeps out of git because they can hold the signing key. - `configs/config.example.yml` is still sent. `config.yml`, which Getting - Started creates, is in neither file: - https://git.eeqj.de/sneak/pixa/issues/212. -- 2026-10-04 `cmd/pixad/main.go` is one call into `internal/` (closes #206): - what it did (the command line and its `--config` flag, setting - `PIXA_CONFIG_PATH`, ignoring `SIGPIPE`, starting the fx app) is now `Run` in - `internal/app`, unchanged, and `main` calls it with `Version`, which the build - still sets through `-X main.Version`. That code had no tests to move. -- 2026-10-04 `.gitignore` ignores `.claude/` (closes #204): the entry and its - comment are copied from the canonical `.gitignore` in `sneak/prompts`, - unanchored so it matches at every depth. `.dockerignore` already has - `.claude`. -- 2026-10-04 `.dockerignore` keeps secrets out at every depth (closes #205): the - file is now the standard one from `sneak/prompts`, whose patterns match in - every directory and, for environment files and private keys, in any letter - case, so a nested `.env` or `server.key` no longer reaches the build context. - pixa still sends `.git` without `.git/config` in place of the standard file's - `.git` line, and still leaves out `.gitignore`, `/bin` and `/data`. -- 2026-10-04 `REPO_POLICIES.md` matches the canonical copy again (closes #196): - it is replaced, unchanged, by `prompts/REPO_POLICIES.md` from `sneak/prompts` - `main`. The rules it adds that pixa's tree breaks are filed: - https://git.eeqj.de/sneak/pixa/issues/202 (lint and tests as `Dockerfile` - phases built with `--no-cache`), https://git.eeqj.de/sneak/pixa/issues/203 - (the workflow's `script/docker-smoke` step), - https://git.eeqj.de/sneak/pixa/issues/204 (`.claude/` in `.gitignore`), - https://git.eeqj.de/sneak/pixa/issues/205 (`.dockerignore` patterns at every - depth), https://git.eeqj.de/sneak/pixa/issues/206 (a thin `cmd/pixad/main.go`) - and https://git.eeqj.de/sneak/pixa/issues/208 (`fetch-depth: 0` on the CI - checkout, so the build sees the tags). Its rule that no build stage runs - `git describe` is not followed: pixa takes the version from the `.git` in the - build context, per https://git.eeqj.de/sneak/pixa/issues/166, as the copy on - `sneak/prompts` `next` already says. -- 2026-10-04 an integration test of the image proxy flow (closes #80): - `TestImageProxyFlow` in `internal/server` starts the database, handlers and - middleware from the constructors `pixad` uses, with a fresh state directory, - and replaces only the upstream origin with a local test server. For a resize - with a change to JPEG and for `orig`, the first request goes through the - router, the real fetcher, libvips, the disk cache and SQLite and answers 200 - with the right content type and size and `X-Pixa-Cache: MISS`; the second - answers `HIT` with the same image and the upstream has had one request; the - source and the converted image are then in `cache/sources` and - `cache/variants`, with their rows in `source_content`, `source_metadata` and - `variant_content`. Two optional fields make this possible, which `pixad` does - not set and the config file and environment cannot: - `httpfetcher.Config.DialContext` connects in place of the dialer that refuses - internal addresses, the URL and redirect checks still running, and - `handlers.Params.Fetcher` replaces the fetcher the handlers build. -- 2026-10-04 a URL made on the generator page with a `ttl` is tested to expire - (closes #199): a new test in `internal/handlers` makes a URL on the generator - page with a `ttl` of one second, checks that `/v1/e/` serves it at once, waits - two seconds and checks that it then answers 410. The test waits for real, as - pixa reads the clock directly when it makes and checks a URL; it waits two - seconds because the time a URL expires is kept in whole seconds. Test only. -- 2026-10-04 referer blocklist (closes #90): `referer_blocklist` - (`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for - `allowlist_hosts` with the same matcher; an entry of either list that is - neither a host name (letters, digits, hyphens, underscores and dots, with at - most one leading dot) nor an IP address, such as one with a port or a `*.` - wildcard, aborts startup naming the setting and the entry. Both image routes - refuse a request whose `Referer` names a listed host with 403 and a JSON error - before the signature, the cache and the upstream fetch, so it fetches nothing - and is refused whether or not the image is cached. A request with no - `Referer`, or one that does not parse as a URL with a host, is served, so the - list is easily got around; `README.md` and `configs/config.example.yml` say - so. It does not apply to the login and generator pages. -- 2026-10-04 fewer files in the repository root (closes #97): - `config.example.yml` moved unchanged to `configs/config.example.yml`, and - `README.md`, the comments in `internal/config/config.go` and the startup error - for the placeholder signing key name the new path; `scripts/manual-test.sh` - and its directory are deleted, as the handler tests in `internal/handlers` - cover every check it made except two: fetching a real image from the internet, - and a URL made on the generator page with a `ttl` answering 410 once the `ttl` - has passed (https://git.eeqj.de/sneak/pixa/issues/199); `CONVENTIONS.md` is - deleted, as `REPO_POLICIES.md` links the canonical Go HTTP server conventions. -- 2026-10-04 SQLite writes no longer fail with "database is locked" (closes - #198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write - that finds another in progress on another connection waits up to five seconds - for it, and the default `db_url` turns on WAL mode with - `_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a - parameter the driver reads, so the database was never in WAL mode. -- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup` only - passes through a periodic pass (closes #189): it slept for three eviction - intervals before writing its file, and a startup pass still running then could - adopt the file itself. It now holds the test database's only connection until - the startup pass waits for it after walking the empty variant directory, - writes the file and lets the connection go, as - `TestEvictionRunsOnPeriodicSchedule` does, so only a periodic reconciliation - pass can adopt the file. Test only. -- 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have - handler tests (closes #77): new tests in `internal/handlers`, with no network, - check that `GET /` without a login session shows the login form; a wrong key - shows it again with an error and sets no session cookie; the right key answers - 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and - `SameSite=Strict`, with which `GET /` shows the generator page; `GET /logout` - answers 303 to `/` with an empty session cookie sent with `Max-Age=0`; - `POST /generate` without a login session answers 303 to `/`; `/v1/e/` serves - the image for a valid token, answers 410 for an expired one and 400 for one - with a character changed, cut short or made with another signing key; and a - URL made on the generator page is served by `/v1/e/`. No code changes. -- 2026-10-04 `TODO.md` merges with git's union merge (closes #190): a root - `.gitattributes`, copied from `sneak/prompts`, marks it `merge=union`, so two - branches that each add an entry at the top of Completed Steps merge without a - conflict and keep both entries. Git now never reports a conflict in `TODO.md`: - a real one keeps both versions of the lines, and two entries that share an - identical line can end up one inside the other, which a rebase can do to an - entry already on `next`. The Workflow above says to read the merged entries - after every merge or rebase. -- 2026-10-04 the default `cache_max_bytes` no longer shrinks as the cache fills - (closes #184): for an omitted key, the cache works out the limit when it - opens, after the database is open, as 75% of the sum of the free space on the - filesystem containing `/cache/` and what the cache already holds by - its own size accounting, at least 500 MiB, so a cache filled to its limit - keeps that limit across a restart. The computation and its tests moved from - `internal/config` to `internal/imgcache`; the config only records whether the - key was set. -- 2026-10-04 `TestEvictionRunsOnPeriodicSchedule` no longer races the evictor - (closes #183): it wrote each variant file and then inserted its accounting row - by hand, and a reconciliation pass between the two adopted the file first, so - the insert failed. It now writes the files only, while holding the test - database's only connection so the evictor's startup pass waits after walking - the empty variant directory; a periodic reconciliation pass then adopts the - files and the eviction pass after it evicts them. No other test in - `internal/imgcache` inserts a row by hand after starting the evictor. Test - only. -- 2026-10-04 a config file pixa cannot read aborts startup (closes #176): of the - places pixa looks for its config file on its own, only one where the file does - not exist is passed over; any other error, such as a directory on the path - that pixa may not enter, aborts startup naming the file, as a file that does - not parse already did. -- 2026-10-04 `.golangci.yml` re-vendored from the canonical copy (closes #57): - the deprecated `gomodguard` is switched off, so lint runs print no deprecation - warning; its successor `gomodguard_v2` runs with the shared module block list, - and `depguard` keeps `net/http/httptest` out of files that are not tests. The - tree needed no code changes. -- 2026-10-04 the Content-Security-Policy allows no inline script or style - (closes #125): `script-src` and `style-src` are `'self'` only. The generator - page's two inline `onclick` handlers moved into - `internal/static/generator.js`, attached with `addEventListener`; the bundled - Tailwind script, which built styles in the browser, is replaced by a small - hand-written `internal/static/style.css` with only the rules the login and - generator pages use, the templates carrying a few plain class names in place - of Tailwind's. No build step. The pages keep their layout, not every pixel of - it. -- 2026-10-04 deployment guide and example Caddy config (closes #89): - "Deployment" in `README.md` says what the reverse proxy in front of pixa must - do (terminate TLS; pass `Host`, `Origin` and `Referer` on unchanged; set - `X-Forwarded-For`, with `trusted_proxies` to match; wait at least - `downstream_timeout`; optionally refuse `/metrics`) and what pixa does itself, - that the state directory needs a persistent volume and what `cache_max_bytes` - counts, the health check for a load balancer, what a stop does and its exit - codes, and what running outside Docker needs; `configs/Caddyfile` is the - example, checked with `caddy validate`. -- 2026-10-04 the metrics basic auth, CORS preflight, request logging and metrics - recording have tests (closes #79): `MetricsAuth` on its own answers 401 with a - challenge without credentials or with a wrong username or password and lets - the configured ones through; a preflight request gets `*` for any origin when - `access_control_allow_origin` is `*` and no `Access-Control-Allow-Origin` from - another origin than the configured one; a `POST /` carrying the signing key - leaves no trace of it in the request log line, and the login handler's own log - lines leave out the submitted key; the metrics middleware on its own records a - request it served, and the router records nothing while no metrics username is - set. Not tested: that the router puts the basic auth in front of `/metrics` - and records requests when a metrics username is set. Only one test per package - can set up `/metrics`, and in `internal/server` that is - `TestMaintenanceModeKeepsOtherRoutes`, which needs the owner's approval to - change; #180 holds it. Tests only; the basic auth library already compares the - password in constant time. -- 2026-10-04 the image route's signature check and error answers are tested - (closes #76): new tests in `internal/handlers`, with no network, check the - status and JSON error body for a missing, wrong, unpadded, upper-case or - expired signature on a host not on the allowlist, or a valid one sent for its - parent domain, a sibling host, a subdomain or the host with another domain - appended (401), an unparseable path (400), `localhost` as the upstream host - (403) and an upstream error (502); that an allowlisted host is served without - a signature, another host only with a valid one; and the answers of - `/robots.txt` and the health check. No code changes. -- 2026-10-04 request IDs returned and passed on, and `/v1/e/` revalidates - (closes #84): pixa's own `RequestID` middleware, in place of chi's, gives each - request an ID, its own `X-Request-ID` when that is at most 64 letters, digits, - `-`, `_` or `.` and a random one otherwise, stores it where chi's did and - sends it back as `X-Request-ID` on every response; the upstream fetch sends - that ID, and the "upstream fetched", "image converted" and "image served" log - lines carry it as `request_id`, a fetch shared by several requests carrying - the first request's; `/v1/e/` sets `ETag`, answers a matching `If-None-Match` - with 304 and is routed for `HEAD`, the `ETag` and 304 code being - `notModified`, which `/v1/image/` calls too; its token checks moved unchanged - into `parseImageEncRequest` to keep `HandleImageEnc` within the line limit; no - `Vary` is added, as no response depends on a request header except the image - routes' CORS headers, for which `go-chi/cors` already sends `Vary: Origin`; - `Vary: Accept` is left to #88. -- 2026-10-04 routes, encrypted URLs and config file documented (closes #75): - "Routes" in `README.md` lists every route with its method, purpose, what it - needs and the status codes it answers with, and says `q` and `fit` are part of - what is cached; "Encrypted URLs" covers logging in, making one on the - generator page, how long it lasts and the 410 once it has expired; - "Configuration" gives the order in which pixa looks for its config file; - `config.example.yml` lists `db_url` and `env` and gives every key's default; - `scripts/manual-test.sh` is left to #97. -- 2026-10-04 shutdown stops cache eviction in progress (closes #102): - `StartEviction` runs the eviction goroutine with its own context, which - `StopEviction` cancels, so a pass in progress stops at its next database call, - file, row or eviction candidate instead of running to completion, and no pass - starts after it, so a stop logs at most one warning; `StopEviction` takes a - context and, when that context ends before the goroutine exits, stops waiting - and returns its error; the handlers' stop hook passes fx's stop context, so an - eviction still running when fx's stop deadline ends fails the stop and makes - the exit code 1. -- 2026-10-04 dead code in `internal/imgcache` is gone (closes #73): `Purge`, - which only returned an error and which nothing called, is no longer part of - the `ImageCache` interface or `Service`; the `SignatureValidator`, `Allowlist` - and `Storage` interfaces, which nothing implemented or used, are deleted. - Nothing else changes. -- 2026-10-04 upstream host semaphores and variant `.meta` files no longer - outlive their use (closes #87): the fetcher counts the fetches holding or - waiting for a slot of each upstream host's semaphore and removes the host's - semaphore once none is left, so fetches from many hosts no longer leave one - semaphore each until restart; `VariantStorage.Delete` removes the variant's - `.meta` file along with it, a missing `.meta` file not being an error, and - `DeleteWithMeta`, which eviction called for that, is gone. -- 2026-10-04 `README.md` matches the code (closes #74): "Storage" names the - cache directories pixa uses (`cache/sources`, `cache/metadata`, - `cache/variants`) and how files are named in each, and the comments in - `001_schema.sql` name the same paths; the routes and the signature section - list the same output formats, `jpg` and `original` included; the TLS sentence - names `allow_http` as its exception; "Metrics" says only generic HTTP and Go - runtime metrics exist, measured and served only when the metrics username and - password are set. -- 2026-10-03 shutdown sets the exit code and waits for image processing (closes - #86): fx alone handles SIGINT and SIGTERM, and the server's own signal handler - is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's code: 0 for a - signal, 1 when the HTTP server cannot listen or the app fails to start or to - stop; the server's stop hook, which fx waits for, stops the HTTP server, waits - for the images still being processed, both within 5 seconds, then flushes - Sentry; images still being processed after that are logged with their count - and make the exit code 1; a Sentry DSN that cannot be used fails startup, so - the stop hooks of what had already started run, instead of exiting the process - from a goroutine; the eviction loop is left to #102. -- 2026-10-03 every `script/cibuild` and `script/docker` run executes the checks - (closes #101): the `Dockerfile` declares `CHECK_EPOCH` above `make fmt-check` - and `make lint` in the lint stage and above `make test` in the build stage, - and each of those steps names it in its command; both scripts pass a new value - on every run, so Docker runs the checks instead of reusing cached results, - while the `script/bootstrap` steps stay cached; a plain `docker build .` still - works, leaves it empty, and reuses the check steps only for an identical build - context; the `script/cibuild` comment and `README.md` no longer say that any - successful build implies a green repo. -- 2026-09-29 share concurrent misses (closes #65): requests that miss the same - variant at once (the same cache key, so quality and fit included) share one - upstream fetch or cached source read and one transcode through - `golang.org/x/sync/singleflight`; the first request's processing ignores its - cancellation but keeps its deadline, and the others wait for its image or - error holding no upstream connection or processing slot, and stop waiting when - their own context ends; the request doing the processing waits for it even - then, until it ends, which can be after its deadline; a request whose context - has already ended starts nothing; each request counts one miss, and the - processing counts its fetch and transcode once; a panic while processing is - reported to Sentry when `sentry_dsn` is set and becomes an error for every - waiting request instead of stopping pixad; documented in `README.md`. -- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS - middleware, with the `access_control_allow_origin` origin, moved from the - router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it - still answers a preflight `OPTIONS` request; the login and URL generator - pages, `/metrics` and the other routes send no `Access-Control-Allow-Origin`; - documented in `README.md` and `config.example.yml`. -- 2026-10-02 a plain `docker build .` stamps the tag or short commit, not `dev` - (closes #166): `.dockerignore` lets `.git` into the build context, without - `.git/config`; with no `VERSION` build argument the `Dockerfile` takes the - version from `git describe --tags --always`, and fails the build if the - context carries `.git` and no version comes out; `ARG VERSION` has no default; - pixad logs its version, with its name and architecture, as its first log line - at startup. -- 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes #159): - `deploy/docker-entrypoint.sh` creates the directory if it is missing, gives - the directory and everything in it to `pixad` when the directory or one of its - top-level entries belongs to another user or group, sets its mode to `750`, - then runs the server as `pixad`; data left by an earlier run under another uid - is taken over this way; "Running under upaas" in `README.md` no longer tells - the operator to create or chown the host directory. -- 2026-09-29 variant content types kept in memory (closes #70): - `Cache.metaCache` holds the content types of up to 10,000 variants in an LRU - (`github.com/hashicorp/golang-lru/v2`), filled by `StoreVariant` and by - `GetVariant` after it reads a `.meta` file, where a type `StoreVariant` added - meanwhile is kept over the one read, and never with the - `application/octet-stream` served for a variant without one; for a variant it - holds, `GetVariant` skips the `.meta` read, still opening the variant file and - taking the size from it; eviction removes the entry before deleting the files, - and `GetVariant` removes it when the file will not open; the cap is a - constant, not a setting; the unused `variantMeta` type is gone; `README.md` - describes it. -- 2026-09-29 maintenance mode refuses image requests (closes #71): while - `maintenance_mode` is on, `/v1/image/` and `/v1/e/` answer 503 with a - `Retry-After` header and the JSON error body, from one middleware in - `internal/server/routes.go`; the health check stays 200 and reports - `maintenance_mode`, as the image's Docker `HEALTHCHECK` requests it and upaas - marks a deploy failed when its container is unhealthy; the login and URL - generator pages and `/metrics` keep working; documented in `README.md` and - `config.example.yml`. -- 2026-09-29 bound concurrent image processing and upstream fetches (closes - #64): `max_concurrent_processing` (default the number of CPUs pixa can use) - limits the images decoded and encoded at once, and `upstream_connections` - (default 64) the connections to all upstream hosts together, on top of - `upstream_connections_per_host`; a fetch holds its connection until its image - has been processed, and a request whose source is cached reads it only once it - has a processing slot; a request that finds either limit reached waits up to - 10 seconds for a free one, then gets 503 `server busy, try again later`; - libvips runs one worker thread per image with its operation cache off; - documented in `README.md` and `config.example.yml`. -- 2026-09-29 Dockerfiles install through `script/bootstrap` (closes #95): the - `Dockerfile` lint and build stages and `Dockerfile.lint` copy `script/`, - `go.mod` and `go.sum`, then run `script/bootstrap` in place of their own - `apk add` lines, so the build dependencies are listed in one place; - `script/bootstrap` now also installs a C compiler when `gcc` is missing; the - build uses `-trimpath` and `-s -w` and keeps `CGO_ENABLED=1` for govips; - `ARG VERSION` sits just above the build, so a new version reruns neither - `script/bootstrap` nor the tests. -- 2026-09-29 migrations at the path `REPO_POLICIES.md` sets (closes #96): the - migration files moved, contents unchanged, from `internal/database/schema/` to - `internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the - `internal/db/migrations` package embeds them and `internal/database` reads - them through its `FS()`; the `internal/database` package itself stays; the - version still comes from the filename prefix, so a database that has recorded - versions 0 and 1 runs neither again. -- 2026-09-29 `trusted_proxies` advice and signature padding in `README.md` - (closes #150): the login-limit paragraph, the `trusted_proxies` entry and - `config.example.yml` say to set `trusted_proxies` to the address pixa sees for - requests that come through the proxy, which the request log shows as - `remoteIP` while it is not trusted; for a proxy on the Docker host that - connects over `127.0.0.1` that is the Docker network's gateway, not the - proxy's own address; the signature section says `sig` is base64url with the - `=` padding kept, and gives the example's `sig` for a stated signing key. -- 2026-09-29 fixed uid and gid for `pixad` (closes #151): the image creates the - `pixad` group with gid 65532 and the `pixad` user with uid 65532, instead of - the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad` is - not owned on the host by a person's login account; the first-run step of - "Running under upaas" in `README.md` names the uid and gid. -- 2026-09-29 `max-age` never outlives an expiring URL (closes #63): both image - routes build `Cache-Control` from the request's `Expires`, which an encrypted - URL's expiry now fills too; `max-age` is one year, or the whole seconds left - until the `exp` of a `/v1/image/` URL or the expiry of an encrypted URL when - that is sooner, never negative; an allowlisted host's URL that has an `exp` - follows it too; `immutable` stays, as freshness now ends at the expiry; - documented in `README.md`. -- 2026-09-28 add the four settings `README.md` documented but pixa did not have, - which aborted startup as unknown keys (closes #61): - `access_control_allow_origin` (default `*`, the CORS origin), - `upstream_fetch_timeout` (default `30s`), `upstream_max_response_size` - (default 50 MiB) and `downstream_timeout` (default `60s`, both the server's - write timeout and the per-request timeout); each has a `PIXA_` variable; - durations are positive Go duration strings, the size a whole number of bytes - up to 1 GiB, the origin `*` or one `http` or `https` origin as `README.md` - describes it; an invalid value aborts startup naming the key and the value; - documented in `config.example.yml` and `README.md`. -- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` counts - the cached source images and processed variants (`source_content` plus - `variant_content`) and takes their size from `Cache.UsageBytes`, instead of - reading `request_cache` and `output_content`, which nothing writes; those two - tables are left in the schema; a disabled disk cache reports no items and no - size. A hit is counted even when the request context has ended. A miss is - counted after it is served or fails, also when the request context has ended - by then, with the bytes it read from upstream, so `upstream_fetch_count` and - `upstream_fetch_bytes` move, including for an upstream body that fails partway - or a fetched source that then fails the magic byte check; `transform_count` - counts each image the image processor transcodes. -- 2026-09-28 strip metadata from processed images (closes #82): every output is - exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC - profile; the image is first turned upright with `AutoRotate` (before sizes are - worked out) and, when it has an ICC profile, converted to sRGB; the `orig` - format is re-encoded and stripped like any other, as pixa never serves the - source bytes; there is no setting to keep metadata; documented in `README.md`. -- 2026-09-28 rate limit the login form (closes #66): `POST /` is limited to 5 - attempts per minute per client address, and an attempt over the limit is - refused with 429 and a `Retry-After` header; the address is the one - `internal/clientip` resolves through `trusted_proxies`, an IPv6 client is - counted by its /64, and an IPv4-mapped address as the IPv4 address it carries; - the limit is a `RateLimit` middleware in `internal/middleware` on - `github.com/go-chi/httprate`, which the image routes can reuse; the library - keeps counts for the current and the previous minute only; documented in - `README.md`. -- 2026-09-28 refuse an unparseable `exp` on `/v1/image/` and log swallowed cache - errors (closes #72): an `exp` in the URL that is not a whole number, an empty - `exp=` included, is a 400 naming `exp` and the value, instead of being ignored - and answered with 401 as if the URL had no `exp`; only an `exp` missing from - the URL is unchanged; `README.md` says so where it documents `exp`. A failed - variant `.meta` write, source metadata JSON write, `Stats` count query, stats - counter update, negative cache write or expired negative cache delete is now - logged at `warn` with the path or key and the error, and stays non-fatal. -- 2026-09-28 refuse an empty `fit` on `/v1/image/` (closes #139): a `fit` in the - URL with an empty value (`fit=`) is a 400 naming `fit`, instead of being - served as `cover` and verified against a signature made for `cover`; only a - `fit` missing from the URL is still `cover`; any other value still goes - through the existing fit-mode check; `README.md` says so where it documents - `fit`. -- 2026-09-28 refuse an invalid `q` on `/v1/image/` (closes #134): a `q` that is - not a whole number from 1 to 100, an empty `q` included, is a 400 naming `q` - and the value, instead of being served at the default 85; the route reads `q` - with the generator's quality check (`parseFormInt` with `minQuality` and - `maxQuality`); only a `q` missing from the URL is still 85; a query string - that cannot be decoded, such as `q=80%`, is a 400 showing it; any query - parameter given more than once (`q`, `fit`, `sig`, `exp` alike) is a 400 - naming it, so none is read from its first value only; `README.md` states the - range and both query-string rules. -- 2026-09-28 unknown `PIXA_` environment variables abort startup (closes #133): - a variable whose name starts with `PIXA_` but is neither a setting's variable - nor `PIXA_CONFIG_PATH` aborts startup naming it, as an unknown config key - does, and `PIXA_PORT` is named with a pointer to `PORT`; the check runs after - the config file loads, so the variables the file's `env:` section sets are - checked too; documented in `README.md`. -- 2026-09-28 start on a fresh upaas volume (closes #129): the image starts as - root only to give `/var/lib/pixa` to `pixad` when `pixad` does not own it - (`deploy/docker-entrypoint.sh`), then runs the server as `pixad` through - `su-exec`, so a root-owned host directory bind-mounted there no longer stops - the container at startup; `README.md` gains a "Running under upaas" section. -- 2026-09-28 run all linting in Docker via `Dockerfile.lint` + `script/lint` - (closes #104): `make lint` calls `script/lint`, the only way the linter is - run; inside a container (both Dockerfiles set `container=docker`) it runs - `golangci-lint`, anywhere else it builds the hash-pinned `Dockerfile.lint`, - whose last step runs `script/lint` again; the `Dockerfile` lint stage runs - `make lint`; no host or nix-shell `golangci-lint` path remains - (`script/bootstrap` installs no linter); a per-run `CACHEBUST` build-arg keeps - the lint step from being served from cache, and a tmpfs mount on that step - keeps Go's and golangci-lint's caches out of its layer, so a run leaves no - large build cache behind; `golangci-lint config verify` stays out, as it - fetches its schema over an unpinned live HTTPS call -- 2026-09-28 every setting as an environment variable (closes #128, also covers - #99): each config key can be set by `PIXA_` plus the key in upper case (`.` - written as `_`), and the port by `PORT`; a variable present in the - environment, even empty, wins over the config file, which wins over the - default; the typed getters read the variable first, so every existing check - applies to it and a bad value aborts startup naming the variable; lists are - comma-separated, and an empty variable (or `""` in the file) is an empty list; - the Docker image no longer bakes in `config.docker.yml` or passes `--config`, - and its `HEALTHCHECK` probes `PORT` (default `8080`); the config file is - looked for under `/etc/pixa` and `~/.config/pixa` instead of the daemon name - `pixad`; documented in `README.md` and `config.example.yml`. -- 2026-09-28 quality and fit in the URL signature (closes #60): the signed data - is now `host:path:query:width:height:format:expiration:quality:fit`, using - `85` and `cover` when the URL has no `q` or `fit`, so one signed URL can no - longer be replayed across other quality and fit values to create unauthorized - cache entries and transcodes; the known-answer vectors in - `internal/signature/golden_test.go` and the README signature specification - describe the new format. -- 2026-09-28 Docker image healthcheck (closes #111): a `HEALTHCHECK` in the - runtime stage probing `/.well-known/healthcheck.json` with busybox `wget`; - `script/docker-smoke` (`make docker-smoke`) builds the image, starts it with a - throwaway `PIXA_SIGNING_KEY`, and passes only once Docker reports it healthy - within 30 seconds, removing the container on exit; the Gitea workflow runs it - after `script/cibuild`. -- 2026-09-21 trusted-proxy client IP resolution (closes #94): a - `trusted_proxies` config key taking a list of CIDRs, parsed by the same - `net/netip` list parser as `blocked_networks` (an invalid entry aborts startup - naming the key and value; an omitted key defaults to the RFC 1918 private - ranges, an explicitly empty list trusts no one, and an explicit list replaces - the default); a new `internal/clientip` package resolves the client address by - honoring `X-Forwarded-For` only when the direct peer is a trusted proxy, - walking the chain right-to-left to the rightmost non-proxy entry, so a client - connecting directly cannot spoof its address; the resolved address is stored - in the request context by a new middleware and used by the request-logging - middleware and the login-attempt logs in place of the raw peer address; - documented in `README.md` and `config.example.yml`. -- 2026-09-21 blocked networks configuration extending SSRF protection: a - `blocked_networks` config key taking a list of CIDRs (parsed with `net/netip`, - an invalid entry aborts startup naming the key and value), added to the - built-in blocklist rather than replacing it; the built-in ranges extended to - CGNAT `100.64.0.0/10`, IETF protocol assignments `192.0.0.0/24`, benchmark - `198.18.0.0/15`, and NAT64 `64:ff9b::/96` (IPv4-mapped forms covered); - enforcement stays in the dial-time re-resolution so the DNS-rebinding window - remains closed; documented in `README.md` and `config.example.yml`. -- 2026-09-21 validate dimensions and fit mode on the encrypted-URL route and the - token generator (closes #62): `imgcache.ValidateDimension` alone holds the - `MaxDimension` bound and is used by the path parser, by the new - `ValidateImageRequest` (which also applies `ValidateFitMode`) and by the - generator; both the `/v1/image/` and `/v1/e/` routes call - `ValidateImageRequest`, so an over-limit size or an unknown fit mode is a 400 - rather than an out-of-memory or a 500 from the processor; the URL generator - answers 400 naming the field for a `width` or `height` that is not a number or - fails the shared check, a `quality` that is not a number from 1 to 100, a - `ttl` that is not a number from 0 to the largest number of seconds the expiry - calculation can hold, or an unknown `fit`; an empty `quality` is 85 and an - empty `ttl` never expires; the form's width and height inputs stop at 8192 -- 2026-09-21 http.Server hardening (closes #92): added `HTTPReadHeaderTimeout` - (10s, bounds the slowloris header dribble) and `HTTPIdleTimeout` (120s, bounds - keep-alive reuse) alongside the existing timeouts and wired them onto the - server; added a `LimitBody` middleware capping the two form POST bodies - (`POST /`, `POST /generate`) at `MaxFormBytes` (1 MiB) and returning 413, - applied ahead of the CSRF middleware so an oversized body is refused as 413 - rather than being read as a missing CSRF token (403); left `WriteTimeout` at - 60s unchanged -- 2026-08-07 update golangci-lint to v2.12.2 with the canonical `.golangci.yml` - (v2 schema, `default: all` minus six disabled linters, `lll` 88, tests - included): bumped the pinned `golangci/golangci-lint:v2.12.2-alpine` image in - `Dockerfile` and the release-archive sha256 pins in `script/bootstrap`; fixed - the findings the stricter config surfaced (notably `paralleltest`, `wsl_v5`, - `goconst`, `lll`, `noinlineerr`, `err113`, `errcheck`, `testpackage` — - white-box test files renamed to `*_internal_test.go`), including #55's code - absorbed after it merged, iterating the pinned linter to `0 issues.`; no - single finding total is substantiable, since golangci-lint's `uniq-by-line` - reveals new findings on a line as others there are fixed — the documented - re-measurements were 81 after the #53 merge and 149 after the #55 merge; three - behavior changes, so not a pure no-op: `Cache.StoreVariant` now takes a - `context.Context` (`noctx`), so a cancelled request skips its best-effort - accounting row; `MetadataStorage.Store`'s cleanup defer was dead on `main` and - leaked `.tmp-*.json` on failure, now fixed with explicit removals; and the - `signing_key` validation error text gained `value too short: `; the eviction - loop's uncancellable context is deferred to #102 under a - `//nolint:contextcheck`; three `//nolint:tagliatelle` directives keep the - snake_case JSON wire/disk formats unchanged; `make check` green -- 2026-08-07 implement cache size management and eviction (closes #51): new - `cache_max_bytes` config key validated by the startup framework (explicit - values used exactly with no floor, `0` disables the disk cache entirely, - omitted defaults to max(75% of free space on the filesystem containing - `/cache/`, 500 MiB), logged at startup); processed variants are now - tracked in the database (a new `variant_content` table and an LRU timestamp on - `source_content`) so total usage is two SUMs, never a directory scan on the - hot path; a background goroutine evicts globally least-recently-used entries - (variants and source blobs merged) to the limit, woken by a periodic ticker - and by write-pressure notifications from stores; a source blob and ALL of its - `source_metadata` references are deleted in one transaction before the file is - unlinked, so multi-referenced blobs are never removed while referenced and - rows never point at deleted files; a startup and periodic reconciliation pass - adopts untracked variant files, drops rows for missing files, removes - unreachable source blobs, and sweeps stale temp files -- 2026-08-07 validate configuration on startup, fail fast on bad config (closes - #52): a config value that is set but unparseable or invalid aborts startup - naming the key and value (defaults apply only to omitted keys), unknown config - keys abort startup, a malformed config file aborts instead of being skipped, - and `state_dir` is verified creatable and writable before the listener binds -- 2026-08-07 manual test pass of the auth and encrypted URL flows against a - locally built and running `pixad` (built from `main` at `6573b9d`, port 18099, - local throwaway config); all six checks passed, plus all nine tests in - `scripts/manual-test.sh` (closes #49): - - [x] visit `/` and see the login form: HTTP 200, `Pixa - Login` page with - `name="key"` password form - - [x] wrong key shows an error: POST `/` with `key=wrong-key` returned HTTP - 200 login page containing "Invalid signing key" - - [x] correct signing key shows the generator form: POST `/` returned HTTP - 303 to `/` with - `Set-Cookie: pixa_session=...; HttpOnly; Secure; SameSite=Strict`; GET - `/` with that cookie rendered `Pixa - URL Generator` with the - `/generate` form and logout link - - [x] a generated encrypted URL serves the image: POST `/generate` - (ttl=3600) produced a `/v1/e//img.jpeg` URL that returned HTTP - 200, `Content-Type: image/jpeg`, an 800x600 baseline JPEG of 61706 - bytes - - [x] an expired URL (short TTL) returns 410: a ttl=1 URL fetched after 3 s - returned HTTP 410 Gone with - `{"error":"URL has expired","status":410,...}` - - [x] logout redirects back to login: GET `/logout` returned HTTP 303 to `/` - with `Set-Cookie: pixa_session=; Max-Age=0`; subsequent GET `/` - rendered the login form again -- 2026-08-07 fix the two remaining gosec findings (G124 in internal/session): - session cookies now always carry Secure/HttpOnly/SameSite=Strict on both the - set and clear paths; `make check` green (closes #47) -- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile - shims, README Entrypoints section -- 2026-04-07 extract magic byte detection into internal/magic (#42) -- 2026-03-25 extract allowlist package from internal/imgcache (#41) -- 2026-03-25 move schema_migrations table creation into 000.sql (#36) -- 2026-03-20 enforce and document exact-match-only signature verification (#40) -- 2026-03-20 bound imageprocessor.Process input read to prevent unbounded memory - use (#37); consolidate appname into an internal/globals constant (#34) -- 2026-03-18 parse version prefix from migration filenames (#33) -- 2026-03-15 QA audit fixes for 1.0/MVP readiness (#25) -- 2026-03-02 split Dockerfile with pre-built golangci-lint stage for faster CI - (#23) -- 2026-02-25 repo policy compliance: CI workflow, hash-pinned images, - golangci-lint and gosec fixes of that date (#14); arm64 Docker build fix (#16) -- 2026-01-08 WebP and AVIF encoding support via govips (both former P0 image - processing items, now done) - # Future Steps - P2: HTTP response handling - Last-Modified headers -- P2: configuration - - YAML config file support - P2: operational - - optional Sentry error reporting - - comprehensive request logging - - Prometheus performance metrics + - Prometheus metrics still missing: + - cache hits and misses, split into source images and variants + - cache size on disk and its limit + - evictions and the bytes they free + - upstream fetch duration and errors + - transcode duration + - images in processing - measure the 1k to 5k req/s target with `script/loadtest` on a machine not shared with other work From e34463acf8e536b0061c2177ca9011e4d47c5816 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 13:37:15 +0100 Subject: [PATCH 121/131] Check cached sources with upstream after the cache TTL (closes #69) (#350) --- README.md | 75 ++- internal/db/migrations/001_schema.sql | 28 +- .../handlers/client_gone_internal_test.go | 2 +- internal/handlers/handlers_internal_test.go | 86 ++- .../handlers/image_errors_internal_test.go | 2 +- .../referer_blocklist_internal_test.go | 3 +- .../httpfetcher/conditional_internal_test.go | 76 +++ .../httpfetcher/dial_context_internal_test.go | 4 +- internal/httpfetcher/fetch_internal_test.go | 8 +- internal/httpfetcher/httpfetcher.go | 68 ++- .../httpfetcher/httpfetcher_internal_test.go | 6 +- .../max_connections_internal_test.go | 24 +- internal/httpfetcher/mock.go | 5 +- .../httpfetcher/request_id_internal_test.go | 2 +- .../imageproxy/cache_ttl_internal_test.go | 493 ++++++++++++++++++ .../concurrent_misses_internal_test.go | 5 +- internal/imageproxy/imageproxy.go | 371 +++++++++++-- .../imageproxy/imageproxy_internal_test.go | 93 ---- internal/imageproxy/stats_internal_test.go | 2 +- internal/imgcache/cache.go | 367 +++++++++++-- internal/imgcache/cache_hit_internal_test.go | 18 +- ...vict_not_checked_recently_internal_test.go | 88 ++++ internal/imgcache/eviction.go | 82 ++- internal/imgcache/eviction_internal_test.go | 10 +- internal/imgcache/filestore.go | 29 +- internal/imgcache/filestore_internal_test.go | 6 +- .../imgcache/source_content_internal_test.go | 4 +- internal/providers/providers.go | 1 + 28 files changed, 1689 insertions(+), 269 deletions(-) create mode 100644 internal/httpfetcher/conditional_internal_test.go create mode 100644 internal/imageproxy/cache_ttl_internal_test.go create mode 100644 internal/imgcache/evict_not_checked_recently_internal_test.go diff --git a/README.md b/README.md index c72daae..1cc426f 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ results. A web page asks pixa for an image at a size and in a format. pixa fetches the original image, makes the requested image from it, and serves that. It keeps -both on disk. Later requests for the same image are served from disk, without a -fetch or a conversion. +both on disk. Later requests for the same image are served from disk. After 24 +hours pixa first asks the server that holds the original whether it changed (see +Cache TTL). In this document, the upstream host is the server that holds an original image. @@ -232,8 +233,8 @@ These terms are the names the code uses: - **Variant**: the image pixa serves for an image request, made from its source with its transform. - **Variant key**: the SHA-256 hash of a variant's source, size, format, quality - and fit. It names the variant's file and its row in the database, and gives - its `ETag`. + and fit. It names the variant's file and its row in the database. The + variant's `ETag` is made from it and the file's modification time. - **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image request and its expiry `exp` with `signing_key`. - **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any @@ -244,9 +245,12 @@ These terms are the names the code uses: after a fetch failed in one of these ways: - the upstream answered with an error status; - the body's first bytes were not those of the format its `Content-Type` - names. + names; + - a check of a cached image failed in any way (see Cache TTL). - **Image cache**: the part of pixa that keeps the sources, the variants and the negative cache (see Storage). +- **Cache TTL**: how long pixa uses a cached source and serves its variants + before it checks the source with upstream again: 24 hours (see Cache TTL). - **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached sources and variants may reach together. - **Image proxy**: the part of pixa that serves a variant from the image cache @@ -517,8 +521,10 @@ seconds left until then, at most one year. So no browser or proxy cache keeps the image after pixa would refuse the URL. A URL with no expiry gets one year. `immutable` only stops a client revalidating while its copy is fresh. The response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache, -and `MISS` for one made for this request. The `ETag` is the first 16 characters -of the variant key, in quotes. +and `MISS` for one made for this request or older than 24 hours (see Cache TTL). +The `ETag` is made from the first 16 characters of the variant key and the +variant file's modification time (see Cache TTL). So a variant made again from a +changed source gets a new `ETag`. ### Signed URLs @@ -824,10 +830,17 @@ and `state.sqlite3-shm` beside the file while pixa runs. The image cache keeps these tables in the database: -- `source_content`: a row for each source, by source key, with its size and when - it was last used +- `source_content`: a row for each source, by source key, with: + - its size and when it was last used + - when it was fetched, and when upstream last sent it or confirmed it + unchanged + - the `ETag` and `Last-Modified` headers upstream sent with it - `variant_content`: a row for each variant, by variant key (column - `cache_key`), with its size and when it was last used + `cache_key`), with: + - its size and when it was last used + - its source's check time when the variant was made or last renewed + - the `ETag` and `Last-Modified` headers upstream sent with the source it + was made from - `cache_usage`: the total size of the cached sources and variants, which eviction reads - `negative_cache`: a row for each source URL in the negative cache, with when @@ -881,6 +894,48 @@ within `cache_max_bytes`. It runs after each store, and every five minutes. It also removes the expired rows of the negative cache every five minutes, at most 1000 per statement, so a request waits behind 1000 deletions at most. +### Cache TTL + +pixa uses a cached source for 24 hours after upstream last sent it or confirmed +it unchanged. During that time it serves the source's variants from the cache. +After that, the next request for an image of that source checks the source with +upstream first: + +- pixa asks upstream to send the source only if it changed. It sends the + source's `ETag` as `If-None-Match` and its `Last-Modified` as + `If-Modified-Since`, when upstream gave them. With neither, upstream sends the + whole image again. +- When upstream answers 304 Not Modified, pixa uses the cached source and serves + its variants for another 24 hours. Nothing is downloaded or converted again. +- When upstream sends the image, it replaces the cached source. Each variant + made from the old image is converted again from the new one when it is next + requested. +- When the check fails, pixa serves the cached image. A failure can be an error + status, a timeout or no connection. The failure puts the source URL in the + negative cache, so pixa does not ask upstream again for 5 minutes, however + often it serves the image. + +pixa evicts a source and its variants 30 days after upstream last sent the +source or confirmed it unchanged. So pixa uses a source whose checks keep +failing for at most 30 days. Eviction to stay under the disk cache limit can +remove images sooner. + +That eviction can remove a source and keep its variants. Each variant's row +keeps the `ETag` and `Last-Modified` of the source it was made from. So pixa +checks such a variant with upstream in the same way after 24 hours: + +- When upstream answers 304 Not Modified, pixa serves the cached variant for + another 24 hours. Nothing is downloaded or converted again. +- When upstream sends the image, pixa caches it as the source and converts the + variant again. +- When the check fails, pixa serves the cached variant and puts the source URL + in the negative cache. + +A variant file's modification time is its source's check time when the variant +was made or last renewed. So a cache hit reads no database row to know whether +the variant is older than 24 hours. A request that checks its source with +upstream counts as a miss. + ### Configuration Every setting can be given as an environment variable, in a config file diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql index 8aa1696..7819d9f 100644 --- a/internal/db/migrations/001_schema.sql +++ b/internal/db/migrations/001_schema.sql @@ -3,14 +3,24 @@ -- Source images, one row per source URL -- Files stored at: cache/sources///, where source_key --- is the SHA-256 of the source's host, path and query +-- is the SHA-256 of the source's host, path and query. +-- fetched_at is when the cached bytes were fetched. checked_at is when +-- upstream last sent them or answered that they had not changed. +-- upstream_etag and upstream_last_modified are the ETag and Last-Modified +-- headers upstream sent with the bytes, or '' when it sent none. CREATE TABLE IF NOT EXISTS source_content ( source_key TEXT PRIMARY KEY, size_bytes INTEGER NOT NULL, - last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + upstream_etag TEXT NOT NULL DEFAULT '', + upstream_last_modified TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed ON source_content(last_accessed_at); +CREATE INDEX IF NOT EXISTS idx_source_content_checked + ON source_content(checked_at); -- Processed variant blobs -- Files stored at: cache/variants///. Tracked here @@ -18,14 +28,24 @@ CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed -- without a directory scan, and so LRU eviction has a timestamp to order -- on. A cache hit does not read this table: it takes the content type -- from the file's first bytes, and its last_accessed_at is written --- later, in a batch. +-- later, in a batch. checked_at is the checked_at of the source the +-- variant was made from, when it was made or last renewed; the variant +-- file's modification time holds the same time for cache hits. +-- upstream_etag and upstream_last_modified are those of the source bytes +-- the variant was made from, so the variant can still be checked with +-- upstream after its source is evicted. CREATE TABLE IF NOT EXISTS variant_content ( cache_key TEXT PRIMARY KEY, size_bytes INTEGER NOT NULL, - last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + upstream_etag TEXT NOT NULL DEFAULT '', + upstream_last_modified TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed ON variant_content(last_accessed_at); +CREATE INDEX IF NOT EXISTS idx_variant_content_checked + ON variant_content(checked_at); -- Total cache usage: the sum of size_bytes over source_content and -- variant_content, kept by the triggers below in the same statement diff --git a/internal/handlers/client_gone_internal_test.go b/internal/handlers/client_gone_internal_test.go index 108a5a9..df91306 100644 --- a/internal/handlers/client_gone_internal_test.go +++ b/internal/handlers/client_gone_internal_test.go @@ -26,7 +26,7 @@ type heldFetcher struct { // Fetch sends on f.started, waits for f.release to be closed, and returns // httpfetcher.ErrUpstreamError. func (f heldFetcher) Fetch( - context.Context, imagerequest.Source, + context.Context, imagerequest.Source, string, string, ) (*httpfetcher.FetchResult, error) { f.started <- struct{}{} diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go index de83bdf..0be15a7 100644 --- a/internal/handlers/handlers_internal_test.go +++ b/internal/handlers/handlers_internal_test.go @@ -131,7 +131,7 @@ func newMockFetcher(fs fs.FS) *mockFetcher { } func (f *mockFetcher) Fetch( - _ context.Context, source imagerequest.Source, + _ context.Context, source imagerequest.Source, _, _ string, ) (*httpfetcher.FetchResult, error) { path := source.Host + source.Path @@ -257,6 +257,90 @@ func TestHandleImage_ConditionalRequest_IfNoneMatch_DifferentETag(t *testing.T) } } +// TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage serves an +// image, changes it upstream, and requests it again with the ETag first +// served, once the cache TTL has passed. The changed image must be served, +// not 304 Not Modified. +func TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage(t *testing.T) { + t.Parallel() + + const host = "goodhost.example.com" + + upstream := fstest.MapFS{ + host + photoPath: &fstest.MapFile{ + Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}), + }, + } + + // A cache TTL of one nanosecond has passed by the second request + cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ + StateDir: t.TempDir(), + NegativeTTL: 5 * time.Minute, + CacheTTL: time.Nanosecond, + MaxBytes: 1 << 30, + }, prometheus.NewRegistry()) + if err != nil { + t.Fatalf("imgcache.NewCache() error = %v", err) + } + + svc, err := imageproxy.New(&imageproxy.Config{ + Cache: cache, + Fetcher: newMockFetcher(upstream), + Processor: imageprocessor.New(imageprocessor.Params{ + MaxInputBytes: 50 << 20, + MaxConcurrentProcessing: 2, + }), + MaxResponseSize: 50 << 20, + Logger: slog.New(slog.DiscardHandler), + }, prometheus.NewRegistry()) + if err != nil { + t.Fatalf("imageproxy.New() error = %v", err) + } + + h := &Handlers{ + imgSvc: svc, + log: slog.New(slog.DiscardHandler), + signer: signature.New(testSigningKey), + allowlist: allowlist.New([]string{host}), + } + + r := chi.NewRouter() + r.Get("/v1/image/*", h.HandleImage()) + + target := "/v1/image/" + host + photoPath + "/50x50.jpeg" + + first := httptest.NewRecorder() + r.ServeHTTP(first, + httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)) + + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("first request: status = %d, ETag = %q, want %d and an ETag", + first.Code, etag, http.StatusOK) + } + + upstream[host+photoPath] = &fstest.MapFile{ + Data: generateTestJPEG(t, 100, 100, color.RGBA{0, 0, 255, 255}), + } + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil) + req.Header.Set("If-None-Match", etag) + + second := httptest.NewRecorder() + r.ServeHTTP(second, req) + t.Logf("first ETag %s; with it after the change: status %d, ETag %s, %d bytes", + etag, second.Code, second.Header().Get("ETag"), second.Body.Len()) + + if second.Code != http.StatusOK { + t.Fatalf("status = %d, want %d with the changed image", + second.Code, http.StatusOK) + } + + if bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) { + t.Error("the image served is the one made before the change") + } +} + func TestHandleImage_ETagHeader(t *testing.T) { t.Parallel() diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index f4fee3c..c8c2a33 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -428,7 +428,7 @@ type failingFetcher struct { // Fetch returns f's error. func (f failingFetcher) Fetch( - context.Context, imagerequest.Source, + context.Context, imagerequest.Source, string, string, ) (*httpfetcher.FetchResult, error) { return nil, f.err } diff --git a/internal/handlers/referer_blocklist_internal_test.go b/internal/handlers/referer_blocklist_internal_test.go index 98691c4..88a8c28 100644 --- a/internal/handlers/referer_blocklist_internal_test.go +++ b/internal/handlers/referer_blocklist_internal_test.go @@ -36,10 +36,11 @@ type countingFetcher struct { // Fetch counts the fetch and passes it on. func (f *countingFetcher) Fetch( ctx context.Context, source imagerequest.Source, + ifNoneMatch, ifModifiedSince string, ) (*httpfetcher.FetchResult, error) { f.fetches.Add(1) - return f.Fetcher.Fetch(ctx, source) + return f.Fetcher.Fetch(ctx, source, ifNoneMatch, ifModifiedSince) } // newRefererRoutes returns both image routes of a Handlers whose diff --git a/internal/httpfetcher/conditional_internal_test.go b/internal/httpfetcher/conditional_internal_test.go new file mode 100644 index 0000000..7200204 --- /dev/null +++ b/internal/httpfetcher/conditional_internal_test.go @@ -0,0 +1,76 @@ +package httpfetcher + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" +) + +// TestFetch_SendsConditionalHeadersAndHandlesNotModified checks that Fetch +// sends a non-empty ifNoneMatch and ifModifiedSince as request headers, that +// upstream's 304 to such a request is a result with status 304 and no +// content, and that a 304 to a request with neither is ErrUpstreamError. +func TestFetch_SendsConditionalHeadersAndHandlesNotModified(t *testing.T) { + t.Parallel() + + const ( + etag = `"abc"` + lastModified = "Mon, 02 Nov 2026 10:00:00 GMT" + ) + + var ( + mu sync.Mutex + received http.Header + ) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + received = r.Header.Clone() + mu.Unlock() + + w.WriteHeader(http.StatusNotModified) + })) + t.Cleanup(srv.Close) + + f, _ := newServerFetcher(t, srv, nil) + + res, err := f.Fetch(testContext(t), upstreamSource("/image"), etag, lastModified) + if err != nil { + t.Fatalf("conditional Fetch() error = %v", err) + } + + body, err := io.ReadAll(res.Content) + _ = res.Content.Close() + + mu.Lock() + t.Logf("status = %d, body = %q, request headers = %v", res.StatusCode, body, received) + gotIfNoneMatch := received.Get("If-None-Match") + gotIfModifiedSince := received.Get("If-Modified-Since") + mu.Unlock() + + if err != nil || len(body) != 0 { + t.Errorf("304 content = %q, error = %v; want empty", body, err) + } + + if res.StatusCode != http.StatusNotModified { + t.Errorf("StatusCode = %d, want 304", res.StatusCode) + } + + if gotIfNoneMatch != etag || gotIfModifiedSince != lastModified { + t.Errorf("If-None-Match = %q, If-Modified-Since = %q; want %q and %q", + gotIfNoneMatch, gotIfModifiedSince, etag, lastModified) + } + + if held := len(f.allHostsSemaphore); held != 0 { + t.Errorf("connection still held after the 304: %d held", held) + } + + _, err = f.Fetch(testContext(t), upstreamSource("/image"), "", "") + if !errors.Is(err, ErrUpstreamError) { + t.Errorf("unconditional Fetch() answered 304: error = %v, want ErrUpstreamError", err) + } +} diff --git a/internal/httpfetcher/dial_context_internal_test.go b/internal/httpfetcher/dial_context_internal_test.go index b632a1f..e40139a 100644 --- a/internal/httpfetcher/dial_context_internal_test.go +++ b/internal/httpfetcher/dial_context_internal_test.go @@ -51,12 +51,12 @@ func TestDialContextReplacesOnlyTheDialer(t *testing.T) { t.Errorf("body = %q, want %q", body, imagePayload) } - _, err := f.Fetch(testContext(t), imageSourceOn("127.0.0.1")) + _, err := f.Fetch(testContext(t), imageSourceOn("127.0.0.1"), "", "") if !errors.Is(err, ErrSSRFBlocked) { t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err) } - _, err = f.Fetch(testContext(t), upstreamSource("/redirect/private")) + _, err = f.Fetch(testContext(t), upstreamSource("/redirect/private"), "", "") if !errors.Is(err, ErrSSRFBlocked) { t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err) } diff --git a/internal/httpfetcher/fetch_internal_test.go b/internal/httpfetcher/fetch_internal_test.go index 7c5199f..fee497b 100644 --- a/internal/httpfetcher/fetch_internal_test.go +++ b/internal/httpfetcher/fetch_internal_test.go @@ -160,7 +160,7 @@ func testContext(t *testing.T) context.Context { func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult { t.Helper() - res, err := f.Fetch(testContext(t), upstreamSource(path)) + res, err := f.Fetch(testContext(t), upstreamSource(path), "", "") if err != nil { t.Fatalf("Fetch(%s) error = %v", path, err) } @@ -172,7 +172,7 @@ func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult { func fetchExpectError(t *testing.T, f *HTTPFetcher, path string) error { t.Helper() - res, err := f.Fetch(testContext(t), upstreamSource(path)) + res, err := f.Fetch(testContext(t), upstreamSource(path), "", "") if err == nil { _ = res.Content.Close() @@ -224,7 +224,7 @@ func TestFetchRedirectToPrivateIPBlocked(t *testing.T) { srv := startUpstream(t) f, dialer := newServerFetcher(t, srv, nil) - _, err := f.Fetch(testContext(t), upstreamSource("/redirect/private")) + _, err := f.Fetch(testContext(t), upstreamSource("/redirect/private"), "", "") if !errors.Is(err, ErrSSRFBlocked) { t.Fatalf("Fetch() error = %v, want ErrSSRFBlocked", err) } @@ -624,7 +624,7 @@ func TestFetchSchemeFollowsAllowHTTP(t *testing.T) { DialContext: dialer.dialContext, }) - res, err := f.Fetch(testContext(t), upstreamSource("/image")) + res, err := f.Fetch(testContext(t), upstreamSource("/image"), "", "") if err == nil { _ = res.Content.Close() } diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index d532288..7f8a8f7 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -80,8 +80,13 @@ var ( // Fetcher retrieves content from upstream origins. type Fetcher interface { - // Fetch retrieves the content of the source image. - Fetch(ctx context.Context, source imagerequest.Source) (*FetchResult, error) + // Fetch retrieves the content of the source image. A non-empty + // ifNoneMatch or ifModifiedSince is sent as the request header of that + // name, and upstream may then answer 304 Not Modified with no content. + Fetch( + ctx context.Context, source imagerequest.Source, + ifNoneMatch, ifModifiedSince string, + ) (*FetchResult, error) } // FetchResult contains the result of fetching from upstream. @@ -223,9 +228,13 @@ func New(config *Config) *HTTPFetcher { // Fetch retrieves the source image with SSRF protection, over https, or over // http when AllowHTTP is set. When MaxConnections fetches are already in // progress, it waits up to ConnectionWaitTimeout for one to finish, then -// fails with ErrTooManyConnections. +// fails with ErrTooManyConnections. A non-empty ifNoneMatch or +// ifModifiedSince is sent as the If-None-Match or If-Modified-Since header. +// Upstream's 304 Not Modified to such a request is a result with StatusCode +// 304 and empty Content; to any other request it is ErrUpstreamError. func (f *HTTPFetcher) Fetch( ctx context.Context, source imagerequest.Source, + ifNoneMatch, ifModifiedSince string, ) (*FetchResult, error) { sourceURL := source.URL(f.config.AllowHTTP) @@ -252,17 +261,7 @@ func (f *HTTPFetcher) Fetch( req := &http.Request{ Method: http.MethodGet, URL: sourceURL, - Header: make(http.Header), - } - - req.Header.Set("User-Agent", userAgent) - req.Header.Set("Accept", strings.Join(allowedContentTypes(), ", ")) - - // The ID of the request this fetch serves, so the fetch can be found in - // the upstream host's logs - requestID := middleware.GetReqID(ctx) - if requestID != "" { - req.Header.Set(middleware.RequestIDHeader, requestID) + Header: requestHeaders(ctx, ifNoneMatch, ifModifiedSince), } // Use httptrace to capture connection details @@ -291,6 +290,20 @@ func (f *HTTPFetcher) Fetch( return nil, fmt.Errorf("upstream request failed: %w", err) } + // The deferred release gives the connection back, as no body follows + conditional := ifNoneMatch != "" || ifModifiedSince != "" + if conditional && resp.StatusCode == http.StatusNotModified { + _ = resp.Body.Close() + + return &FetchResult{ + Content: http.NoBody, + Headers: resp.Header, + StatusCode: resp.StatusCode, + FetchDurationMs: fetchDuration.Milliseconds(), + RemoteAddr: remoteAddr, + }, nil + } + result, err := f.buildResult(resp, remoteAddr, fetchDuration, release) if err != nil { return nil, err @@ -303,6 +316,33 @@ func (f *HTTPFetcher) Fetch( return result, nil } +// requestHeaders returns the headers of a fetch: User-Agent, Accept, the +// ID of the request the fetch serves, so the fetch can be found in the +// upstream host's logs, and a non-empty ifNoneMatch or ifModifiedSince. +func requestHeaders( + ctx context.Context, ifNoneMatch, ifModifiedSince string, +) http.Header { + header := make(http.Header) + + header.Set("User-Agent", userAgent) + header.Set("Accept", strings.Join(allowedContentTypes(), ", ")) + + requestID := middleware.GetReqID(ctx) + if requestID != "" { + header.Set(middleware.RequestIDHeader, requestID) + } + + if ifNoneMatch != "" { + header.Set("If-None-Match", ifNoneMatch) + } + + if ifModifiedSince != "" { + header.Set("If-Modified-Since", ifModifiedSince) + } + + return header +} + // acquireConnection takes a slot for host, then one of the slots shared by // all hosts, and returns the func that gives both back. The host's slot // comes first, so fetches queued for one busy host hold no shared slot. diff --git a/internal/httpfetcher/httpfetcher_internal_test.go b/internal/httpfetcher/httpfetcher_internal_test.go index e1400a2..20d8968 100644 --- a/internal/httpfetcher/httpfetcher_internal_test.go +++ b/internal/httpfetcher/httpfetcher_internal_test.go @@ -210,7 +210,7 @@ func TestMockFetcher_FetchesFile(t *testing.T) { m := NewMock(mockFS) result, err := m.Fetch(context.Background(), - imagerequest.Source{Host: testHost, Path: "/images/photo.jpg"}) + imagerequest.Source{Host: testHost, Path: "/images/photo.jpg"}, "", "") if err != nil { t.Fatalf("Fetch() error = %v", err) } @@ -242,7 +242,7 @@ func TestMockFetcher_MissingFileReturnsUpstreamError(t *testing.T) { m := NewMock(mockFS) _, err := m.Fetch(context.Background(), - imagerequest.Source{Host: testHost, Path: "/missing.jpg"}) + imagerequest.Source{Host: testHost, Path: "/missing.jpg"}, "", "") if !errors.Is(err, ErrUpstreamError) { t.Errorf("Fetch() error = %v, want ErrUpstreamError", err) } @@ -259,7 +259,7 @@ func TestMockFetcher_RespectsContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := m.Fetch(ctx, imagerequest.Source{Host: testHost, Path: "/photo.jpg"}) + _, err := m.Fetch(ctx, imagerequest.Source{Host: testHost, Path: "/photo.jpg"}, "", "") if !errors.Is(err, context.Canceled) { t.Errorf("Fetch() error = %v, want context.Canceled", err) } diff --git a/internal/httpfetcher/max_connections_internal_test.go b/internal/httpfetcher/max_connections_internal_test.go index bb80228..0bc855f 100644 --- a/internal/httpfetcher/max_connections_internal_test.go +++ b/internal/httpfetcher/max_connections_internal_test.go @@ -45,12 +45,12 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond - first, err := f.Fetch(testContext(t), imageSourceOnPort(81)) + first, err := f.Fetch(testContext(t), imageSourceOnPort(81), "", "") if err != nil { t.Fatalf("first Fetch() error = %v", err) } - second, err := f.Fetch(testContext(t), imageSourceOnPort(82)) + second, err := f.Fetch(testContext(t), imageSourceOnPort(82), "", "") if err != nil { t.Fatalf("second Fetch() error = %v", err) } @@ -59,7 +59,7 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { start := time.Now() - _, err = f.Fetch(testContext(t), imageSourceOnPort(83)) + _, err = f.Fetch(testContext(t), imageSourceOnPort(83), "", "") if !errors.Is(err, ErrTooManyConnections) { t.Fatalf("third Fetch() error = %v, want ErrTooManyConnections", err) } @@ -78,7 +78,7 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) { t.Fatalf("close first body: %v", err) } - third, err := f.Fetch(testContext(t), imageSourceOnPort(83)) + third, err := f.Fetch(testContext(t), imageSourceOnPort(83), "", "") if err != nil { t.Fatalf("Fetch() after a response was closed: error = %v", err) } @@ -105,7 +105,7 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) - first, err := f.Fetch(testContext(t), imageSourceOnPort(81)) + first, err := f.Fetch(testContext(t), imageSourceOnPort(81), "", "") if err != nil { t.Fatalf("first Fetch() error = %v", err) } @@ -115,7 +115,7 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - _, err = f.Fetch(ctx, imageSourceOnPort(82)) + _, err = f.Fetch(ctx, imageSourceOnPort(82), "", "") if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("second Fetch() error = %v, want context.DeadlineExceeded", err) } @@ -140,7 +140,7 @@ func TestFetchRemovesIdleHostSemaphores(t *testing.T) { for i := range 100 { wg.Go(func() { - res, err := f.Fetch(ctx, imageSourceOnPort(1+i%50)) + res, err := f.Fetch(ctx, imageSourceOnPort(1+i%50), "", "") if err != nil { t.Errorf("Fetch() error = %v", err) @@ -179,12 +179,12 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond - open, err := f.Fetch(testContext(t), imageSourceOnPort(81)) + open, err := f.Fetch(testContext(t), imageSourceOnPort(81), "", "") if err != nil { t.Fatalf("first Fetch() error = %v", err) } - _, err = f.Fetch(testContext(t), imageSourceOnPort(82)) + _, err = f.Fetch(testContext(t), imageSourceOnPort(82), "", "") if !errors.Is(err, ErrTooManyConnections) { t.Fatalf("Fetch() from another host: error = %v, "+ "want ErrTooManyConnections", err) @@ -193,7 +193,7 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - _, err = f.Fetch(ctx, imageSourceOnPort(81)) + _, err = f.Fetch(ctx, imageSourceOnPort(81), "", "") if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("Fetch() from the busy host: error = %v, "+ "want context.DeadlineExceeded", err) @@ -205,7 +205,7 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) { ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() - _, err = f.Fetch(ctx, imageSourceOnPort(83)) + _, err = f.Fetch(ctx, imageSourceOnPort(83), "", "") if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("Fetch() from a host with nothing open: error = %v, "+ "want context.DeadlineExceeded", err) @@ -255,7 +255,7 @@ func TestFetchReleasesConnectionOnError(t *testing.T) { f, _ := newServerFetcher(t, srv, cfg) f.connectionWaitTimeout = 100 * time.Millisecond - _, err := f.Fetch(testContext(t), tc.source) + _, err := f.Fetch(testContext(t), tc.source, "", "") if !errors.Is(err, tc.want) { t.Fatalf("Fetch() error = %v, want %v", err, tc.want) } diff --git a/internal/httpfetcher/mock.go b/internal/httpfetcher/mock.go index 1e3268b..5101cfc 100644 --- a/internal/httpfetcher/mock.go +++ b/internal/httpfetcher/mock.go @@ -24,9 +24,10 @@ func NewMock(fsys fs.FS) *MockFetcher { return &MockFetcher{fs: fsys} } -// Fetch retrieves the source's content from the mock filesystem. +// Fetch retrieves the source's content from the mock filesystem. It ignores +// ifNoneMatch and ifModifiedSince, and always sends the content. func (m *MockFetcher) Fetch( - ctx context.Context, source imagerequest.Source, + ctx context.Context, source imagerequest.Source, _, _ string, ) (*FetchResult, error) { // Check context cancellation select { diff --git a/internal/httpfetcher/request_id_internal_test.go b/internal/httpfetcher/request_id_internal_test.go index b6e87b8..36d4f09 100644 --- a/internal/httpfetcher/request_id_internal_test.go +++ b/internal/httpfetcher/request_id_internal_test.go @@ -35,7 +35,7 @@ func TestFetchSendsRequestID(t *testing.T) { ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID) - res, err := f.Fetch(ctx, upstreamSource("/image")) + res, err := f.Fetch(ctx, upstreamSource("/image"), "", "") if err != nil { t.Fatalf("Fetch() error = %v", err) } diff --git a/internal/imageproxy/cache_ttl_internal_test.go b/internal/imageproxy/cache_ttl_internal_test.go new file mode 100644 index 0000000..263ff6c --- /dev/null +++ b/internal/imageproxy/cache_ttl_internal_test.go @@ -0,0 +1,493 @@ +package imageproxy + +import ( + "bytes" + "context" + "errors" + "fmt" + "image/color" + "io" + "net/http" + "os" + "sync" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/httpfetcher" + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// Validators the test upstream sends with its image. +const ( + testETag = `"v1"` + testLastModified = "Mon, 02 Nov 2026 10:00:00 GMT" +) + +// errConnectionRefused stands for an upstream that cannot be reached. +var errConnectionRefused = errors.New("connection refused") + +// upstreamFailure is one way a fetch from upstream fails. +type upstreamFailure struct { + name string + err error +} + +// upstreamFailures returns the ways a check of a cached image with upstream +// can fail: an error status, a timeout and no connection. +func upstreamFailures() []upstreamFailure { + return []upstreamFailure{ + {"error status", fmt.Errorf("%w: status 503", httpfetcher.ErrUpstreamError)}, + {"timeout", httpfetcher.ErrUpstreamTimeout}, + {"no connection", errConnectionRefused}, + } +} + +// conditionalUpstream serves one JPEG image with an ETag or a Last-Modified +// header, or both. It answers 304 Not Modified to a fetch whose If-None-Match +// is its ETag, or, when it has no ETag, whose If-Modified-Since is its +// Last-Modified. While err is set, every fetch fails with it. It records the +// conditional headers of every fetch. +type conditionalUpstream struct { + mu sync.Mutex + image []byte + etag string + lastModified string + err error + + ifNoneMatch []string + ifModifiedSince []string +} + +func (u *conditionalUpstream) Fetch( + _ context.Context, _ imagerequest.Source, + ifNoneMatch, ifModifiedSince string, +) (*httpfetcher.FetchResult, error) { + u.mu.Lock() + defer u.mu.Unlock() + + u.ifNoneMatch = append(u.ifNoneMatch, ifNoneMatch) + u.ifModifiedSince = append(u.ifModifiedSince, ifModifiedSince) + + if u.err != nil { + return nil, u.err + } + + unchanged := ifNoneMatch == u.etag + if u.etag == "" { + unchanged = ifModifiedSince == u.lastModified + } + + if unchanged { + return &httpfetcher.FetchResult{ + Content: http.NoBody, + StatusCode: http.StatusNotModified, + }, nil + } + + headers := http.Header{} + if u.etag != "" { + headers.Set("ETag", u.etag) + } + + if u.lastModified != "" { + headers.Set("Last-Modified", u.lastModified) + } + + return &httpfetcher.FetchResult{ + Content: io.NopCloser(bytes.NewReader(u.image)), + ContentLength: int64(len(u.image)), + ContentType: testContentTypeJPEG, + Headers: headers, + StatusCode: http.StatusOK, + }, nil +} + +// change replaces the upstream image and its ETag. +func (u *conditionalUpstream) change(image []byte, etag string) { + u.mu.Lock() + defer u.mu.Unlock() + + u.image = image + u.etag = etag +} + +// fail makes every later fetch fail with err. +func (u *conditionalUpstream) fail(err error) { + u.mu.Lock() + defer u.mu.Unlock() + + u.err = err +} + +// fetches returns how many fetches upstream was asked for. +func (u *conditionalUpstream) fetches() int { + u.mu.Lock() + defer u.mu.Unlock() + + return len(u.ifNoneMatch) +} + +// lastConditionalHeaders returns the If-None-Match and If-Modified-Since of +// the last fetch. +func (u *conditionalUpstream) lastConditionalHeaders() (string, string) { + u.mu.Lock() + defer u.mu.Unlock() + + last := len(u.ifNoneMatch) - 1 + + return u.ifNoneMatch[last], u.ifModifiedSince[last] +} + +// setupConditionalUpstreamProxy returns a proxy whose upstream is a +// conditionalUpstream serving a red JPEG with an ETag. +func setupConditionalUpstreamProxy( + t *testing.T, +) (*testProxy, *TestFixtures, *conditionalUpstream) { + t.Helper() + + svc, fixtures := setupTestProxy(t) + upstream := &conditionalUpstream{ + image: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}), + etag: testETag, + } + svc.fetcher = upstream + + return svc, fixtures, upstream +} + +// TestProxy_Get_FreshVariantAsksUpstreamNothing checks that a variant whose +// source was checked with upstream less than the cache TTL ago is served from +// the cache without asking upstream. +func TestProxy_Get_FreshVariantAsksUpstreamNothing(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + variant := photoVariant(fixtures, 85, imagerequest.FitCover) + + first, _ := getImage(t, svc, variant) + second, status := getImage(t, svc, variant) + + t.Logf("cache status = %s, upstream fetches = %d", status, upstream.fetches()) + + if status != CacheHit { + t.Errorf("CacheStatus = %q, want %q", status, CacheHit) + } + + if !bytes.Equal(first, second) { + t.Error("the cached variant differs from the one first served") + } + + if n := upstream.fetches(); n != 1 { + t.Errorf("upstream fetches = %d, want 1", n) + } +} + +// TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain caches a variant, +// sets its source's check time and the variant's file time two days back, +// and requests it again, with the source still cached and with the source's +// row and file removed, as eviction does. Upstream must be asked with the +// validator it sent, answer 304, and send no bytes; the variant must be served +// again without being made again, and then be served from the cache until the +// TTL passes. +func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + sourceEvicted bool + etag string + lastModified string + wantIfNoneMatch string + wantIfModifiedSince string + }{ + {"ETag", false, testETag, "", testETag, ""}, + {"Last-Modified only", false, "", testLastModified, "", testLastModified}, + {"both", false, testETag, testLastModified, testETag, testLastModified}, + {"ETag, source evicted", true, testETag, "", testETag, ""}, + {"Last-Modified only, source evicted", true, "", testLastModified, "", + testLastModified}, + {"both, source evicted", true, testETag, testLastModified, testETag, + testLastModified}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + upstream.etag = tt.etag + upstream.lastModified = tt.lastModified + variant := photoVariant(fixtures, 85, imagerequest.FitCover) + + first, _ := getImage(t, svc, variant) + setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2)) + + if tt.sourceEvicted { + evictSource(t, svc, variant.Source) + } + + second, status := getImage(t, svc, variant) + + ifNoneMatch, ifModifiedSince := upstream.lastConditionalHeaders() + counters := readCounters(t, svc) + t.Logf("cache status = %s, If-None-Match = %q, If-Modified-Since = %q, "+ + "counters = %+v", status, ifNoneMatch, ifModifiedSince, counters) + + if ifNoneMatch != tt.wantIfNoneMatch { + t.Errorf("If-None-Match = %q, want %q", ifNoneMatch, tt.wantIfNoneMatch) + } + + if ifModifiedSince != tt.wantIfModifiedSince { + t.Errorf("If-Modified-Since = %q, want %q", + ifModifiedSince, tt.wantIfModifiedSince) + } + + if !bytes.Equal(first, second) { + t.Error("the variant served after the check differs from the cached one") + } + + if counters.upstreamFetchCount != 1 || counters.transformCount != 1 { + t.Errorf("upstream downloads = %d, transforms = %d; want 1 and 1", + counters.upstreamFetchCount, counters.transformCount) + } + + _, status = getImage(t, svc, variant) + if status != CacheHit { + t.Errorf("CacheStatus after the check = %q, want %q", status, CacheHit) + } + + if n := upstream.fetches(); n != 2 { + t.Errorf("upstream fetches = %d, want 2", n) + } + }) + } +} + +// TestProxy_Get_ChangedSourceAfterTTLReplacesSourceAndVariants caches two +// variants of one source, changes the image upstream, and sets the source's +// check time and both variants' file times two days back. Each variant must +// then be made again from the new image, which is downloaded once. +func TestProxy_Get_ChangedSourceAfterTTLReplacesSourceAndVariants(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + variants := []*imagerequest.Request{ + photoVariant(fixtures, 85, imagerequest.FitCover), + photoVariant(fixtures, 40, imagerequest.FitCover), + } + + oldImages := make([][]byte, len(variants)) + for i, variant := range variants { + oldImages[i], _ = getImage(t, svc, variant) + } + + newSource := generateTestJPEG(t, 120, 90, color.RGBA{0, 0, 255, 255}) + upstream.change(newSource, `"v2"`) + + twoDaysAgo := time.Now().AddDate(0, 0, -2) + for _, variant := range variants { + setBackSourceAndVariant(t, svc, variant, twoDaysAgo) + } + + for i, variant := range variants { + image, _ := getImage(t, svc, variant) + if bytes.Equal(image, oldImages[i]) { + t.Errorf("variant %d is still made from the old image", i) + } + } + + counters := readCounters(t, svc) + t.Logf("counters = %+v, upstream fetches = %d", counters, upstream.fetches()) + + if counters.upstreamFetchCount != 2 { + t.Errorf("upstream downloads = %d, want 2", counters.upstreamFetchCount) + } + + cached, err := os.ReadFile(sourceFilePath(svc, variants[0].Source.Key())) + if err != nil { + t.Fatalf("reading the cached source: %v", err) + } + + if !bytes.Equal(cached, newSource) { + t.Error("the cached source is not the new image") + } +} + +// TestProxy_Get_UpstreamFailureAfterTTLServesCachedImage caches a variant, +// sets its source's check time and the variant's file time two days back, and +// makes upstream fail. The cached variant must still be served, and a variant +// not yet cached must be made from the cached source. +func TestProxy_Get_UpstreamFailureAfterTTLServesCachedImage(t *testing.T) { + t.Parallel() + + for _, failure := range upstreamFailures() { + t.Run(failure.name, func(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + variant := photoVariant(fixtures, 85, imagerequest.FitCover) + + first, _ := getImage(t, svc, variant) + setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2)) + upstream.fail(failure.err) + + second, _ := getImage(t, svc, variant) + t.Logf("upstream fetches = %d", upstream.fetches()) + + if upstream.fetches() != 2 { + t.Errorf("upstream fetches = %d, want 2", upstream.fetches()) + } + + if !bytes.Equal(first, second) { + t.Error("the variant served while upstream fails differs from the cached one") + } + + getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) + }) + } +} + +// TestProxy_Get_FailingUpstreamIsAskedOnceWhileImageIsServedOften caches a +// variant, sets its source's check time and the variant's file time two days +// back, and makes every fetch time out. Serving the variant ten times, and a +// variant not yet cached, must ask upstream once: the failed check puts the +// source URL in the negative cache. +func TestProxy_Get_FailingUpstreamIsAskedOnceWhileImageIsServedOften(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + variant := photoVariant(fixtures, 85, imagerequest.FitCover) + + getImage(t, svc, variant) + setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2)) + upstream.fail(httpfetcher.ErrUpstreamTimeout) + + for range 10 { + getImage(t, svc, variant) + } + + getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover)) + + t.Logf("upstream fetches = %d", upstream.fetches()) + + if n := upstream.fetches(); n != 2 { + t.Errorf("upstream fetches = %d, want 2: the first fetch and one check", n) + } +} + +// TestProxy_Get_UpstreamFailureAfterTTLServesVariantOfEvictedSource caches a +// variant, sets its file time two days back, removes its source's row and +// file, as eviction does, and makes upstream fail. The cached variant must +// still be served. +func TestProxy_Get_UpstreamFailureAfterTTLServesVariantOfEvictedSource(t *testing.T) { + t.Parallel() + + for _, failure := range upstreamFailures() { + t.Run(failure.name, func(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + variant := photoVariant(fixtures, 85, imagerequest.FitCover) + + first, _ := getImage(t, svc, variant) + setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2)) + evictSource(t, svc, variant.Source) + upstream.fail(failure.err) + + second, _ := getImage(t, svc, variant) + t.Logf("upstream fetches = %d", upstream.fetches()) + + if !bytes.Equal(first, second) { + t.Error("the variant served while upstream fails differs from the cached one") + } + + if n := upstream.fetches(); n != 2 { + t.Errorf("upstream fetches = %d, want 2", n) + } + }) + } +} + +// TestProxy_Get_FailingUpstreamIsAskedOnceForVariantOfEvictedSource caches a +// variant, sets its file time two days back, removes its source's row and +// file, as eviction does, and makes every fetch time out. Serving the variant +// ten times must ask upstream once: the failed check puts the source URL in +// the negative cache. +func TestProxy_Get_FailingUpstreamIsAskedOnceForVariantOfEvictedSource(t *testing.T) { + t.Parallel() + + svc, fixtures, upstream := setupConditionalUpstreamProxy(t) + variant := photoVariant(fixtures, 85, imagerequest.FitCover) + + first, _ := getImage(t, svc, variant) + setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2)) + evictSource(t, svc, variant.Source) + upstream.fail(httpfetcher.ErrUpstreamTimeout) + + for range 10 { + image, _ := getImage(t, svc, variant) + if !bytes.Equal(image, first) { + t.Fatal("the variant served while upstream fails differs from the cached one") + } + } + + t.Logf("upstream fetches = %d", upstream.fetches()) + + if n := upstream.fetches(); n != 2 { + t.Errorf("upstream fetches = %d, want 2: the first fetch and one check", n) + } +} + +// evictSource removes the row and then the file of src's cached source, as +// the cache's eviction does, and leaves its variants cached. +func evictSource(t *testing.T, svc *testProxy, src imagerequest.Source) { + t.Helper() + + _, err := svc.db.ExecContext(t.Context(), + `DELETE FROM source_content WHERE source_key = ?`, string(src.Key())) + if err != nil { + t.Fatalf("deleting the source's row: %v", err) + } + + err = os.Remove(sourceFilePath(svc, src.Key())) + if err != nil { + t.Fatalf("removing the source's file: %v", err) + } +} + +// setBackSourceAndVariant sets back to when the times the cache keeps for +// req's source and for the variant req asks for: the source's fetch and check +// times, both last access times, the variant's check time, and the variant +// file's modification time. +func setBackSourceAndVariant( + t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time, +) { + t.Helper() + + key := req.Source.Key() + setSourceLastAccessed(t, svc.db, key, when) + + _, err := svc.db.ExecContext(t.Context(), + `UPDATE source_content SET fetched_at = ?, checked_at = ? + WHERE source_key = ?`, when.UTC(), when.UTC(), string(key)) + if err != nil { + t.Fatalf("setting back the source's fetch and check times: %v", err) + } + + variantKey := imagerequest.CacheKey(req) + setVariantLastAccessed(t, svc.db, variantKey, when) + + _, err = svc.db.ExecContext(t.Context(), + `UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`, + when.UTC(), string(variantKey)) + if err != nil { + t.Fatalf("setting back the variant's check time: %v", err) + } + + path := variantFilePath(svc, variantKey) + + err = os.Chtimes(path, when, when) + if err != nil { + t.Fatalf("setting back the times of %s: %v", path, err) + } +} diff --git a/internal/imageproxy/concurrent_misses_internal_test.go b/internal/imageproxy/concurrent_misses_internal_test.go index 4c7409f..a5c5e03 100644 --- a/internal/imageproxy/concurrent_misses_internal_test.go +++ b/internal/imageproxy/concurrent_misses_internal_test.go @@ -37,6 +37,7 @@ type heldFetcher struct { func (f *heldFetcher) Fetch( ctx context.Context, source imagerequest.Source, + ifNoneMatch, ifModifiedSince string, ) (*httpfetcher.FetchResult, error) { f.fetches.Add(1) @@ -48,7 +49,7 @@ func (f *heldFetcher) Fetch( return nil, ctx.Err() } - return f.upstream.Fetch(ctx, source) + return f.upstream.Fetch(ctx, source, ifNoneMatch, ifModifiedSince) } // releaseFetches lets every held fetch, and every later one, go on. @@ -489,7 +490,7 @@ func TestProxy_Get_WaiterGetsProcessingTimedOut(t *testing.T) { type panickingFetcher struct{} func (panickingFetcher) Fetch( - context.Context, imagerequest.Source, + context.Context, imagerequest.Source, string, string, ) (*httpfetcher.FetchResult, error) { panic("upstream fetcher panicked") } diff --git a/internal/imageproxy/imageproxy.go b/internal/imageproxy/imageproxy.go index ae17947..ea7243a 100644 --- a/internal/imageproxy/imageproxy.go +++ b/internal/imageproxy/imageproxy.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log/slog" + "net/http" "runtime/debug" "time" @@ -157,7 +158,9 @@ const ( // // A cache hit opens the variant file and makes no database call: the cache // hands its access time to the goroutine that writes access times, without -// waiting for it, and its hit is counted in memory. A variant file that +// waiting for it, and its hit is counted in memory. A variant whose source +// was last checked with upstream CacheTTL or more ago is a miss, which checks +// the source first (see processFromSourceOrFetch). A variant file that // cannot be served, such as one whose first bytes are not those of an output // format, is a miss, and the variant is made again. func (p *Proxy) Get( @@ -165,20 +168,20 @@ func (p *Proxy) Get( ) (*ImageResponse, error) { cacheKey := imagerequest.CacheKey(req) - reader, size, contentType, err := p.cache.Variant(cacheKey) + cached, err := p.cache.Variant(cacheKey) if err == nil { p.cache.IncrementStats(true) return &ImageResponse{ - Content: reader, - ContentLength: size, - ContentType: contentType, + Content: cached.Content, + ContentLength: cached.Size, + ContentType: cached.ContentType, CacheStatus: CacheHit, - ETag: formatETag(cacheKey), + ETag: formatETag(cacheKey, cached.CheckedAt), }, nil } - if !errors.Is(err, imgcache.ErrNotFound) { + if !errors.Is(err, imgcache.ErrNotFound) && !errors.Is(err, imgcache.ErrStale) { p.log.Warn("cached variant not served, making it again", "cache_key", cacheKey, "error", err) } @@ -285,7 +288,7 @@ func (p *Proxy) processOrWait( ContentLength: int64(len(variant.data)), ContentType: variant.contentType, FetchedBytes: variant.fetchedBytes, - ETag: formatETag(cacheKey), + ETag: formatETag(cacheKey, variant.checkedAt), }, nil } @@ -337,13 +340,50 @@ func (p *Proxy) loadCachedSource( return reader, size, format } -// processFromSourceOrFetch processes an image, using cached source content -// if available. +// processFromSourceOrFetch makes the variant req asks for, from the cached +// source when there is one and from a fetched source otherwise. +// +// A cached source last checked with upstream CacheTTL or more ago is checked +// first: upstream is asked to send it only if it changed since. When it +// changed, the new bytes replace the cached source and the variant is made +// from them. When it did not change, or the check fails, the cached source +// is used: the variant's cached file is served again if it was made from the +// cached source's bytes, and otherwise the variant is made from them. When +// the source is not cached but the variant is, the variant is checked with +// upstream the same way (see fetchAndProcess). func (p *Proxy) processFromSourceOrFetch( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { + cached, err := p.cache.LookupSource(ctx, req.Source) + if err != nil { + if !errors.Is(err, imgcache.ErrNotFound) { + p.log.Warn("failed to look up cached source, fetching", "error", err) + } + + return p.fetchAndProcess(ctx, req, cacheKey) + } + + if p.cache.Stale(cached.CheckedAt) { + fetched, err := p.fetchSource(ctx, req, cached.ETag, cached.LastModified) + + switch { + case errors.Is(err, errSourceNotModified): + cached.CheckedAt = time.Now().UTC() + p.renewSource(ctx, req.Source, cached.CheckedAt) + case err != nil: + p.recordFailedCheck(ctx, req, err) + default: + return p.processFetched(ctx, req, cacheKey, fetched) + } + } + + variant := p.renewVariant(ctx, cacheKey, cached) + if variant != nil { + return variant, nil + } + source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source) // Fetch from upstream if we don't have source data or it's empty @@ -356,18 +396,199 @@ func (p *Proxy) processFromSourceOrFetch( // Process using cached source; nothing was fetched from upstream. The // image processor reads the source only once it has a processing slot, // so a request waiting for one holds none of it in memory. - return p.processAndStore(ctx, req, cacheKey, source, sourceFormat, sourceSize) + return p.processAndStore(ctx, req, cacheKey, cached, source, sourceFormat, + sourceSize) } -// fetchAndProcess fetches from upstream, processes, and caches the result. -// It counts the fetch with the bytes read from upstream, including when -// reading the response or a later step fails. It fetches nothing while the -// source URL is in the negative cache. +// recordFailedCheck logs a failed check of req's source with upstream, after +// which the image is still served from the cache. It puts the source URL in +// the negative cache, whatever the failure, a timeout included, so upstream +// is asked at most once per NegativeTTL while it fails, however often the +// image is served. A failure that was a negative cache hit records nothing. +func (p *Proxy) recordFailedCheck( + ctx context.Context, req *imagerequest.Request, err error, +) { + if errors.Is(err, ErrNegativeCached) { + return + } + + p.log.Warn("checking source with upstream failed, serving from the cache", + "request_id", middleware.GetReqID(ctx), + "host", req.Source.Host, "path", req.Source.Path, "error", err) + p.storeNegative(ctx, req) +} + +// renewSource records that upstream answered at checkedAt that src's cached +// source had not changed. A failure to record it is only logged: the source +// is then checked again on its next use. +func (p *Proxy) renewSource( + ctx context.Context, src imagerequest.Source, checkedAt time.Time, +) { + err := p.cache.RenewSource(ctx, src, checkedAt) + if err != nil { + p.log.Warn("failed to renew cached source", "source_key", src.Key(), + "error", err) + } +} + +// renewVariant returns the cached variant of cacheKey when it was made from +// the bytes of the cached source that source describes, and renews it to +// source.CheckedAt (see imgcache.Cache.RenewVariant). It returns nil when +// there is no such variant or it cannot be read. +func (p *Proxy) renewVariant( + ctx context.Context, cacheKey imagerequest.VariantKey, + source imgcache.SourceInfo, +) *processedVariant { + cached, err := p.cache.RenewVariant(ctx, cacheKey, source) + if err != nil { + if !errors.Is(err, imgcache.ErrNotFound) { + p.log.Warn("cached variant not renewed, making it again", + "cache_key", cacheKey, "error", err) + } + + return nil + } + + return p.readCachedVariant(cacheKey, cached) +} + +// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever +// its age, with what its row keeps about the source bytes it was made from +// (see imgcache.Cache.LookupVariant). It returns nil when the variant is not +// cached or cannot be read. The SourceInfo is empty when the variant is not +// returned, or its row is missing or cannot be read. +func (p *Proxy) cachedVariantIgnoringTTL( + ctx context.Context, cacheKey imagerequest.VariantKey, +) (*processedVariant, imgcache.SourceInfo) { + cached, err := p.cache.VariantIgnoringTTL(cacheKey) + if err != nil { + if !errors.Is(err, imgcache.ErrNotFound) { + p.log.Warn("cached variant not served", "cache_key", cacheKey, + "error", err) + } + + return nil, imgcache.SourceInfo{} + } + + variant := p.readCachedVariant(cacheKey, cached) + if variant == nil { + return nil, imgcache.SourceInfo{} + } + + madeFrom, err := p.cache.LookupVariant(ctx, cacheKey) + if err != nil && !errors.Is(err, imgcache.ErrNotFound) { + p.log.Warn("failed to look up cached variant", "cache_key", cacheKey, + "error", err) + } + + return variant, madeFrom +} + +// readCachedVariant reads the cached variant of cacheKey from its opened file, +// and closes the file. It returns nil when the variant cannot be read. +func (p *Proxy) readCachedVariant( + cacheKey imagerequest.VariantKey, file imgcache.VariantFile, +) *processedVariant { + defer func() { _ = file.Content.Close() }() + + data, err := io.ReadAll(file.Content) + if err != nil { + p.log.Warn("failed to read cached variant", "cache_key", cacheKey, + "error", err) + + return nil + } + + return &processedVariant{ + data: data, + contentType: file.ContentType, + checkedAt: file.CheckedAt, + } +} + +// fetchAndProcess fetches the source from upstream, stores it, and makes +// and stores the variant from it. +// +// The variant can still be cached after eviction removed its source. The +// fetch then sends the ETag and Last-Modified of the source bytes the variant +// was made from, so upstream sends the source only if it changed since. When +// it did not change, the cached variant is renewed and served, with no +// download and no conversion. When the fetch fails, the cached variant is +// served whatever its age, and the failure is recorded as a failed check (see +// recordFailedCheck). func (p *Proxy) fetchAndProcess( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, ) (*processedVariant, error) { + variant, madeFrom := p.cachedVariantIgnoringTTL(ctx, cacheKey) + + fetched, err := p.fetchSource(ctx, req, madeFrom.ETag, madeFrom.LastModified) + + switch { + case err == nil: + return p.processFetched(ctx, req, cacheKey, fetched) + case variant == nil: + return nil, err + case errors.Is(err, errSourceNotModified): + variant.checkedAt = time.Now().UTC() + + err = p.cache.SetVariantCheckedAt(ctx, cacheKey, variant.checkedAt) + if err != nil { + p.log.Warn("failed to renew cached variant", "cache_key", cacheKey, + "error", err) + } + default: + p.recordFailedCheck(ctx, req, err) + } + + return variant, nil +} + +// processFetched makes and stores the variant from the source fetchSource +// fetched, then closes the fetch's response. +func (p *Proxy) processFetched( + ctx context.Context, + req *imagerequest.Request, + cacheKey imagerequest.VariantKey, + fetched *fetchedSource, +) (*processedVariant, error) { + // Closing the response frees the upstream connection. It is closed only + // after processing, so the fetcher's connection limit also bounds the + // fetched sources held in memory while their requests wait for a + // processing slot. + defer func() { _ = fetched.response.Close() }() + + return p.processAndStore(ctx, req, cacheKey, fetched.info, + bytes.NewReader(fetched.data), fetched.format, int64(len(fetched.data))) +} + +// errSourceNotModified is returned by fetchSource when upstream answers 304 +// Not Modified: the cached source has not changed. +var errSourceNotModified = errors.New("upstream source not modified") + +// fetchedSource is a source fetchSource fetched and stored in the cache. +type fetchedSource struct { + data []byte + format imagerequest.Format + // info is what the cache keeps about the source + info imgcache.SourceInfo + // response is the fetch's response, still open; closing it frees the + // upstream connection + response io.Closer +} + +// fetchSource fetches req's source from upstream and stores it in the cache. +// A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source +// only if it changed since; errSourceNotModified says it did not. It counts +// the fetch with the bytes read from upstream, including when reading the +// response or a later step fails. It fetches nothing while the source URL is +// in the negative cache. The caller closes the returned response. +func (p *Proxy) fetchSource( + ctx context.Context, + req *imagerequest.Request, + ifNoneMatch, ifModifiedSince string, +) (*fetchedSource, error) { // Check negative cache - skip fetching for recently-failed URLs negHit, err := p.cache.Failure(ctx, req.Source) if err != nil { @@ -387,7 +608,8 @@ func (p *Proxy) fetchAndProcess( p.log.Debug("fetching from upstream", "host", req.Source.Host, "path", req.Source.Path, "query", req.Source.Query) - fetchResult, err := p.fetcher.Fetch(ctx, req.Source) + fetchResult, err := p.fetcher.Fetch(ctx, req.Source, + ifNoneMatch, ifModifiedSince) if err != nil { // Store negative cache for certain errors if isNegativeCacheable(err) { @@ -397,11 +619,15 @@ func (p *Proxy) fetchAndProcess( return nil, fmt.Errorf("upstream fetch failed: %w", err) } - // Closing the body frees the upstream connection. It is closed only - // after processing, so the fetcher's connection limit also bounds the - // fetched sources held in memory while their requests wait for a - // processing slot. - defer func() { _ = fetchResult.Content.Close() }() + if fetchResult.StatusCode == http.StatusNotModified { + _ = fetchResult.Content.Close() + + p.log.Info("upstream source not modified", + "request_id", middleware.GetReqID(ctx), "host", req.Source.Host, + "path", req.Source.Path, "fetch_ms", fetchResult.FetchDurationMs) + + return nil, errSourceNotModified + } // Read and validate the source content sourceData, err := io.ReadAll(fetchResult.Content) @@ -411,31 +637,12 @@ func (p *Proxy) fetchAndProcess( p.incrementUpstreamFetch(fetchBytes) if err != nil { + _ = fetchResult.Content.Close() + return nil, fmt.Errorf("failed to read upstream response: %w", err) } - // Calculate download bitrate - var downloadRate string - - if fetchResult.FetchDurationMs > 0 { - seconds := float64(fetchResult.FetchDurationMs) / 1000.0 //nolint:mnd // ms to seconds - bitsPerSecond := float64(fetchBytes*8) / seconds //nolint:mnd // bytes to bits - downloadRate = humanize.SI(bitsPerSecond, "bps") - } - - // Log upstream fetch details - p.log.Info("upstream fetched", - "request_id", middleware.GetReqID(ctx), - "host", req.Source.Host, - "path", req.Source.Path, - "bytes", fetchBytes, - "fetch_ms", fetchResult.FetchDurationMs, - "rate", downloadRate, - "remote_addr", fetchResult.RemoteAddr, - "http", fetchResult.HTTPVersion, - "tls", fetchResult.TLSVersion, - "cipher", fetchResult.TLSCipherSuite, - ) + p.logUpstreamFetch(ctx, req, fetchResult, fetchBytes) // The source's format, detected from its first bytes, which must match // its content type. A body that does not is the upstream's error, so it @@ -448,18 +655,70 @@ func (p *Proxy) fetchAndProcess( "content_type", fetchResult.ContentType, "error", err) p.storeNegative(ctx, req) + _ = fetchResult.Content.Close() + return nil, fmt.Errorf("content validation failed: %w", err) } + info := fetchedSourceInfo(fetchResult) + // Store source content - _, err = p.cache.PutSource(ctx, req.Source, bytes.NewReader(sourceData)) + _, err = p.cache.PutSource(ctx, req.Source, bytes.NewReader(sourceData), info) if err != nil { p.log.Warn("failed to store source content", "error", err) // Continue even if caching fails } - return p.processAndStore( - ctx, req, cacheKey, bytes.NewReader(sourceData), sourceFormat, fetchBytes, + return &fetchedSource{ + data: sourceData, + format: sourceFormat, + info: info, + response: fetchResult.Content, + }, nil +} + +// fetchedSourceInfo returns what the cache keeps about a source upstream +// just sent: fetched and checked now, with the ETag and Last-Modified +// headers of fetchResult. +func fetchedSourceInfo(fetchResult *httpfetcher.FetchResult) imgcache.SourceInfo { + now := time.Now().UTC() + upstreamHeaders := http.Header(fetchResult.Headers) + + return imgcache.SourceInfo{ + FetchedAt: now, + CheckedAt: now, + ETag: upstreamHeaders.Get("ETag"), + LastModified: upstreamHeaders.Get("Last-Modified"), + } +} + +// logUpstreamFetch logs a fetch of req's source that read fetchBytes bytes, +// with its download rate and connection details. +func (p *Proxy) logUpstreamFetch( + ctx context.Context, + req *imagerequest.Request, + fetchResult *httpfetcher.FetchResult, + fetchBytes int64, +) { + var downloadRate string + + if fetchResult.FetchDurationMs > 0 { + seconds := float64(fetchResult.FetchDurationMs) / 1000.0 //nolint:mnd // ms to seconds + bitsPerSecond := float64(fetchBytes*8) / seconds //nolint:mnd // bytes to bits + downloadRate = humanize.SI(bitsPerSecond, "bps") + } + + p.log.Info("upstream fetched", + "request_id", middleware.GetReqID(ctx), + "host", req.Source.Host, + "path", req.Source.Path, + "bytes", fetchBytes, + "fetch_ms", fetchResult.FetchDurationMs, + "rate", downloadRate, + "remote_addr", fetchResult.RemoteAddr, + "http", fetchResult.HTTPVersion, + "tls", fetchResult.TLSVersion, + "cipher", fetchResult.TLSCipherSuite, ) } @@ -491,14 +750,19 @@ type processedVariant struct { data []byte contentType string fetchedBytes int64 + // checkedAt is its source's check time when the variant was made or last + // renewed, which its cached file has as its modification time + checkedAt time.Time } // processAndStore processes the image read from source, which is in -// sourceFormat, and stores the result. +// sourceFormat, and stores the result. sourceInfo describes the source's +// bytes, and is stored with the variant (see imgcache.Cache.PutVariant). func (p *Proxy) processAndStore( ctx context.Context, req *imagerequest.Request, cacheKey imagerequest.VariantKey, + sourceInfo imgcache.SourceInfo, source io.Reader, sourceFormat imagerequest.Format, fetchBytes int64, @@ -559,7 +823,8 @@ func (p *Proxy) processAndStore( ) // Store variant to cache - err = p.cache.PutVariant(ctx, cacheKey, bytes.NewReader(processedData)) + err = p.cache.PutVariant(ctx, cacheKey, sourceInfo, + bytes.NewReader(processedData)) if err != nil { p.log.Warn("failed to store variant", "error", err) // Continue even if caching fails @@ -569,6 +834,7 @@ func (p *Proxy) processAndStore( data: processedData, contentType: processResult.ContentType, fetchedBytes: fetchBytes, + checkedAt: sourceInfo.CheckedAt, }, nil } @@ -580,13 +846,16 @@ func isNegativeCacheable(err error) bool { // etagHashLength is the number of hash characters to use for ETags. const etagHashLength = 16 -// formatETag formats a VariantKey as a quoted ETag value. -func formatETag(key imagerequest.VariantKey) string { +// formatETag returns the quoted ETag of the variant of key: the first +// etagHashLength characters of key, then checkedAt in Unix nanoseconds. +// checkedAt is its source's check time when the variant was made or last +// renewed, so a variant made again from new source bytes gets another ETag. +func formatETag(key imagerequest.VariantKey, checkedAt time.Time) string { hash := string(key) // Use first 16 characters of hash for a shorter but still unique ETag if len(hash) > etagHashLength { hash = hash[:etagHashLength] } - return `"` + hash + `"` + return fmt.Sprintf(`"%s-%d"`, hash, checkedAt.UnixNano()) } diff --git a/internal/imageproxy/imageproxy_internal_test.go b/internal/imageproxy/imageproxy_internal_test.go index be2e46a..78a03eb 100644 --- a/internal/imageproxy/imageproxy_internal_test.go +++ b/internal/imageproxy/imageproxy_internal_test.go @@ -3,9 +3,7 @@ package imageproxy import ( "context" "io" - "os" "testing" - "time" "sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/magic" @@ -263,97 +261,6 @@ func TestProxy_Get_Caching(t *testing.T) { } } -// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and -// a variant of it and sets back two days every time pixa stored for them. -// For the source: last_accessed_at in source_content, and the access and -// modification times of its file. For the variant: last_accessed_at in -// variant_content, and the access and modification times of its file. It -// then checks that the variant is still served from the cache and that a new -// variant is made from the cached source without fetching it again. -// -// Left at the present: the files' status change and birth times, which a -// program cannot set, and the times of the directories holding the files, -// which other entries share. -func TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) { - t.Parallel() - - svc, fixtures := setupTestProxy(t) - variant := photoVariant(fixtures, 85, imagerequest.FitCover) - - resp, err := svc.Get(t.Context(), variant) - if err != nil { - t.Fatalf("Get() first request error = %v", err) - } - - _ = resp.Content.Close() - - twoDaysAgo := time.Now().AddDate(0, 0, -2) - backdateSource(t, svc, variant, twoDaysAgo) - backdateVariant(t, svc, variant, twoDaysAgo) - - resp, err = svc.Get(t.Context(), variant) - if err != nil { - t.Fatalf("Get() of the cached variant error = %v", err) - } - - _ = resp.Content.Close() - - if resp.CacheStatus != CacheHit { - t.Errorf("cached variant CacheStatus = %q, want %q", - resp.CacheStatus, CacheHit) - } - - // Another quality is a new variant, made from the cached source - resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover)) - if err != nil { - t.Fatalf("Get() of a new variant error = %v", err) - } - - _ = resp.Content.Close() - - fetches := readCounters(t, svc).upstreamFetchCount - if fetches != 1 { - t.Errorf("upstream fetches = %d, want 1", fetches) - } -} - -// backdateSource sets every time pixa stored for req's cached source to when, -// as listed on TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo. -func backdateSource( - t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time, -) { - t.Helper() - - key := req.Source.Key() - setSourceLastAccessed(t, svc.db, key, when) - - path := sourceFilePath(svc, key) - - err := os.Chtimes(path, when, when) - if err != nil { - t.Fatalf("setting back the times of %s: %v", path, err) - } -} - -// backdateVariant sets every time pixa stored for the cached variant req asks -// for to when, as listed on -// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo. -func backdateVariant( - t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time, -) { - t.Helper() - - key := imagerequest.CacheKey(req) - path := variantFilePath(svc, key) - - setVariantLastAccessed(t, svc.db, key, when) - - err := os.Chtimes(path, when, when) - if err != nil { - t.Fatalf("setting back the times of %s: %v", path, err) - } -} - func TestProxy_Get_DifferentSizes(t *testing.T) { t.Parallel() diff --git a/internal/imageproxy/stats_internal_test.go b/internal/imageproxy/stats_internal_test.go index 14fa7cf..c9cda93 100644 --- a/internal/imageproxy/stats_internal_test.go +++ b/internal/imageproxy/stats_internal_test.go @@ -94,7 +94,7 @@ type fakeUpstream struct { } func (u *fakeUpstream) Fetch( - context.Context, imagerequest.Source, + context.Context, imagerequest.Source, string, string, ) (*httpfetcher.FetchResult, error) { return &httpfetcher.FetchResult{ Content: io.NopCloser(u), diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index ab8a5d2..87c0608 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -3,8 +3,12 @@ // - the variants made from them // - the negative cache: the source URLs whose fetch failed recently // -// Each image is a file with a row in the database. Eviction removes the -// least recently used images to keep the cache under its size limit. +// Each image is a file with a row in the database. A source is used, and its +// variants are served, until the cache TTL has passed since upstream last sent +// the source or confirmed it unchanged. Then the source is due to be checked +// with upstream again (see Stale). Eviction removes: +// - the least recently used images, to keep the cache under its size limit +// - the images not checked with upstream for EvictionAfterLastCheck package imgcache import ( @@ -13,7 +17,9 @@ import ( "errors" "fmt" "io" + "io/fs" "log/slog" + "os" "path/filepath" "time" @@ -38,11 +44,31 @@ const cleanExpiredBatchSize = 1000 // cache. const DefaultNegativeTTL = 5 * time.Minute +// DefaultCacheTTL is how long after upstream last sent a cached source, or +// answered that it had not changed, pixad uses the source and serves its +// variants without asking upstream again. +const DefaultCacheTTL = 24 * time.Hour + +// EvictionAfterLastCheck is how long after upstream last sent a cached +// source, or answered that it had not changed, the source and its variants +// are evicted (see EvictNotCheckedRecently). Until then a source whose checks +// fail is used. +const EvictionAfterLastCheck = 30 * 24 * time.Hour + +// ErrStale is returned by Variant for a variant whose file's modification +// time is CacheTTL or more ago (see Variant). +var ErrStale = errors.New("cached variant is older than the cache TTL") + // Config holds cache configuration. type Config struct { StateDir string NegativeTTL time.Duration + // CacheTTL is how long a cached source is used, and its variants served, + // before upstream is asked again whether the source changed. Zero means + // DefaultCacheTTL. + CacheTTL time.Duration + // MaxBytes is the disk cache size limit in bytes that eviction // enforces. Zero turns the disk cache off entirely: no cache // directories are created, lookups always miss, stores are no-ops, @@ -113,6 +139,10 @@ func NewCache( log = slog.Default() } + if config.CacheTTL == 0 { + config.CacheTTL = DefaultCacheTTL + } + c := &Cache{ db: db, config: config, @@ -156,43 +186,167 @@ func NewCache( return c, nil } -// Variant opens a cached variant and returns a reader, its size and its -// content type, read from the variant file's first bytes. It opens that one -// file and makes no database call. A variant it opens has its access time -// handed to the background writer without waiting (see touchVariant). A -// variant not in the cache, or any variant of a disabled cache, is -// ErrNotFound; a variant file whose first bytes are not those of an output -// format is another error. -func (c *Cache) Variant( - cacheKey imagerequest.VariantKey, -) (io.ReadCloser, int64, string, error) { +// VariantFile is the open file of a cached variant, as Variant, +// VariantIgnoringTTL and RenewVariant return it. +type VariantFile struct { + // Content reads the file. The caller closes it. + Content io.ReadCloser + // Size is the file's size in bytes. + Size int64 + // ContentType is the MIME type of the variant's format, read from the + // file's first bytes. + ContentType string + // CheckedAt is the file's modification time: its source's check time + // when the variant was made or last renewed. + CheckedAt time.Time +} + +// Variant opens a cached variant and returns its file (see VariantFile). It +// opens that one file and makes no database call: the file's modification +// time is its source's check time when the variant was made or last renewed +// (see PutVariant), and a variant whose time is CacheTTL or more ago is +// ErrStale. A variant it opens has its access time handed to the background +// writer without waiting (see touchVariant). A variant not in the cache, or +// any variant of a disabled cache, is ErrNotFound; a variant file whose first +// bytes are not those of an output format is another error. +func (c *Cache) Variant(cacheKey imagerequest.VariantKey) (VariantFile, error) { if c.config.MaxBytes == 0 { - return nil, 0, "", ErrNotFound + return VariantFile{}, ErrNotFound } - file, size, err := c.variants.Open(string(cacheKey)) + file, info, err := c.variants.Open(string(cacheKey)) if err != nil { - return nil, 0, "", err + return VariantFile{}, err + } + + if c.Stale(info.ModTime()) { + _ = file.Close() + + return VariantFile{}, ErrStale + } + + return c.finishOpeningVariant(cacheKey, file, info) +} + +// VariantIgnoringTTL opens a cached variant whatever its age, and returns it +// as Variant does. The image proxy reads it for a variant whose source is not +// cached, before it checks the source with upstream (see LookupVariant). +func (c *Cache) VariantIgnoringTTL( + cacheKey imagerequest.VariantKey, +) (VariantFile, error) { + if c.config.MaxBytes == 0 { + return VariantFile{}, ErrNotFound + } + + file, info, err := c.variants.Open(string(cacheKey)) + if err != nil { + return VariantFile{}, err + } + + return c.finishOpeningVariant(cacheKey, file, info) +} + +// RenewVariant opens a cached variant made from the source bytes that source +// describes, whatever its age, and returns it as Variant does. It moves the +// variant's check time forward to source.CheckedAt (see SetVariantCheckedAt). +// A variant made before source.FetchedAt was made from older bytes, and is +// ErrNotFound, as is a variant not in the cache. +func (c *Cache) RenewVariant( + ctx context.Context, cacheKey imagerequest.VariantKey, source SourceInfo, +) (VariantFile, error) { + if c.config.MaxBytes == 0 { + return VariantFile{}, ErrNotFound + } + + file, info, err := c.variants.Open(string(cacheKey)) + if err != nil { + return VariantFile{}, err + } + + if info.ModTime().Before(source.FetchedAt) { + _ = file.Close() + + return VariantFile{}, ErrNotFound } format, err := readFormat(file) if err != nil { _ = file.Close() - return nil, 0, "", err + return VariantFile{}, err + } + + checkedAt := info.ModTime() + + if checkedAt.Before(source.CheckedAt) { + err = c.SetVariantCheckedAt(ctx, cacheKey, source.CheckedAt) + if err != nil { + _ = file.Close() + + return VariantFile{}, err + } + + checkedAt = source.CheckedAt } c.touchVariant(cacheKey) - return file, size, format.MIMEType(), nil + return VariantFile{ + Content: file, + Size: info.Size(), + ContentType: format.MIMEType(), + CheckedAt: checkedAt, + }, nil } -// PutVariant stores a processed variant by its cache key and records -// it in the size accounting. On a disabled cache it is a no-op. The -// accounting insert is best-effort (the startup reconciliation pass -// adopts any variant file that misses its accounting row). +// SetVariantCheckedAt records checkedAt as when upstream last sent the source +// bytes the cached variant of cacheKey was made from, or answered that they +// had not changed. It sets the variant file's modification time to checkedAt, +// so Variant serves the file until CacheTTL after it. It sets the checked_at +// of the variant's row too, so eviction keeps the variant until +// EvictionAfterLastCheck after it. A failure to write the row is only logged. +// On a disabled cache it is a no-op. +func (c *Cache) SetVariantCheckedAt( + ctx context.Context, cacheKey imagerequest.VariantKey, checkedAt time.Time, +) error { + if c.config.MaxBytes == 0 { + return nil + } + + err := c.variants.SetModTime(string(cacheKey), checkedAt) + if err != nil { + return err + } + + _, err = c.db.ExecContext(ctx, + `UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`, + checkedAt.UTC(), string(cacheKey)) + if err != nil { + c.log.Warn("failed to renew variant's checked_at", + "cache_key", cacheKey, "error", err) + } + + return nil +} + +// Stale reports whether a source last checked with upstream at checkedAt, +// and its variants, are due to be checked again. +func (c *Cache) Stale(checkedAt time.Time) bool { + return time.Since(checkedAt) >= c.config.CacheTTL +} + +// PutVariant stores a processed variant by its cache key and records it in +// the size accounting. source describes the source bytes the variant was made +// from. Its CheckedAt becomes the modification time of the variant's file, +// which Variant and RenewVariant read. The variant's row keeps its CheckedAt, +// ETag and LastModified (see LookupVariant). On a disabled cache it is a +// no-op. The accounting insert is best-effort (the startup reconciliation +// pass adopts any variant file that misses its accounting row). func (c *Cache) PutVariant( - ctx context.Context, cacheKey imagerequest.VariantKey, content io.Reader, + ctx context.Context, + cacheKey imagerequest.VariantKey, + source SourceInfo, + content io.Reader, ) error { if c.config.MaxBytes == 0 { return nil @@ -208,15 +362,33 @@ func (c *Cache) PutVariant( return err } + // A file left with the time of its writing could be served as made from + // bytes fetched after the ones it was made from + err = c.variants.SetModTime(string(cacheKey), source.CheckedAt) + if err != nil { + deleteErr := c.variants.Delete(string(cacheKey)) + if deleteErr != nil { + c.log.Warn("failed to remove variant file whose time was not set", + "cache_key", cacheKey, "error", deleteErr) + } + + return err + } + size := int64(len(data)) _, err = c.db.ExecContext(ctx, ` - INSERT INTO variant_content (cache_key, size_bytes) - VALUES (?, ?) + INSERT INTO variant_content (cache_key, size_bytes, checked_at, + upstream_etag, upstream_last_modified) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(cache_key) DO UPDATE SET size_bytes = excluded.size_bytes, - last_accessed_at = CURRENT_TIMESTAMP - `, string(cacheKey), size) + last_accessed_at = CURRENT_TIMESTAMP, + checked_at = excluded.checked_at, + upstream_etag = excluded.upstream_etag, + upstream_last_modified = excluded.upstream_last_modified + `, string(cacheKey), size, source.CheckedAt.UTC(), source.ETag, + source.LastModified) if err != nil { c.log.Warn("failed to record variant in size accounting", "cache_key", cacheKey, "error", err) @@ -255,7 +427,7 @@ func (c *Cache) Source( return nil, 0, "", ErrNotFound } - file, size, err := c.srcContent.Open(string(key)) + file, info, err := c.srcContent.Open(string(key)) if err != nil { return nil, 0, "", err } @@ -269,19 +441,107 @@ func (c *Cache) Source( c.touchSourceContent(key) - return file, size, format, nil + return file, info.Size(), format, nil } -// PutSource stores fetched source content: it writes the file named by -// the source's key, then the source's row. When the row cannot be written -// the file is removed and the store fails, as a file with no row is never -// served or evicted; if that removal fails too, the next reconciliation -// pass removes the file. On a disabled cache it is a no-op returning an -// empty key. +// SourceInfo is what the cache keeps in a cached source's row besides its +// size and access time. +type SourceInfo struct { + // FetchedAt is when the cached bytes were fetched. + FetchedAt time.Time + // CheckedAt is when upstream last sent the bytes or answered that they + // had not changed. + CheckedAt time.Time + // ETag is the ETag header upstream sent with the bytes, or "". + ETag string + // LastModified is the Last-Modified header upstream sent with the + // bytes, or "". + LastModified string +} + +// LookupSource returns what the cache keeps about the cached source of src. +// A source with no row is ErrNotFound, as is any source of a disabled cache. +func (c *Cache) LookupSource( + ctx context.Context, src imagerequest.Source, +) (SourceInfo, error) { + if c.config.MaxBytes == 0 { + return SourceInfo{}, ErrNotFound + } + + var info SourceInfo + + err := c.db.QueryRowContext(ctx, ` + SELECT fetched_at, checked_at, upstream_etag, upstream_last_modified + FROM source_content WHERE source_key = ? + `, string(src.Key())).Scan( + &info.FetchedAt, &info.CheckedAt, &info.ETag, &info.LastModified) + if errors.Is(err, sql.ErrNoRows) { + return SourceInfo{}, ErrNotFound + } + + if err != nil { + return SourceInfo{}, fmt.Errorf("failed to look up source: %w", err) + } + + return info, nil +} + +// LookupVariant returns what the row of the cached variant of cacheKey keeps +// about the source bytes the variant was made from: the CheckedAt, ETag and +// LastModified they had when the variant was made or last renewed. The row +// keeps no FetchedAt, so it is zero. With the ETag and LastModified the image +// proxy checks the variant with upstream after its source is evicted. A +// variant with no row is ErrNotFound, as is any variant of a disabled cache. +func (c *Cache) LookupVariant( + ctx context.Context, cacheKey imagerequest.VariantKey, +) (SourceInfo, error) { + if c.config.MaxBytes == 0 { + return SourceInfo{}, ErrNotFound + } + + var info SourceInfo + + err := c.db.QueryRowContext(ctx, ` + SELECT checked_at, upstream_etag, upstream_last_modified + FROM variant_content WHERE cache_key = ? + `, string(cacheKey)).Scan(&info.CheckedAt, &info.ETag, &info.LastModified) + if errors.Is(err, sql.ErrNoRows) { + return SourceInfo{}, ErrNotFound + } + + if err != nil { + return SourceInfo{}, fmt.Errorf("failed to look up variant: %w", err) + } + + return info, nil +} + +// RenewSource records that upstream answered at checkedAt that the cached +// source of src had not changed. +func (c *Cache) RenewSource( + ctx context.Context, src imagerequest.Source, checkedAt time.Time, +) error { + _, err := c.db.ExecContext(ctx, + `UPDATE source_content SET checked_at = ? WHERE source_key = ?`, + checkedAt.UTC(), string(src.Key())) + if err != nil { + return fmt.Errorf("failed to renew source: %w", err) + } + + return nil +} + +// PutSource stores fetched source content with what info says about it: it +// writes the file named by the source's key, then the source's row. When the +// row cannot be written the file is removed and the store fails, as a file +// with no row is never served or evicted; if that removal fails too, the +// next reconciliation pass removes the file. On a disabled cache it is a +// no-op returning an empty key. func (c *Cache) PutSource( ctx context.Context, src imagerequest.Source, content io.Reader, + info SourceInfo, ) (imagerequest.SourceKey, error) { if c.config.MaxBytes == 0 { return "", nil @@ -302,12 +562,18 @@ func (c *Cache) PutSource( size := int64(len(data)) _, err = c.db.ExecContext(ctx, ` - INSERT INTO source_content (source_key, size_bytes) - VALUES (?, ?) + INSERT INTO source_content (source_key, size_bytes, fetched_at, + checked_at, upstream_etag, upstream_last_modified) + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(source_key) DO UPDATE SET size_bytes = excluded.size_bytes, - last_accessed_at = CURRENT_TIMESTAMP - `, string(key), size) + last_accessed_at = CURRENT_TIMESTAMP, + fetched_at = excluded.fetched_at, + checked_at = excluded.checked_at, + upstream_etag = excluded.upstream_etag, + upstream_last_modified = excluded.upstream_last_modified + `, string(key), size, info.FetchedAt.UTC(), info.CheckedAt.UTC(), info.ETag, + info.LastModified) if err != nil { deleteErr := c.srcContent.Delete(string(key)) if deleteErr != nil { @@ -449,6 +715,31 @@ func (c *Cache) cleanExpiredBatch(ctx context.Context) (int64, error) { return deleted, nil } +// finishOpeningVariant returns a variant file that FileStore.Open opened, with +// its information info, as Variant does. It reads the content type from the +// file's first bytes and hands the variant's access time to the background +// writer (see touchVariant). It closes the file when its first bytes are not +// those of an output format. +func (c *Cache) finishOpeningVariant( + cacheKey imagerequest.VariantKey, file *os.File, info fs.FileInfo, +) (VariantFile, error) { + format, err := readFormat(file) + if err != nil { + _ = file.Close() + + return VariantFile{}, err + } + + c.touchVariant(cacheKey) + + return VariantFile{ + Content: file, + Size: info.Size(), + ContentType: format.MIMEType(), + CheckedAt: info.ModTime(), + }, nil +} + // readFormat reads the first bytes of a source or variant file, without // moving its offset, and returns the format they show. Sources and variants // are in the same six formats. diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index 828ce50..3d7edb2 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -49,16 +49,19 @@ func TestVariant_ContentTypeFromFirstBytes(t *testing.T) { key := imagerequest.VariantKey("aabbccdd10" + string(format)) - err = cache.PutVariant(t.Context(), key, bytes.NewReader(image)) + err = cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()}, + bytes.NewReader(image)) if err != nil { t.Fatalf("PutVariant(%s) error = %v", format, err) } - reader, size, contentType, err := cache.Variant(key) + variant, err := cache.Variant(key) if err != nil { t.Fatalf("Variant(%s) error = %v", format, err) } + reader, size, contentType := variant.Content, variant.Size, variant.ContentType + served, err := io.ReadAll(reader) _ = reader.Close() @@ -109,16 +112,19 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) { t.Fatalf("%s: storing the variant file: %v", tt.name, err) } - reader, _, contentType, err := cache.Variant(key) + variant, err := cache.Variant(key) t.Logf("%s: Variant() error = %v", tt.name, err) + reader, contentType := variant.Content, variant.ContentType if reader != nil || contentType != "" || !errors.Is(err, tt.wantErr) { t.Errorf("%s: Variant() = reader %v, content type %q, error %v; "+ "want no reader and %v", tt.name, reader, contentType, err, tt.wantErr) } } - reader, _, _, err := cache.Variant(testVariantKeyFour) + variant, err := cache.Variant(testVariantKeyFour) + + reader := variant.Content if reader != nil || !errors.Is(err, ErrNotFound) { t.Errorf("Variant() of a variant with no file = reader %v, error %v; "+ "want no reader and %v", reader, err, ErrNotFound) @@ -126,7 +132,9 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) { disabled, _ := newEvictionTestCache(t, 0) - reader, _, _, err = disabled.Variant(testVariantKeyOne) + variant, err = disabled.Variant(testVariantKeyOne) + + reader = variant.Content if reader != nil || !errors.Is(err, ErrNotFound) { t.Errorf("Variant() on a disabled cache = reader %v, error %v; "+ "want no reader and %v", reader, err, ErrNotFound) diff --git a/internal/imgcache/evict_not_checked_recently_internal_test.go b/internal/imgcache/evict_not_checked_recently_internal_test.go new file mode 100644 index 0000000..a7232af --- /dev/null +++ b/internal/imgcache/evict_not_checked_recently_internal_test.go @@ -0,0 +1,88 @@ +package imgcache + +import ( + "bytes" + "os" + "slices" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// TestEvictNotCheckedRecently stores a source and a variant last checked with +// upstream 31 days ago and a source and a variant checked 29 days ago. +// EvictNotCheckedRecently must remove the rows and files of the first two and +// keep the other two. +func TestEvictNotCheckedRecently(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + old := time.Now().Add(-31 * 24 * time.Hour) + recent := time.Now().Add(-29 * 24 * time.Hour) + + oldSource := imagerequest.Source{Host: testHostCDN, Path: "/old.jpg"} + recentSource := imagerequest.Source{Host: testHostCDN, Path: "/recent.jpg"} + + for src, checkedAt := range map[imagerequest.Source]time.Time{ + oldSource: old, recentSource: recent, + } { + _, err := cache.PutSource(t.Context(), src, bytes.NewReader([]byte("source")), + SourceInfo{FetchedAt: checkedAt, CheckedAt: checkedAt}) + if err != nil { + t.Fatalf("PutSource(%s) error = %v", src.Path, err) + } + } + + oldVariant := imagerequest.VariantKey("aabbccdd0001") + recentVariant := imagerequest.VariantKey("aabbccdd0002") + + for key, checkedAt := range map[imagerequest.VariantKey]time.Time{ + oldVariant: old, recentVariant: recent, + } { + err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: checkedAt}, + bytes.NewReader([]byte("variant"))) + if err != nil { + t.Fatalf("PutVariant(%s) error = %v", key, err) + } + } + + err := cache.EvictNotCheckedRecently(t.Context()) + if err != nil { + t.Fatalf("EvictNotCheckedRecently() error = %v", err) + } + + sourceRows, err := queryStringColumn[imagerequest.SourceKey](t.Context(), + cache.db, `SELECT source_key FROM source_content`, "sources", "source") + if err != nil { + t.Fatal(err) + } + + variantRows, err := queryStringColumn[imagerequest.VariantKey](t.Context(), + cache.db, `SELECT cache_key FROM variant_content`, "variants", "variant") + if err != nil { + t.Fatal(err) + } + + t.Logf("source rows left = %v, variant rows left = %v", sourceRows, variantRows) + + if !slices.Equal(sourceRows, []imagerequest.SourceKey{recentSource.Key()}) { + t.Errorf("source rows left = %v, want only %s", sourceRows, recentSource.Key()) + } + + if !slices.Equal(variantRows, []imagerequest.VariantKey{recentVariant}) { + t.Errorf("variant rows left = %v, want only %s", variantRows, recentVariant) + } + + for path, wantKept := range map[string]bool{ + cache.srcContent.keyToPath(string(oldSource.Key())): false, + cache.srcContent.keyToPath(string(recentSource.Key())): true, + cache.variants.keyToPath(string(oldVariant)): false, + cache.variants.keyToPath(string(recentVariant)): true, + } { + _, statErr := os.Stat(path) + if kept := statErr == nil; kept != wantKept { + t.Errorf("file %s kept = %t, want %t (%v)", path, kept, wantKept, statErr) + } + } +} diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 6746411..3e53c84 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -131,6 +131,56 @@ func (c *Cache) EvictToLimit(ctx context.Context) error { } } +// EvictNotCheckedRecently evicts each source and variant whose checked_at is +// EvictionAfterLastCheck or more ago, evictionBatchSize of each at a time, +// until none is left. It is a no-op when the cache is disabled. +func (c *Cache) EvictNotCheckedRecently(ctx context.Context) error { + if c.config.MaxBytes == 0 { + return nil + } + + cutoff := time.Now().UTC().Add(-EvictionAfterLastCheck) + + for { + sources, err := queryStringColumn[imagerequest.SourceKey](ctx, c.db, ` + SELECT source_key FROM source_content WHERE checked_at < ? LIMIT ? + `, "sources not checked recently", "source key", cutoff, evictionBatchSize) + if err != nil { + return err + } + + for _, key := range sources { + err = c.evictSourceBlob(ctx, key) + if err != nil { + return err + } + } + + variants, err := queryStringColumn[imagerequest.VariantKey](ctx, c.db, ` + SELECT cache_key FROM variant_content WHERE checked_at < ? LIMIT ? + `, "variants not checked recently", "variant key", cutoff, evictionBatchSize) + if err != nil { + return err + } + + for _, key := range variants { + err = c.evictVariant(ctx, key) + if err != nil { + return err + } + } + + if len(sources) > 0 || len(variants) > 0 { + c.log.Info("evicted cache content not checked with upstream recently", + "sources", len(sources), "variants", len(variants)) + } + + if len(sources) < evictionBatchSize && len(variants) < evictionBatchSize { + return nil + } + } +} + // evictBatch fetches one batch of LRU candidates across variants and // source blobs and evicts them oldest-first until excessBytes are // freed or the batch is exhausted. It returns the bytes freed. A @@ -344,8 +394,9 @@ func (c *Cache) notifyWritePressure() { } // StartEviction launches the background eviction goroutine, which -// removes the expired negative cache rows at startup and again on every -// periodic tick thereafter, reconciles the database accounting with the +// removes the expired negative cache rows, and the sources and variants not +// checked for EvictionAfterLastCheck, at startup and again on every periodic +// tick thereafter, reconciles the database accounting with the // cache directories at startup and again on the first periodic tick // that comes at least ten times as long after a reconciliation pass // ended as that pass took, and evicts to the configured limit on the given @@ -392,6 +443,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { nextReconciliation := c.runReconciliationPass(ctx) c.runCleanExpired(ctx) + c.runEvictNotCheckedRecently(ctx) c.runEvictionPass(ctx) ticker := time.NewTicker(interval) @@ -417,6 +469,7 @@ func (c *Cache) evictionLoop(ctx context.Context, interval time.Duration) { } c.runCleanExpired(ctx) + c.runEvictNotCheckedRecently(ctx) case <-c.evictionPressure: } @@ -474,6 +527,21 @@ func (c *Cache) runCleanExpired(ctx context.Context) { } } +// runEvictNotCheckedRecently runs EvictNotCheckedRecently, logging a failure +// instead of propagating it (the loop must keep running). It does nothing +// once ctx is cancelled. +func (c *Cache) runEvictNotCheckedRecently(ctx context.Context) { + if ctx.Err() != nil { + return + } + + err := c.EvictNotCheckedRecently(ctx) + if err != nil { + c.log.Warn("evicting cache content not checked recently failed", + "error", err) + } +} + // reconcileAccounting synchronizes the database size accounting with // the actual contents of the cache directories. It runs at startup and // then on the first periodic eviction tick that comes at least ten times @@ -609,16 +677,18 @@ func (c *Cache) reconcileVariantFile( } // adoptVariantFile inserts an accounting row for a variant file that -// has none, using the file's size and modification time. +// has none, using the file's size and modification time, which is also +// the variant's checked_at. func (c *Cache) adoptVariantFile( ctx context.Context, info fs.FileInfo, cacheKey imagerequest.VariantKey, ) error { modTime := info.ModTime().UTC().Format(sqliteTimestampLayout) _, err := c.db.ExecContext(ctx, ` - INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at) - VALUES (?, ?, ?) - `, string(cacheKey), info.Size(), modTime) + INSERT INTO variant_content (cache_key, size_bytes, last_accessed_at, + checked_at) + VALUES (?, ?, ?, ?) + `, string(cacheKey), info.Size(), modTime, info.ModTime().UTC()) if err != nil { return fmt.Errorf("failed to adopt variant file into accounting: %w", err) } diff --git a/internal/imgcache/eviction_internal_test.go b/internal/imgcache/eviction_internal_test.go index fe64af1..aaefe36 100644 --- a/internal/imgcache/eviction_internal_test.go +++ b/internal/imgcache/eviction_internal_test.go @@ -101,7 +101,7 @@ func storeEvictionTestSource( } key, err := cache.PutSource( - context.Background(), req.Source, bytes.NewReader(content), + context.Background(), req.Source, bytes.NewReader(content), SourceInfo{}, ) if err != nil { t.Fatalf("PutSource(%s%s) failed: %v", host, path, err) @@ -117,7 +117,8 @@ func storeEvictionTestVariant( ) { t.Helper() - err := cache.PutVariant(t.Context(), key, bytes.NewReader(content)) + err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: time.Now()}, + bytes.NewReader(content)) if err != nil { t.Fatalf("PutVariant(%s) failed: %v", key, err) } @@ -432,13 +433,14 @@ func assertDisabledCacheWritesAreNoOps( ctx := t.Context() err := cache.PutVariant( - ctx, imagerequest.CacheKey(req), bytes.NewReader([]byte("data")), + ctx, imagerequest.CacheKey(req), SourceInfo{}, bytes.NewReader([]byte("data")), ) if err != nil { t.Fatalf("PutVariant on disabled cache must be a no-op, got error: %v", err) } - key, err := cache.PutSource(ctx, req.Source, bytes.NewReader([]byte("data"))) + key, err := cache.PutSource(ctx, req.Source, bytes.NewReader([]byte("data")), + SourceInfo{}) if err != nil { t.Fatalf("PutSource on disabled cache must be a no-op, got error: %v", err) } diff --git a/internal/imgcache/filestore.go b/internal/imgcache/filestore.go index 779ff65..b362619 100644 --- a/internal/imgcache/filestore.go +++ b/internal/imgcache/filestore.go @@ -3,8 +3,10 @@ package imgcache import ( "errors" "fmt" + "io/fs" "os" "path/filepath" + "time" ) // Storage constants. @@ -54,26 +56,26 @@ func (s *FileStore) Write(key string, data []byte) error { return writeFileSynced(path, data) } -// Open opens the file for key and returns it with its size in bytes. A -// missing file is ErrNotFound. -func (s *FileStore) Open(key string) (*os.File, int64, error) { +// Open opens the file for key and returns it with its information, which +// gives its size and its modification time. A missing file is ErrNotFound. +func (s *FileStore) Open(key string) (*os.File, fs.FileInfo, error) { f, err := os.Open(s.keyToPath(key)) if err != nil { if os.IsNotExist(err) { - return nil, 0, ErrNotFound + return nil, nil, ErrNotFound } - return nil, 0, fmt.Errorf("failed to open content: %w", err) + return nil, nil, fmt.Errorf("failed to open content: %w", err) } - stat, err := f.Stat() + info, err := f.Stat() if err != nil { _ = f.Close() - return nil, 0, fmt.Errorf("failed to stat content: %w", err) + return nil, nil, fmt.Errorf("failed to stat content: %w", err) } - return f, stat.Size(), nil + return f, info, nil } // Size returns the size in bytes of the file for key. A missing file is @@ -91,6 +93,17 @@ func (s *FileStore) Size(key string) (int64, error) { return stat.Size(), nil } +// SetModTime sets the modification time of the file for key to modTime, +// leaving its access time as it is. +func (s *FileStore) SetModTime(key string, modTime time.Time) error { + err := os.Chtimes(s.keyToPath(key), time.Time{}, modTime) + if err != nil { + return fmt.Errorf("failed to set modification time: %w", err) + } + + return nil +} + // Delete removes the file for key. A missing file is not an error. func (s *FileStore) Delete(key string) error { err := os.Remove(s.keyToPath(key)) diff --git a/internal/imgcache/filestore_internal_test.go b/internal/imgcache/filestore_internal_test.go index 06264c9..1801e7f 100644 --- a/internal/imgcache/filestore_internal_test.go +++ b/internal/imgcache/filestore_internal_test.go @@ -38,15 +38,15 @@ func TestFileStore_WriteAndOpen(t *testing.T) { } // Open and verify size and content - r, size, err := storage.Open(key) + r, info, err := storage.Open(key) if err != nil { t.Fatalf("Open() error = %v", err) } defer func() { _ = r.Close() }() - if size != int64(len(content)) { - t.Errorf("Open() size = %d, want %d", size, len(content)) + if info.Size() != int64(len(content)) { + t.Errorf("Open() size = %d, want %d", info.Size(), len(content)) } loaded, err := io.ReadAll(r) diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go index 60a9be4..d3b3d70 100644 --- a/internal/imgcache/source_content_internal_test.go +++ b/internal/imgcache/source_content_internal_test.go @@ -128,7 +128,7 @@ func TestPutSource_FailedRowWriteLeavesNoFile(t *testing.T) { Source: imagerequest.Source{Host: testHostCDN, Path: testPathCat}, } key, err := cache.PutSource(t.Context(), req.Source, - bytes.NewReader([]byte("source bytes"))) + bytes.NewReader([]byte("source bytes")), SourceInfo{}) t.Logf("PutSource() = %q, error = %v", key, err) if err == nil { @@ -170,7 +170,7 @@ func TestSource_FormatFromFirstBytes(t *testing.T) { t.Fatalf("reading the %s image: %v", format, err) } - _, err = cache.PutSource(t.Context(), src, bytes.NewReader(image)) + _, err = cache.PutSource(t.Context(), src, bytes.NewReader(image), SourceInfo{}) if err != nil { t.Fatalf("PutSource(%s) error = %v", format, err) } diff --git a/internal/providers/providers.go b/internal/providers/providers.go index dda53ca..03824ce 100644 --- a/internal/providers/providers.go +++ b/internal/providers/providers.go @@ -83,6 +83,7 @@ func newCache(lc fx.Lifecycle, params cacheParams) (*imgcache.Cache, error) { cache, err := imgcache.NewCache(db, imgcache.Config{ StateDir: cfg.StateDir, NegativeTTL: imgcache.DefaultNegativeTTL, + CacheTTL: imgcache.DefaultCacheTTL, MaxBytes: maxBytes, Logger: params.Logger.Get(), }, params.MetricsRegistry) From a041e735507e23c1cc169e0e2f9fb08b71e978da Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 14:05:59 +0000 Subject: [PATCH 122/131] Test the eviction, source cache, upstream and transform metrics The tests check the metrics on a variant hit, on misses that fetch or use the cached source, on a failed fetch, and on both kinds of eviction. A source whose cached file is gone must count as a source cache miss. A transform waiting for a processing slot must count as in progress. The tests fail until the metrics exist. Model: opus-5-5 --- internal/imageproxy/metrics_internal_test.go | 213 +++++++++++++++++++ internal/imgcache/metrics_internal_test.go | 137 ++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 internal/imageproxy/metrics_internal_test.go create mode 100644 internal/imgcache/metrics_internal_test.go diff --git a/internal/imageproxy/metrics_internal_test.go b/internal/imageproxy/metrics_internal_test.go new file mode 100644 index 0000000..2745897 --- /dev/null +++ b/internal/imageproxy/metrics_internal_test.go @@ -0,0 +1,213 @@ +package imageproxy + +import ( + "os" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// metricValues holds the source cache, upstream fetch and transform metrics +// of a proxy and its cache. A histogram is given by its count of +// observations. +type metricValues struct { + sourceCacheHits uint64 + sourceCacheMisses uint64 + fetchDurations uint64 + fetchErrors uint64 + transformDuration uint64 + inProgress float64 +} + +// readMetricValues reads the metrics of svc and its cache that +// metricValues holds from their metrics registry. +func readMetricValues(t *testing.T, svc *testProxy) metricValues { + t.Helper() + + families, err := svc.registry.Gather() + if err != nil { + t.Fatalf("failed to gather the metrics: %v", err) + } + + var values metricValues + + for _, family := range families { + metric := family.GetMetric()[0] + + switch family.GetName() { + case "pixa_source_cache_hits_total": + values.sourceCacheHits = uint64(metric.GetCounter().GetValue()) + case "pixa_source_cache_misses_total": + values.sourceCacheMisses = uint64(metric.GetCounter().GetValue()) + case "pixa_upstream_fetch_duration_seconds": + values.fetchDurations = metric.GetHistogram().GetSampleCount() + case "pixa_upstream_fetch_errors_total": + values.fetchErrors = uint64(metric.GetCounter().GetValue()) + case "pixa_transform_duration_seconds": + values.transformDuration = metric.GetHistogram().GetSampleCount() + case "pixa_transforms_in_progress": + values.inProgress = metric.GetGauge().GetValue() + } + } + + return values +} + +// TestProxy_Get_CountsSourceCacheFetchesAndTransforms walks Get through a +// miss that fetches the source, a hit, a miss that uses the cached source +// and a miss whose source is not found upstream, checking the source cache, +// upstream fetch and transform metrics after each. +func TestProxy_Get_CountsSourceCacheFetchesAndTransforms(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + + // want is source cache hits, source cache misses, fetch durations + // observed, fetch errors, transform durations observed, transforms in + // progress. + steps := []struct { + name string + path string + size int + wantErr bool + want metricValues + }{ + {"miss that fetches the source", testPathPhoto, 50, false, + metricValues{0, 1, 1, 0, 1, 0}}, + {"hit", testPathPhoto, 50, false, + metricValues{0, 1, 1, 0, 1, 0}}, + {"miss that uses the cached source", testPathPhoto, 25, false, + metricValues{1, 1, 1, 0, 2, 0}}, + {"miss whose source is not found", "/images/absent.jpg", 50, true, + metricValues{1, 2, 1, 1, 2, 0}}, + } + + for _, step := range steps { + resp, err := svc.Get(t.Context(), &imagerequest.Request{ + Source: imagerequest.Source{ + Host: fixtures.GoodHost, + Path: step.path, + }, + Transform: imagerequest.Transform{ + Size: imagerequest.Size{Width: step.size, Height: step.size}, + Format: imagerequest.FormatJPEG, + Quality: 85, + FitMode: imagerequest.FitCover, + }, + }) + if (err != nil) != step.wantErr { + t.Fatalf("%s: Get() error = %v, want error %t", step.name, err, step.wantErr) + } + + if err == nil { + _ = resp.Content.Close() + } + + got := readMetricValues(t, svc) + t.Logf("after the %s: metrics = %+v", step.name, got) + + if got != step.want { + t.Fatalf("after the %s: metrics = %+v, want %+v", step.name, got, step.want) + } + } +} + +// TestProxy_Get_CountsMissingSourceFileAsSourceCacheMiss deletes the file of +// a cached source but keeps its row. A request for a second variant of that +// source must fetch the source again, and count a source cache miss, not a +// hit. +func TestProxy_Get_CountsMissingSourceFileAsSourceCacheMiss(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + + // A first request caches the photo as a source. + resp, err := svc.Get(t.Context(), widthOnlyRequest(fixtures, 50)) + if err != nil { + t.Fatalf("first Get() error = %v", err) + } + + _ = resp.Content.Close() + + source := widthOnlyRequest(fixtures, 50).Source + + err = os.Remove(sourceFilePath(svc, source.Key())) + if err != nil { + t.Fatalf("failed to delete the cached source file: %v", err) + } + + before := readMetricValues(t, svc) + + resp, err = svc.Get(t.Context(), widthOnlyRequest(fixtures, 25)) + if err != nil { + t.Fatalf("Get() of a second variant error = %v", err) + } + + _ = resp.Content.Close() + + after := readMetricValues(t, svc) + t.Logf("before the second variant: metrics = %+v; after: %+v", before, after) + + if after.sourceCacheHits != before.sourceCacheHits || + after.sourceCacheMisses != before.sourceCacheMisses+1 { + t.Errorf("source cache hits went from %d to %d and misses from %d to %d, "+ + "want no new hit and one more miss", before.sourceCacheHits, + after.sourceCacheHits, before.sourceCacheMisses, after.sourceCacheMisses) + } +} + +// TestProxy_Get_CountsTransformWaitingForSlotAsInProgress holds the only +// processing slot and starts a request that must transform the photo. While +// the request waits for the slot, pixa_transforms_in_progress must be 1; once +// the request returns, 0. +func TestProxy_Get_CountsTransformWaitingForSlotAsInProgress(t *testing.T) { + t.Parallel() + + svc, fixtures := setupTestProxy(t) + svc.processor = imageprocessor.New( + imageprocessor.Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}, + ) + + release := holdProcessingSlot(t, svc.processor) + + result := startGet(t.Context(), svc, widthOnlyRequest(fixtures, 50)) + + // The request fetches the photo first, then waits for the slot. + deadline := time.Now().Add(5 * time.Second) + for readMetricValues(t, svc).inProgress != 1 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + + waiting := readMetricValues(t, svc).inProgress + t.Logf("while the request waits for the slot: transforms in progress = %v", + waiting) + + select { + case got := <-result: + t.Fatalf("Get() returned while the only processing slot was held, "+ + "error = %v", got.err) + default: + } + + if waiting != 1 { + t.Fatalf("transforms in progress = %v while the request waits for the "+ + "slot, want 1", waiting) + } + + release() + + got := <-result + if got.err != nil { + t.Fatalf("Get() error = %v", got.err) + } + + returned := readMetricValues(t, svc).inProgress + t.Logf("once the request returned: transforms in progress = %v", returned) + + if returned != 0 { + t.Errorf("transforms in progress = %v once the request returned, want 0", + returned) + } +} diff --git a/internal/imgcache/metrics_internal_test.go b/internal/imgcache/metrics_internal_test.go new file mode 100644 index 0000000..461e047 --- /dev/null +++ b/internal/imgcache/metrics_internal_test.go @@ -0,0 +1,137 @@ +package imgcache + +import ( + "bytes" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// gatheredValue returns the value of the metric name on registry: a +// counter's or a gauge's value. +func gatheredValue(t *testing.T, registry *prometheus.Registry, name string) float64 { + t.Helper() + + families, err := registry.Gather() + if err != nil { + t.Fatalf("failed to gather the metrics: %v", err) + } + + for _, family := range families { + if family.GetName() != name { + continue + } + + metric := family.GetMetric()[0] + if metric.GetGauge() != nil { + return metric.GetGauge().GetValue() + } + + return metric.GetCounter().GetValue() + } + + t.Fatalf("the metric %s is not on the registry", name) + + return 0 +} + +// TestEvictToLimit_CountsEvictionsAndUsage stores four variants of 1000 +// bytes in a cache limited to 3000 bytes. EvictToLimit must count the one +// eviction and its bytes, and set the usage and limit gauges. +func TestEvictToLimit_CountsEvictionsAndUsage(t *testing.T) { + t.Parallel() + + const limit = 3000 + + registry := prometheus.NewRegistry() + + cache, err := NewCache(evictionTestDB(t), + Config{StateDir: t.TempDir(), MaxBytes: limit}, registry) + if err != nil { + t.Fatalf("NewCache() error = %v", err) + } + + keys := []imagerequest.VariantKey{ + testVariantKeyOne, testVariantKeyTwo, + testVariantKeyThree, testVariantKeyFour, + } + + for i, key := range keys { + storeEvictionTestVariant(t, cache, key, bytes.Repeat([]byte{0x01}, 1000)) + setVariantLastAccessed(t, cache, key, + time.Now().Add(-time.Duration(len(keys)-i)*time.Hour)) + } + + err = cache.EvictToLimit(t.Context()) + if err != nil { + t.Fatalf("EvictToLimit() error = %v", err) + } + + want := map[string]float64{ + "pixa_cache_evictions_total": 1, + "pixa_cache_evicted_bytes_total": 1000, + "pixa_cache_usage_bytes": 3000, + "pixa_cache_max_bytes": limit, + } + + for name, value := range want { + got := gatheredValue(t, registry, name) + t.Logf("%s = %v", name, got) + + if got != value { + t.Errorf("%s = %v, want %v", name, got, value) + } + } +} + +// TestEvictNotCheckedRecently_CountsEvictions stores a source of 6 bytes and +// a variant of 7 bytes last checked with upstream 31 days ago. +// EvictNotCheckedRecently must count both evictions and their 13 bytes. +func TestEvictNotCheckedRecently_CountsEvictions(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + + cache, err := NewCache(evictionTestDB(t), + Config{StateDir: t.TempDir(), MaxBytes: 1 << 30}, registry) + if err != nil { + t.Fatalf("NewCache() error = %v", err) + } + + old := time.Now().Add(-31 * 24 * time.Hour) + + _, err = cache.PutSource(t.Context(), + imagerequest.Source{Host: testHostCDN, Path: "/old.jpg"}, + bytes.NewReader([]byte("source")), + SourceInfo{FetchedAt: old, CheckedAt: old}) + if err != nil { + t.Fatalf("PutSource() error = %v", err) + } + + err = cache.PutVariant(t.Context(), testVariantKeyOne, + SourceInfo{CheckedAt: old}, bytes.NewReader([]byte("variant"))) + if err != nil { + t.Fatalf("PutVariant() error = %v", err) + } + + err = cache.EvictNotCheckedRecently(t.Context()) + if err != nil { + t.Fatalf("EvictNotCheckedRecently() error = %v", err) + } + + want := map[string]float64{ + "pixa_cache_evictions_total": 2, + "pixa_cache_evicted_bytes_total": 13, + } + + for name, value := range want { + got := gatheredValue(t, registry, name) + t.Logf("%s = %v", name, got) + + if got != value { + t.Errorf("%s = %v, want %v", name, got, value) + } + } +} From fa2ec3da6a9a24b302bb2d960c20de6e4ae67c94 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 11 Nov 2026 14:06:06 +0000 Subject: [PATCH 123/131] Export cache, eviction, upstream and transform metrics (closes #85) New pixa_ metrics, none with labels: - source cache hits and misses, counted once per variant made or checked; a source whose cached file cannot be used is a miss - cache usage and limit gauges; usage is set each time eviction reads it - eviction count and bytes, from the size_bytes the deleted row returns - upstream fetch duration histogram and fetch error counter - transform duration histogram and an in-progress gauge, both including the wait for a processing slot Removing a damaged file during reconciliation is not counted as an eviction. README lists every metric; the metrics config comment now says the pixa_ metrics are measured whether or not /metrics is served. Model: opus-5-5 --- README.md | 23 +++++ TODO.md | 7 -- configs/config.example.yml | 5 +- internal/imageproxy/imageproxy.go | 138 +++++++++++++++++++++++------- internal/imgcache/cache.go | 29 +++---- internal/imgcache/counters.go | 31 ------- internal/imgcache/eviction.go | 109 ++++++++++++++++++----- internal/imgcache/metrics.go | 88 +++++++++++++++++++ internal/providers/providers.go | 2 +- 9 files changed, 319 insertions(+), 113 deletions(-) delete mode 100644 internal/imgcache/counters.go create mode 100644 internal/imgcache/metrics.go diff --git a/README.md b/README.md index 1cc426f..9522af0 100644 --- a/README.md +++ b/README.md @@ -1167,10 +1167,33 @@ Each item below is one setting, named by its config key: - the Go runtime and process metrics - `pixa_cache_hits_total`: image requests served from the cache - `pixa_cache_misses_total`: image requests not served from the cache + - `pixa_source_cache_hits_total`: variants made or checked whose source was + in the cache + - `pixa_source_cache_misses_total`: variants made or checked whose source + was not in the cache + - `pixa_cache_usage_bytes`: the bytes of sources and variants in the cache, + as eviction last read them; eviction runs after each store and every five + minutes + - `pixa_cache_max_bytes`: the disk cache limit, `cache_max_bytes` + - `pixa_cache_evictions_total`: sources and variants removed by eviction + - `pixa_cache_evicted_bytes_total`: the bytes those evictions freed - `pixa_upstream_fetches_total`: fetches of a source image that read any bytes - `pixa_upstream_fetch_bytes_total`: the bytes those fetches read + - `pixa_upstream_fetch_duration_seconds`: a histogram of how long those + fetches took, up to the last byte of the response + - `pixa_upstream_fetch_errors_total`: fetches of a source image that failed, + before or while reading the response - `pixa_transforms_total`: images converted + - `pixa_transform_duration_seconds`: a histogram of how long those + conversions took, including the wait for a processing slot + - `pixa_transforms_in_progress`: images being converted or waiting for a + processing slot + + pixa measures the `pixa_` metrics whether or not `/metrics` is served. They + have no labels, so their number never grows with the hosts and paths pixa + serves. Requests that share work (see Limits) count one source cache hit or + miss between them. A request's `handler` label is the pattern of the route it matches, such as `/v1/image/*`, never its path. Requests that match no route share the label diff --git a/TODO.md b/TODO.md index 9c99a32..b30de34 100644 --- a/TODO.md +++ b/TODO.md @@ -27,12 +27,5 @@ P2: security: per-origin rate limiting - P2: HTTP response handling - Last-Modified headers - P2: operational - - Prometheus metrics still missing: - - cache hits and misses, split into source images and variants - - cache size on disk and its limit - - evictions and the bytes they free - - upstream fetch duration and errors - - transcode duration - - images in processing - measure the 1k to 5k req/s target with `script/loadtest` on a machine not shared with other work diff --git a/configs/config.example.yml b/configs/config.example.yml index 6a0ab17..33a062e 100644 --- a/configs/config.example.yml +++ b/configs/config.example.yml @@ -158,8 +158,9 @@ access_control_allow_origin: "*" # Sentry DSN for error reporting (default: empty, which turns it off) sentry_dsn: "" -# Username and password for /metrics, set together (default: unset). Metrics -# are measured and /metrics is served only when both are set. +# Username and password for /metrics, set together (default: unset). pixa +# serves /metrics, and measures HTTP requests, only when both are set. It +# measures the pixa_ metrics either way. # metrics: # username: "admin" # password: "secret" diff --git a/internal/imageproxy/imageproxy.go b/internal/imageproxy/imageproxy.go index ea7243a..1b46127 100644 --- a/internal/imageproxy/imageproxy.go +++ b/internal/imageproxy/imageproxy.go @@ -37,11 +37,15 @@ type Proxy struct { // same time share one fetch and one transcode. variantsInProgress singleflight.Group - // The counts of upstream fetches and transforms, made and registered by - // New. They are kept in memory, so adding to one waits for nothing. - upstreamFetches prometheus.Counter - upstreamFetchBytes prometheus.Counter - transforms prometheus.Counter + // The metrics of upstream fetches and transforms, made and registered by + // New. They are kept in memory, so updating one waits for nothing. + upstreamFetches prometheus.Counter + upstreamFetchBytes prometheus.Counter + upstreamFetchDurations prometheus.Histogram + upstreamFetchErrors prometheus.Counter + transforms prometheus.Counter + transformDurations prometheus.Histogram + transformsInProgress prometheus.Gauge } // Config holds the proxy's configuration. @@ -62,9 +66,9 @@ type Config struct { // errCacheRequired is returned when the proxy is built without a cache. var errCacheRequired = errors.New("cache is required") -// New creates a new image proxy and registers its counters on +// New creates a new image proxy and registers its metrics on // metricsRegistry. pixad passes Prometheus' default registry, the one -// /metrics serves. It fails when metricsRegistry already holds counters of +// /metrics serves. It fails when metricsRegistry already holds metrics of // the same names. func New( cfg *Config, metricsRegistry prometheus.Registerer, @@ -99,18 +103,39 @@ func New( Name: "pixa_upstream_fetch_bytes_total", Help: "Bytes read from upstream by fetches of source images.", }), + upstreamFetchDurations: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "pixa_upstream_fetch_duration_seconds", + Help: "Duration of the fetches pixa_upstream_fetches_total counts, " + + "until the response's last byte.", + }), + upstreamFetchErrors: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_upstream_fetch_errors_total", + Help: "Fetches of a source image from upstream that failed, " + + "before or while reading the response.", + }), transforms: prometheus.NewCounter(prometheus.CounterOpts{ Name: "pixa_transforms_total", Help: "Images the image processor transformed.", }), + transformDurations: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "pixa_transform_duration_seconds", + Help: "Duration of the transforms pixa_transforms_total counts, " + + "waiting for a processing slot included.", + }), + transformsInProgress: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "pixa_transforms_in_progress", + Help: "Images being transformed or waiting for a processing slot.", + }), } - for _, counter := range []prometheus.Counter{ - p.upstreamFetches, p.upstreamFetchBytes, p.transforms, + for _, metric := range []prometheus.Collector{ + p.upstreamFetches, p.upstreamFetchBytes, p.upstreamFetchDurations, + p.upstreamFetchErrors, p.transforms, p.transformDurations, + p.transformsInProgress, } { - err = metricsRegistry.Register(counter) + err = metricsRegistry.Register(metric) if err != nil { - return nil, fmt.Errorf("failed to register the proxy's counters: %w", err) + return nil, fmt.Errorf("failed to register the proxy's metrics: %w", err) } } @@ -341,7 +366,10 @@ func (p *Proxy) loadCachedSource( } // processFromSourceOrFetch makes the variant req asks for, from the cached -// source when there is one and from a fetched source otherwise. +// source when there is one and from a fetched source otherwise. It counts a +// source cache miss when it fetches the source because the source has no row +// in the cache or its cached file cannot be used, and a source cache hit +// otherwise. // // A cached source last checked with upstream CacheTTL or more ago is checked // first: upstream is asked to send it only if it changed since. When it @@ -362,6 +390,8 @@ func (p *Proxy) processFromSourceOrFetch( p.log.Warn("failed to look up cached source, fetching", "error", err) } + p.cache.IncrementSourceStats(false) + return p.fetchAndProcess(ctx, req, cacheKey) } @@ -375,16 +405,21 @@ func (p *Proxy) processFromSourceOrFetch( case err != nil: p.recordFailedCheck(ctx, req, err) default: + p.cache.IncrementSourceStats(true) + return p.processFetched(ctx, req, cacheKey, fetched) } } variant := p.renewVariant(ctx, cacheKey, cached) if variant != nil { + p.cache.IncrementSourceStats(true) + return variant, nil } source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source) + p.cache.IncrementSourceStats(source != nil) // Fetch from upstream if we don't have source data or it's empty if source == nil { @@ -581,36 +616,32 @@ type fetchedSource struct { // fetchSource fetches req's source from upstream and stores it in the cache. // A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source // only if it changed since; errSourceNotModified says it did not. It counts -// the fetch with the bytes read from upstream, including when reading the -// response or a later step fails. It fetches nothing while the source URL is -// in the negative cache. The caller closes the returned response. +// the fetch with the bytes read from upstream and its duration, including +// when reading the response or a later step fails, and counts a fetch that +// fails or whose response cannot be read as an upstream fetch error. It +// fetches nothing while the source URL is in the negative cache. The caller +// closes the returned response. func (p *Proxy) fetchSource( ctx context.Context, req *imagerequest.Request, ifNoneMatch, ifModifiedSince string, ) (*fetchedSource, error) { - // Check negative cache - skip fetching for recently-failed URLs - negHit, err := p.cache.Failure(ctx, req.Source) + err := p.refuseNegativeCached(ctx, req) if err != nil { - p.log.Warn("negative cache check failed", "error", err) - } - - if negHit { - p.log.Debug("negative cache hit", - "host", req.Source.Host, - "path", req.Source.Path, - ) - - return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached) + return nil, err } // Fetch from upstream p.log.Debug("fetching from upstream", "host", req.Source.Host, "path", req.Source.Path, "query", req.Source.Query) + fetchStart := time.Now() + fetchResult, err := p.fetcher.Fetch(ctx, req.Source, ifNoneMatch, ifModifiedSince) if err != nil { + p.upstreamFetchErrors.Inc() + // Store negative cache for certain errors if isNegativeCacheable(err) { p.storeNegative(ctx, req) @@ -634,9 +665,11 @@ func (p *Proxy) fetchSource( fetchBytes := int64(len(sourceData)) // Counted also when the request context has ended meanwhile - p.incrementUpstreamFetch(fetchBytes) + p.incrementUpstreamFetch(fetchBytes, time.Since(fetchStart)) if err != nil { + p.upstreamFetchErrors.Inc() + _ = fetchResult.Content.Close() return nil, fmt.Errorf("failed to read upstream response: %w", err) @@ -677,6 +710,29 @@ func (p *Proxy) fetchSource( }, nil } +// refuseNegativeCached returns an error wrapping ErrNegativeCached while +// req's source URL is in the negative cache, and nil otherwise. A failure to +// read the negative cache is only logged, and returns nil. +func (p *Proxy) refuseNegativeCached( + ctx context.Context, req *imagerequest.Request, +) error { + negHit, err := p.cache.Failure(ctx, req.Source) + if err != nil { + p.log.Warn("negative cache check failed", "error", err) + } + + if !negHit { + return nil + } + + p.log.Debug("negative cache hit", + "host", req.Source.Host, + "path", req.Source.Path, + ) + + return fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached) +} + // fetchedSourceInfo returns what the cache keeps about a source upstream // just sent: fetched and checked now, with the ETag and Last-Modified // headers of fetchResult. @@ -722,15 +778,16 @@ func (p *Proxy) logUpstreamFetch( ) } -// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes. -// A fetch that read no bytes is not counted. -func (p *Proxy) incrementUpstreamFetch(fetchBytes int64) { +// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes +// and took duration. A fetch that read no bytes is not counted. +func (p *Proxy) incrementUpstreamFetch(fetchBytes int64, duration time.Duration) { if fetchBytes <= 0 { return } p.upstreamFetches.Inc() p.upstreamFetchBytes.Add(float64(fetchBytes)) + p.upstreamFetchDurations.Observe(duration.Seconds()) } // storeNegative stores a negative cache entry for req's source, whose fetch @@ -772,8 +829,7 @@ func (p *Proxy) processAndStore( // ctx has the deadline of the request doing the processing, which can // end while a request waiting for its image has time left - processResult, err := p.processor.Process(ctx, source, sourceFormat, - &req.Transform) + processResult, err := p.transform(ctx, source, sourceFormat, &req.Transform) if errors.Is(err, context.DeadlineExceeded) { return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err) } @@ -786,6 +842,7 @@ func (p *Proxy) processAndStore( // Counted also when the request context has ended meanwhile p.transforms.Inc() + p.transformDurations.Observe(processDuration.Seconds()) // Read processed content processedData, err := io.ReadAll(processResult.Content) @@ -838,6 +895,21 @@ func (p *Proxy) processAndStore( }, nil } +// transform has the image processor transform the image read from source +// as requested says, and counts it in pixa_transforms_in_progress until the +// processor returns or panics. +func (p *Proxy) transform( + ctx context.Context, + source io.Reader, + sourceFormat imagerequest.Format, + requested *imagerequest.Transform, +) (*imageprocessor.Result, error) { + p.transformsInProgress.Inc() + defer p.transformsInProgress.Dec() + + return p.processor.Process(ctx, source, sourceFormat, requested) +} + // isNegativeCacheable returns true if the error should be cached. func isNegativeCacheable(err error) bool { return errors.Is(err, httpfetcher.ErrUpstreamError) diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index 87c0608..e0e4748 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -111,10 +111,16 @@ type Cache struct { pendingAccessTimesDone chan struct{} pendingAccessTimesCancel context.CancelFunc - // The counts of cache hits and misses, made and registered by - // registerCounters (see counters.go) - cacheHits prometheus.Counter - cacheMisses prometheus.Counter + // The cache's metrics, made and registered by registerMetrics (see + // metrics.go) + cacheHits prometheus.Counter + cacheMisses prometheus.Counter + sourceCacheHits prometheus.Counter + sourceCacheMisses prometheus.Counter + usageBytes prometheus.Gauge + maxBytes prometheus.Gauge + evictions prometheus.Counter + evictedBytes prometheus.Counter // reconciliationPageSize is the most rows one read of a content table // returns in the reconciliation pass. NewCache sets it to @@ -128,7 +134,7 @@ type Cache struct { reconciliationReadTestHook func(rows int) } -// NewCache creates a new cache instance and registers its counters on +// NewCache creates a new cache instance and registers its metrics on // metricsRegistry. pixad passes Prometheus' default registry, the one // /metrics serves. It makes no database call. func NewCache( @@ -157,7 +163,7 @@ func NewCache( reconciliationPageSize: defaultReconciliationPageSize, } - err := c.registerCounters(metricsRegistry) + err := c.registerMetrics(metricsRegistry) if err != nil { return nil, err } @@ -656,17 +662,6 @@ func (c *Cache) CleanExpired(ctx context.Context) error { } } -// IncrementStats counts a cache hit or miss. It only adds to the cache's -// Prometheus counters in memory (see counters.go), so it never waits for the -// database or fails. -func (c *Cache) IncrementStats(hit bool) { - if hit { - c.cacheHits.Inc() - } else { - c.cacheMisses.Inc() - } -} - // touchVariant hands the access time of a variant served from the cache to // the goroutine StartPendingAccessTimeWrites starts, which writes it in its // next batch. It never waits: while maxPendingAccessTimes access times of diff --git a/internal/imgcache/counters.go b/internal/imgcache/counters.go deleted file mode 100644 index edb14ac..0000000 --- a/internal/imgcache/counters.go +++ /dev/null @@ -1,31 +0,0 @@ -package imgcache - -import ( - "fmt" - - "github.com/prometheus/client_golang/prometheus" -) - -// registerCounters makes the cache's counters and registers them on -// metricsRegistry. The counters are kept in memory, so adding to one waits -// for nothing and makes no database call. It fails when metricsRegistry -// already holds counters of the same names. -func (c *Cache) registerCounters(metricsRegistry prometheus.Registerer) error { - c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "pixa_cache_hits_total", - Help: "Image requests served from the cache.", - }) - c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "pixa_cache_misses_total", - Help: "Image requests not served from the cache.", - }) - - for _, counter := range []prometheus.Counter{c.cacheHits, c.cacheMisses} { - err := metricsRegistry.Register(counter) - if err != nil { - return fmt.Errorf("failed to register the cache's counters: %w", err) - } - } - - return nil -} diff --git a/internal/imgcache/eviction.go b/internal/imgcache/eviction.go index 3e53c84..c3cc1de 100644 --- a/internal/imgcache/eviction.go +++ b/internal/imgcache/eviction.go @@ -92,8 +92,9 @@ type evictionCandidate struct { } // EvictToLimit evicts least-recently-used cache entries until total -// tracked usage is at or below the configured MaxBytes limit. It is a -// no-op when the cache is disabled. +// tracked usage is at or below the configured MaxBytes limit. It sets the +// pixa_cache_usage_bytes gauge to each usage it reads. It is a no-op when +// the cache is disabled. func (c *Cache) EvictToLimit(ctx context.Context) error { if c.config.MaxBytes == 0 { return nil @@ -105,6 +106,8 @@ func (c *Cache) EvictToLimit(ctx context.Context) error { return err } + c.usageBytes.Set(float64(usage)) + if usage <= c.config.MaxBytes { return nil } @@ -345,37 +348,95 @@ func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, erro return candidates, nil } -// evictVariant removes one variant: accounting row first, then the -// file, so the database never references a deleted file. +// evictVariant removes one variant (see removeVariant) and counts the +// eviction with the size its row recorded. func (c *Cache) evictVariant( ctx context.Context, cacheKey imagerequest.VariantKey, ) error { - _, err := c.db.ExecContext(ctx, - `DELETE FROM variant_content WHERE cache_key = ?`, string(cacheKey)) - if err != nil { - return fmt.Errorf("failed to delete variant accounting row: %w", err) - } - - err = c.variants.Delete(string(cacheKey)) + sizeBytes, err := c.removeVariant(ctx, cacheKey) if err != nil { return err } + c.countEviction(sizeBytes) + return nil } -// evictSourceBlob removes one source: its row first, then its file, as -// evictVariant does for a variant. It holds no lock: a store of the same -// source in between can leave the source's row with no file, which -// Source treats as a miss. -func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error { - _, err := c.db.ExecContext(ctx, - `DELETE FROM source_content WHERE source_key = ?`, string(key)) +// removeVariant removes one variant: accounting row first, then the +// file, so the database never references a deleted file. It returns the +// size the row recorded, or 0 when the variant had no row. +func (c *Cache) removeVariant( + ctx context.Context, cacheKey imagerequest.VariantKey, +) (int64, error) { + sizeBytes, err := c.deleteRowReturningSize(ctx, + `DELETE FROM variant_content WHERE cache_key = ? RETURNING size_bytes`, + string(cacheKey)) if err != nil { - return fmt.Errorf("failed to delete source content row: %w", err) + return 0, fmt.Errorf("failed to delete variant accounting row: %w", err) } - return c.srcContent.Delete(string(key)) + err = c.variants.Delete(string(cacheKey)) + if err != nil { + return 0, err + } + + return sizeBytes, nil +} + +// evictSourceBlob removes one source (see removeSourceBlob) and counts the +// eviction with the size its row recorded. +func (c *Cache) evictSourceBlob(ctx context.Context, key imagerequest.SourceKey) error { + sizeBytes, err := c.removeSourceBlob(ctx, key) + if err != nil { + return err + } + + c.countEviction(sizeBytes) + + return nil +} + +// removeSourceBlob removes one source: its row first, then its file, as +// removeVariant does for a variant, and returns the size the row recorded, +// or 0 when the source had no row. It holds no lock: a store of the same +// source in between can leave the source's row with no file, which Source +// treats as a miss. +func (c *Cache) removeSourceBlob( + ctx context.Context, key imagerequest.SourceKey, +) (int64, error) { + sizeBytes, err := c.deleteRowReturningSize(ctx, + `DELETE FROM source_content WHERE source_key = ? RETURNING size_bytes`, + string(key)) + if err != nil { + return 0, fmt.Errorf("failed to delete source content row: %w", err) + } + + err = c.srcContent.Delete(string(key)) + if err != nil { + return 0, err + } + + return sizeBytes, nil +} + +// deleteRowReturningSize runs deleteQuery, which deletes the row of key and +// returns its size_bytes, and returns that size, or 0 when there was no row. +func (c *Cache) deleteRowReturningSize( + ctx context.Context, deleteQuery string, key string, +) (int64, error) { + var sizeBytes int64 + + err := c.db.QueryRowContext(ctx, deleteQuery, key).Scan(&sizeBytes) + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + + if err != nil { + return 0, err + } + + return sizeBytes, nil } // notifyWritePressure wakes the background evictor after a store, so @@ -673,7 +734,9 @@ func (c *Cache) reconcileVariantFile( c.log.Warn("removing damaged cached variant file", "path", path, "file_bytes", info.Size(), "recorded_bytes", recordedSize) - return c.evictVariant(ctx, cacheKey) + _, err = c.removeVariant(ctx, cacheKey) + + return err } // adoptVariantFile inserts an accounting row for a variant file that @@ -835,7 +898,9 @@ func (c *Cache) reconcileSourceFile( c.log.Warn("removing damaged cached source file", "path", path, "file_bytes", info.Size(), "recorded_bytes", recordedSize) - return c.evictSourceBlob(ctx, key) + _, err = c.removeSourceBlob(ctx, key) + + return err } // removeUntrackedSourceFile deletes a source blob file that has no diff --git a/internal/imgcache/metrics.go b/internal/imgcache/metrics.go new file mode 100644 index 0000000..c42ff99 --- /dev/null +++ b/internal/imgcache/metrics.go @@ -0,0 +1,88 @@ +package imgcache + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" +) + +// registerMetrics makes the cache's metrics and registers them on +// metricsRegistry. The metrics are kept in memory, so updating one waits +// for nothing and makes no database call. It fails when metricsRegistry +// already holds metrics of the same names. +func (c *Cache) registerMetrics(metricsRegistry prometheus.Registerer) error { + c.cacheHits = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_cache_hits_total", + Help: "Image requests served from the cache.", + }) + c.cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_cache_misses_total", + Help: "Image requests not served from the cache.", + }) + c.sourceCacheHits = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_source_cache_hits_total", + Help: "Variants made or checked whose source was in the cache.", + }) + c.sourceCacheMisses = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_source_cache_misses_total", + Help: "Variants made or checked whose source was not in the cache.", + }) + c.usageBytes = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "pixa_cache_usage_bytes", + Help: "Bytes of sources and variants in the cache, as eviction last read them.", + }) + c.maxBytes = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "pixa_cache_max_bytes", + Help: "The disk cache limit in bytes, cache_max_bytes.", + }) + c.evictions = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_cache_evictions_total", + Help: "Sources and variants removed by eviction.", + }) + c.evictedBytes = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pixa_cache_evicted_bytes_total", + Help: "Bytes of the sources and variants removed by eviction.", + }) + + c.maxBytes.Set(float64(c.config.MaxBytes)) + + for _, metric := range []prometheus.Collector{ + c.cacheHits, c.cacheMisses, c.sourceCacheHits, c.sourceCacheMisses, + c.usageBytes, c.maxBytes, c.evictions, c.evictedBytes, + } { + err := metricsRegistry.Register(metric) + if err != nil { + return fmt.Errorf("failed to register the cache's metrics: %w", err) + } + } + + return nil +} + +// IncrementStats counts a cache hit or miss of an image request. It only +// adds to the cache's counters in memory, so it never waits for the +// database or fails. +func (c *Cache) IncrementStats(hit bool) { + if hit { + c.cacheHits.Inc() + } else { + c.cacheMisses.Inc() + } +} + +// IncrementSourceStats counts whether a variant being made or checked found +// its source in the cache. Like IncrementStats, it never waits or fails. +func (c *Cache) IncrementSourceStats(hit bool) { + if hit { + c.sourceCacheHits.Inc() + } else { + c.sourceCacheMisses.Inc() + } +} + +// countEviction counts one source or variant of sizeBytes bytes removed by +// eviction. +func (c *Cache) countEviction(sizeBytes int64) { + c.evictions.Inc() + c.evictedBytes.Add(float64(sizeBytes)) +} diff --git a/internal/providers/providers.go b/internal/providers/providers.go index 03824ce..b79502e 100644 --- a/internal/providers/providers.go +++ b/internal/providers/providers.go @@ -35,7 +35,7 @@ import ( // Constructors returns the constructors of the parts this package builds, // for fx.Provide. The fx graph must also provide the prometheus.Registerer -// the cache and the image service register their counters on. +// the cache and the image service register their metrics on. func Constructors() []any { return []any{ newKeys, From 7f87aa17ca217bce3a11b435980b0811261f1384 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 16:25:44 +0100 Subject: [PATCH 124/131] Set the expiry in TestHandleImageEnc_MaxAge inside each subtest (closes #352) The table set the expiry when it was built. The parallel subtests can wait more than 10 seconds behind other tests on a busy host, so the 60 second case saw a max-age under 50. The table now holds the TTL, and each subtest computes the expiry just before generating its token. Model: opus-5-5 --- .../image_cache_control_internal_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go index 97db3af..c056719 100644 --- a/internal/handlers/image_cache_control_internal_test.go +++ b/internal/handlers/image_cache_control_internal_test.go @@ -185,13 +185,13 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { tests := []struct { name string - expires time.Time + ttl time.Duration wantAtLeast int wantAtMost int }{ - {"60 second TTL", time.Now().Add(time.Minute), 50, 60}, - {"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour), 31536000, 31536000}, - {"no TTL", time.Time{}, 31536000, 31536000}, + {"60 second TTL", time.Minute, 50, 60}, + {"two-year TTL", 2 * 365 * 24 * time.Hour, 31536000, 31536000}, + {"no TTL", 0, 31536000, 31536000}, } for _, tt := range tests { @@ -200,6 +200,14 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) + // The expiry is set here, not in the table: a parallel subtest + // can wait behind other tests for more than the 10 seconds the + // 60 second case allows. + var expires time.Time + if tt.ttl != 0 { + expires = time.Now().Add(tt.ttl) + } + token, err := h.encGen.Generate(&imagerequest.Request{ Source: imagerequest.Source{ Host: signedHost, @@ -209,7 +217,7 @@ func TestHandleImageEnc_MaxAge(t *testing.T) { Size: imagerequest.Size{Width: 50, Height: 50}, Format: imagerequest.FormatJPEG, }, - }, tt.expires) + }, expires) if err != nil { t.Fatalf("Generate() error = %v", err) } From 092176d9178e2c257f7082163298077d349c152d Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 16:56:21 +0100 Subject: [PATCH 125/131] Delete a negative-cache test that tests nothing (closes #347) TestProxy_Get_ReturnsErrorForNegativeCachedURL never called Get; it only checked ErrNegativeCached against itself. The test above it covers Get returning ErrNegativeCached. Deletion approved by sneak on the issue. Model: opus-5-5 --- internal/imageproxy/negative_cache_internal_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/imageproxy/negative_cache_internal_test.go b/internal/imageproxy/negative_cache_internal_test.go index 96257cc..54ab4a2 100644 --- a/internal/imageproxy/negative_cache_internal_test.go +++ b/internal/imageproxy/negative_cache_internal_test.go @@ -82,15 +82,3 @@ func TestProxy_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) { err, ErrNegativeCached) } } - -func TestProxy_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) { - t.Parallel() - - // This test verifies that Proxy.Get() checks the negative cache - // We can't easily test the full pipeline without vips, but we can - // verify the error type - err := ErrNegativeCached - if !errors.Is(err, ErrNegativeCached) { - t.Error("ErrNegativeCached should be identifiable with errors.Is") - } -} From 0de6972c2db5421d54f9068fb1c478871087a894 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 21:55:58 +0100 Subject: [PATCH 126/131] Run make devserver from environment variables, not a config file (closes #360) The target mounted config.dev.yml, which nothing creates, so Docker made a directory in its place and pixad exited. It now passes a new random PIXA_SIGNING_KEY each run, printed so the developer can log in, and PIXA_DEBUG=true so the login form works over plain HTTP. The unused .gitignore entry for config.dev.yml is removed. Model: opus-5-5 --- .gitignore | 1 - Makefile | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 6c4d819..c8e02b7 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,3 @@ vendor/ # Local dev configs config.yml config.yaml -config.dev.yml diff --git a/Makefile b/Makefile index 22ac9c9..a074315 100644 --- a/Makefile +++ b/Makefile @@ -79,12 +79,15 @@ docker-versioned: docker-test: @script/test -# Run local dev server in Docker +# Run local dev server in Docker, with debug on and a new random signing +# key each run. Log in to the page with the key it prints. devserver: docker-versioned devserver-stop + @key="$$(head -c 32 /dev/urandom | base64)" && \ docker run -d --name pixad-dev -p 8080:8080 \ - -v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \ - pixa:latest - @echo "pixad running at http://localhost:8080" + -e PIXA_SIGNING_KEY="$$key" -e PIXA_DEBUG=true \ + pixa:latest >/dev/null && \ + echo "pixad running at http://localhost:8080" && \ + echo "signing key: $$key" # Stop dev server devserver-stop: From 851a374e53f38bdbddb5989c05c691ec23e757ff Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 22:26:52 +0100 Subject: [PATCH 127/131] Test that pixad writes only its own errors to stderr (closes #359) Both tests run pixad as a child process. They fail against this code: every start writes the dependency-injection library's event log to stderr, and a startup error is printed twice inside its frames. Model: opus-5-5 --- go.mod | 2 +- internal/app/app.go | 45 +++++++- internal/app/app_internal_test.go | 65 +++++++++++ internal/app/app_test.go | 177 ++++++++++++++++++++++++++++++ 4 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 internal/app/app_internal_test.go create mode 100644 internal/app/app_test.go diff --git a/go.mod b/go.mod index 618e2e9..2707998 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/prometheus/client_golang v1.23.2 github.com/slok/go-http-metrics v0.13.0 github.com/spf13/cobra v1.10.2 + go.uber.org/dig v1.19.0 go.uber.org/fx v1.24.0 golang.org/x/crypto v0.41.0 golang.org/x/sync v0.19.0 @@ -126,7 +127,6 @@ require ( go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect - go.uber.org/dig v1.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect diff --git a/internal/app/app.go b/internal/app/app.go index 0d9daf9..1481ef7 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -2,13 +2,16 @@ package app import ( + "context" "fmt" + "io" "os" "os/signal" "syscall" "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" + "go.uber.org/dig" "go.uber.org/fx" "sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/database" @@ -53,7 +56,10 @@ func run(_ *cobra.Command, _ []string) { // A write to a closed stdout or stderr must not end the process. signal.Ignore(syscall.SIGPIPE) - fx.New( + app := fx.New( + // pixad logs with slog only; runApp prints a startup or shutdown + // error itself. + fx.NopLogger, fx.Provide( config.New, database.New, @@ -71,5 +77,40 @@ func run(_ *cobra.Command, _ []string) { func(log *logger.Logger) { log.Identify() }, func(*server.Server) {}, ), - ).Run() + ) + + exitCode := runApp(app, os.Stderr) + if exitCode != 0 { + os.Exit(exitCode) + } +} + +// runApp starts app, waits for a signal or a shutdown request, stops app, +// and returns the status pixad exits with, as fx.App.Run does. It prints +// a start or stop error to stderr, once, as the error that caused it: +// dig.RootCause drops the frames fx and dig wrap around it. +func runApp(app *fx.App, stderr io.Writer) int { + startCtx, cancelStart := context.WithTimeout(context.Background(), app.StartTimeout()) + defer cancelStart() + + err := app.Start(startCtx) + if err != nil { + _, _ = fmt.Fprintln(stderr, dig.RootCause(err)) + + return 1 + } + + shutdown := <-app.Wait() + + stopCtx, cancelStop := context.WithTimeout(context.Background(), app.StopTimeout()) + defer cancelStop() + + err = app.Stop(stopCtx) + if err != nil { + _, _ = fmt.Fprintln(stderr, dig.RootCause(err)) + + return 1 + } + + return shutdown.ExitCode } diff --git a/internal/app/app_internal_test.go b/internal/app/app_internal_test.go new file mode 100644 index 0000000..d59b25d --- /dev/null +++ b/internal/app/app_internal_test.go @@ -0,0 +1,65 @@ +package app + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + + "go.uber.org/fx" +) + +var errTestStopFailed = errors.New("test stop hook failed") + +// TestRunAppPrintsStopErrorOnce checks that when a stop hook fails, runApp +// prints the hook's error once and returns exit status 1. +func TestRunAppPrintsStopErrorOnce(t *testing.T) { + t.Parallel() + + app := fx.New( + fx.NopLogger, + fx.Invoke(func(lifecycle fx.Lifecycle, shutdowner fx.Shutdowner) { + lifecycle.Append(fx.Hook{ + // Ask to shut down as soon as the app has started, so + // that runApp goes on to stop it. + OnStart: func(context.Context) error { return shutdowner.Shutdown() }, + OnStop: func(context.Context) error { return errTestStopFailed }, + }) + }), + ) + + var stderr bytes.Buffer + + exitCode := runApp(app, &stderr) + if exitCode != 1 { + t.Errorf("runApp() = %d, want 1", exitCode) + } + + want := errTestStopFailed.Error() + "\n" + if stderr.String() != want { + t.Errorf("stderr = %q, want %q", stderr.String(), want) + } +} + +// TestRunAppReturnsShutdownExitCode checks that runApp returns the exit +// code a shutdown request names. +func TestRunAppReturnsShutdownExitCode(t *testing.T) { + t.Parallel() + + app := fx.New( + fx.NopLogger, + fx.Invoke(func(lifecycle fx.Lifecycle, shutdowner fx.Shutdowner) { + lifecycle.Append(fx.Hook{ + OnStart: func(context.Context) error { + return shutdowner.Shutdown(fx.ExitCode(3)) + }, + }) + }), + ) + + exitCode := runApp(app, io.Discard) + if exitCode != 3 { + t.Errorf("runApp() = %d, want 3", exitCode) + } +} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 0000000..a6d94bf --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,177 @@ +package app_test + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "strconv" + "syscall" + "testing" + "time" + + "sneak.berlin/go/pixa/internal/app" +) + +// These tests run pixad in a child process: the test binary itself, +// started again with runPixadInChildEnvVar set, which makes TestMain run +// pixad instead of the tests. +const runPixadInChildEnvVar = "TEST_RUN_PIXAD_IN_CHILD" + +const validTestSigningKey = "test-signing-key-for-app-tests-0123456789" + +func TestMain(m *testing.M) { + if os.Getenv(runPixadInChildEnvVar) == "1" { + os.Args = []string{"pixad"} + + app.Run("test") + os.Exit(0) + } + + os.Exit(m.Run()) +} + +// pixadCommand returns a command that runs pixad with only the given +// environment, in an empty directory with an empty HOME, so that no +// config file is found. +func pixadCommand(t *testing.T, env ...string) (*exec.Cmd, *bytes.Buffer) { + t.Helper() + + testBinary := os.Args[0] + cmd := exec.CommandContext(t.Context(), testBinary) //nolint:gosec // not tainted + cmd.Dir = t.TempDir() + // A test binary built with -cover warns on stderr at exit when + // GOCOVERDIR is not set. + cmd.Env = append([]string{ + runPixadInChildEnvVar + "=1", + "HOME=" + t.TempDir(), + "GOCOVERDIR=" + t.TempDir(), + }, env...) + + var stderr bytes.Buffer + + cmd.Stderr = &stderr + + return cmd, &stderr +} + +// TestRunPrintsStartupErrorOnce checks that an invalid setting is +// printed once, as the line that names it, and nothing else goes to +// stderr. +func TestRunPrintsStartupErrorOnce(t *testing.T) { + t.Parallel() + + cmd, stderr := pixadCommand(t, "PIXA_SIGNING_KEY=short") + + err := cmd.Run() + + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("pixad: got %v, want exit status 1; stderr:\n%s", err, stderr) + } + + want := `config key "signing_key" (environment variable PIXA_SIGNING_KEY): ` + + "value too short: must be at least 32 characters, got 5\n" + if stderr.String() != want { + t.Errorf("stderr:\n%s\nwant:\n%s", stderr, want) + } +} + +// TestRunWritesNothingToStderrOnStartAndStop starts pixad, waits until +// it answers its health check, stops it with SIGTERM, and checks that it +// wrote nothing to stderr. +func TestRunWritesNothingToStderrOnStartAndStop(t *testing.T) { + t.Parallel() + + port := freePort(t) + cmd, stderr := pixadCommand(t, + "PORT="+strconv.Itoa(port), + "PIXA_STATE_DIR="+t.TempDir(), + "PIXA_SIGNING_KEY="+validTestSigningKey, + ) + + err := cmd.Start() + if err != nil { + t.Fatalf("starting pixad: %v", err) + } + + waitForHealthCheck(t, cmd, port, stderr) + + err = cmd.Process.Signal(syscall.SIGTERM) + if err != nil { + t.Fatalf("signalling pixad: %v", err) + } + + err = cmd.Wait() + if err != nil { + t.Fatalf("pixad: got %v, want exit status 0; stderr:\n%s", err, stderr) + } + + if stderr.Len() != 0 { + t.Errorf("stderr is not empty:\n%s", stderr) + } +} + +// freePort returns a TCP port on the loopback address that nothing +// listens on. +func freePort(t *testing.T) int { + t.Helper() + + var listenConfig net.ListenConfig + + listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("finding a free port: %v", err) + } + + addr, ok := listener.Addr().(*net.TCPAddr) + if !ok { + t.Fatalf("listener address %v is not a TCP address", listener.Addr()) + } + + err = listener.Close() + if err != nil { + t.Fatalf("closing listener: %v", err) + } + + return addr.Port +} + +// waitForHealthCheck waits up to 30 seconds for pixad, started as cmd, to +// answer its health check on port. If it does not, it stops pixad and +// fails the test with what pixad wrote to stderr. +func waitForHealthCheck(t *testing.T, cmd *exec.Cmd, port int, stderr *bytes.Buffer) { + t.Helper() + + url := fmt.Sprintf("http://127.0.0.1:%d/.well-known/healthcheck.json", port) + deadline := time.Now().Add(30 * time.Second) + + for time.Now().Before(deadline) { + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, nil) + if err != nil { + t.Fatalf("health check request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err == nil { + _ = resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return + } + } + + time.Sleep(100 * time.Millisecond) + } + + // Stop pixad first: stderr is complete, and safe to read, only once + // cmd.Wait has returned. + _ = cmd.Process.Kill() + _ = cmd.Wait() + + t.Fatalf("pixad did not answer its health check on port %d; stderr:\n%s", + port, stderr) +} From 9bccaaf85a75e8981b04d205b90c2755d418527a Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 22:40:59 +0100 Subject: [PATCH 128/131] README Getting Started ends with a signed image request (closes #357) Getting Started now runs script/bootstrap --cgo before make build, says that logging in at http://localhost:8080/ needs debug, and ends with a signed /v1/image/ request for an image on any public HTTPS server. Its curl exits non-zero when pixa refuses the request. Model: opus-5-5 --- README.md | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9522af0..91c1277 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,10 @@ In this document, the upstream host is the server that holds an original image. ## Getting Started ```bash -# clone and build +# clone, install what compiling needs, and build git clone https://git.eeqj.de/sneak/pixa.git cd pixa +script/bootstrap --cgo make build # make a config file from the example and put a real signing key in it @@ -26,18 +27,30 @@ openssl rand -base64 32 # prints a new signing key $EDITOR config.yml # replace the signing_key placeholder with it ./bin/pixad --config config.yml -# or build and run the Docker image +# or build and run the Docker image with that signing key make docker -docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest +docker run -p 8080:8080 -e PIXA_SIGNING_KEY='the new signing key' pixa:latest ``` -`make build` compiles pixa on your machine into `bin/pixad`. `pixad` is the -server program. Compiling needs the libraries `script/bootstrap --cgo` installs -(see Deployment). +`script/bootstrap --cgo` installs Go, a C compiler and the image libraries pixa +needs (see Deployment). `make build` then compiles pixa on your machine into +`bin/pixad`. `pixad` is the server program. The signing key is pixa's one required secret. It must be 32 characters or longer. The placeholder in the example file is refused at startup. +A container takes its settings from environment variables (see Configuration). +Only `PIXA_SIGNING_KEY` is required. Without it, the container exits at startup +and names the variable. Every other setting has a built-in default. A config +file mounted at `/etc/pixa/config.yml` is read when present. An environment +variable wins over the same setting in the file. + +pixa's login page is at `http://localhost:8080/`. Logging in there works only +with `debug: true` in `config.yml`, or `-e PIXA_DEBUG=true` for the container. +Otherwise pixa refuses the login with 403, because the page was opened over +plain HTTP. Use `debug` only on your own machine. A pixa that others use serves +its pages over HTTPS through a reverse proxy (see Deployment). + Once pixa runs, it serves images through URLs of this form: ``` @@ -60,11 +73,25 @@ The example config allowlists no host. Every image on an allowlisted host is served to anyone, without a signature, at any size and in any format. So allowlist only hosts you control, where nobody else can put images. -A container takes its settings from environment variables (see Configuration). -Only `PIXA_SIGNING_KEY` is required. Without it, the container exits at startup -and names the variable. Every other setting has a built-in default. A config -file mounted at `/etc/pixa/config.yml` is read when present. An environment -variable wins over the same setting in the file. +To see pixa serve an image, sign a URL for an image on any public HTTPS server. +Run this in a second terminal while pixa runs, with your own signing key and +image: + +```bash +key='the new signing key' +image_host=cdn.example.com # a public HTTPS server +image_path=/photos/cat.jpg # an image on it +exp=$(($(date +%s) + 3600)) # the URL expires in one hour +sig=$(printf '%s' "$image_host:$image_path::400:300:webp:$exp:85:cover" | + openssl dgst -sha256 -hmac "$key" -binary | base64 | tr '+/' '-_') +curl --fail-with-body -o cat.webp \ + "http://localhost:8080/v1/image/$image_host$image_path/400x300.webp?sig=$sig&exp=$exp" +``` + +`cat.webp` is then the image at 400x300. If pixa refuses the request, curl +reports an error and `cat.webp` holds pixa's error message instead. The image +path must not hold a `%`, a `:` or a query. Signed URLs explains how to sign +those. ## Deployment From b32077c380bedd2272a0a9c28cd1cb21e3936db4 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 11 Nov 2026 22:55:39 +0100 Subject: [PATCH 129/131] Name the cause on the page for a refused form (closes #358) The CSRF middleware answered a refused login or generator form with the library's bare text, such as "Forbidden - origin invalid". It now answers 403 with a page that names the cause and what to do, and logs a warning with the request ID and the library's reason. A page opened over plain HTTP with debug off and a form from another host both fail the origin check; the page tells them apart by the host in the Origin or Referer header. Model: opus-5-5 --- README.md | 4 + internal/handlers/auth_csrf_internal_test.go | 2 +- internal/handlers/bodylimit_internal_test.go | 2 +- internal/handlers/csrf.go | 108 ++++++++- .../handlers/csrf_refused_internal_test.go | 205 ++++++++++++++++++ internal/handlers/handlers.go | 6 +- internal/templates/form_refused.html | 22 ++ 7 files changed, 343 insertions(+), 6 deletions(-) create mode 100644 internal/handlers/csrf_refused_internal_test.go create mode 100644 internal/templates/form_refused.html diff --git a/README.md b/README.md index 91c1277..786deb9 100644 --- a/README.md +++ b/README.md @@ -403,6 +403,10 @@ instead. Plain HTTP is for development on the browser's own machine. The login session cookie is always marked `Secure`. Over plain HTTP a browser keeps such a cookie only for its own machine (`localhost`), if at all. +A form refused with 403 gets a page that names the cause and what to do about +it. pixa logs a `form refused` warning with the request ID and the form-token +library's reason, such as `origin invalid`. + The image routes answer the errors listed for them with a JSON body holding `error`, `status` and `timestamp`. diff --git a/internal/handlers/auth_csrf_internal_test.go b/internal/handlers/auth_csrf_internal_test.go index 6237798..65b6d82 100644 --- a/internal/handlers/auth_csrf_internal_test.go +++ b/internal/handlers/auth_csrf_internal_test.go @@ -55,7 +55,7 @@ func newCSRFTestRouter(t *testing.T) (*Handlers, http.Handler) { config: cfg, sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), encGen: encurl.NewGenerator(keys.EncryptedURLKey), - csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug), + csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug, slog.New(slog.DiscardHandler)), } r := chi.NewRouter() diff --git a/internal/handlers/bodylimit_internal_test.go b/internal/handlers/bodylimit_internal_test.go index 496eccd..895a866 100644 --- a/internal/handlers/bodylimit_internal_test.go +++ b/internal/handlers/bodylimit_internal_test.go @@ -43,7 +43,7 @@ func newBodyLimitTestRouter( config: cfg, sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), encGen: encurl.NewGenerator(keys.EncryptedURLKey), - csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug), + csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug, slog.New(slog.DiscardHandler)), } r := chi.NewRouter() diff --git a/internal/handlers/csrf.go b/internal/handlers/csrf.go index 03e4de7..1139361 100644 --- a/internal/handlers/csrf.go +++ b/internal/handlers/csrf.go @@ -1,18 +1,43 @@ package handlers import ( + "bytes" + "errors" "html/template" + "log/slog" "net/http" + "net/url" + "github.com/go-chi/chi/v5/middleware" "github.com/gorilla/csrf" "sneak.berlin/go/pixa/internal/seal" + "sneak.berlin/go/pixa/internal/templates" +) + +// The causes the form-refused page names, each with what to do about it. +const ( + causePlainHTTP = "This page was opened over plain HTTP while debug is " + + "off. Open it over HTTPS, or turn debug on to use plain HTTP on " + + "your own machine." + causeHTTPSWithDebug = "This page was opened over HTTPS while debug is " + + "on. Turn debug off, or open the page over plain HTTP." + causeOtherHost = "The form was sent from another host than the one in " + + "the request's Host header. A reverse proxy in front of pixa must " + + "pass the Host header on unchanged." + causeNoReferer = "The browser sent neither an Origin nor a Referer " + + "header, and over HTTPS pixa needs one to check where the form " + + "came from. Let the browser send the Referer header to this site." + causeStaleToken = "The form token is missing or out of date. Reload " + + "the page and submit the form again." ) // newCSRFProtect builds the CSRF-protection middleware for the // state-mutating HTML form routes. The token lives in its own cookie, // independent of the session cookie, so it also protects the login POST -// where no session exists yet (login CSRF). key signs that cookie. +// where no session exists yet (login CSRF). key signs that cookie. A +// refused form is answered by the handler that handleFormRefused returns. +// log gets a warning with the library's reason, such as "origin invalid". // // When plaintext is true (local HTTP development), requests are marked // plaintext so the library neither demands an https Referer nor sets a @@ -20,13 +45,14 @@ import ( // service runs behind a TLS-terminating proxy, so plaintext is false and // the library enforces its https Referer origin check. func newCSRFProtect( - key [seal.KeySize]byte, plaintext bool, + key [seal.KeySize]byte, plaintext bool, log *slog.Logger, ) func(http.Handler) http.Handler { protect := csrf.Protect( key[:], csrf.Path("/"), csrf.Secure(!plaintext), csrf.SameSite(csrf.SameSiteStrictMode), + csrf.ErrorHandler(handleFormRefused(plaintext, log)), ) if !plaintext { @@ -42,6 +68,84 @@ func newCSRFProtect( } } +// handleFormRefused returns the handler the CSRF middleware calls when it +// refuses a request. It logs the library's reason with the request ID and +// answers 403 with a page naming the cause and what to do. A reason that is +// not a refusal, such as a failure to make a token, answers 500. +func handleFormRefused(plaintext bool, log *slog.Logger) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reason := csrf.FailureReason(r) + requestID := middleware.GetReqID(r.Context()) + + cause := formRefusedCause(r, reason, plaintext) + if cause == "" { + log.Error("CSRF middleware failed", + "request_id", requestID, "error", reason) + http.Error(w, "Internal server error", + http.StatusInternalServerError) + + return + } + + log.Warn("form refused", "request_id", requestID, "reason", reason) + + var page bytes.Buffer + + err := templates.Render(&page, "form_refused.html", + struct{ Cause string }{Cause: cause}) + if err != nil { + log.Error("failed to render form refused template", "error", err) + http.Error(w, "Internal server error", + http.StatusInternalServerError) + + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusForbidden) + + _, err = page.WriteTo(w) + if err != nil { + log.Error("failed to write form refused page", "error", err) + } + }) +} + +// formRefusedCause returns the cause the form-refused page names for the +// CSRF middleware's reason, or "" when the reason is not a refusal. +func formRefusedCause(r *http.Request, reason error, plaintext bool) string { + switch { + case errors.Is(reason, csrf.ErrNoToken), errors.Is(reason, csrf.ErrBadToken): + return causeStaleToken + case errors.Is(reason, csrf.ErrNoReferer): + return causeNoReferer + case errors.Is(reason, csrf.ErrBadOrigin): + return pageAddressCause(r.Header.Get("Origin"), r.Host, plaintext) + case errors.Is(reason, csrf.ErrBadReferer): + return pageAddressCause(r.Referer(), r.Host, plaintext) + default: + return "" + } +} + +// pageAddressCause returns the cause for a form whose page address, the +// Origin or Referer header, the CSRF middleware refused. The middleware +// expects the request's own host, with https when plaintext is false and +// http when it is true. So on the request's own host, only the scheme was +// wrong. +func pageAddressCause(pageAddress, host string, plaintext bool) string { + page, err := url.Parse(pageAddress) + if err != nil || page.Host != host { + return causeOtherHost + } + + if plaintext { + return causeHTTPSWithDebug + } + + return causePlainHTTP +} + // CSRF returns the CSRF-protection middleware for the login and generator // form routes. func (s *Handlers) CSRF() func(http.Handler) http.Handler { diff --git a/internal/handlers/csrf_refused_internal_test.go b/internal/handlers/csrf_refused_internal_test.go new file mode 100644 index 0000000..98777c7 --- /dev/null +++ b/internal/handlers/csrf_refused_internal_test.go @@ -0,0 +1,205 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "html/template" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/go-chi/chi/v5/middleware" + "github.com/gorilla/csrf" + + "sneak.berlin/go/pixa/internal/seal" +) + +// refusedFormRequestID is the request ID of the requests postRefusedForm +// sends, which the warning for a refused form must carry. +const refusedFormRequestID = "test-request-id" + +// TestFormRefusedNamesCause verifies that a form the CSRF middleware refuses +// answers 403 with a page naming the cause, and logs a warning with the +// request ID and the middleware's reason. +func TestFormRefusedNamesCause(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + debug bool + origin string + referer string + wantCause string + wantReason error + }{ + { + name: "plain HTTP origin with debug off", + origin: "http://example.com", + wantCause: causePlainHTTP, + wantReason: csrf.ErrBadOrigin, + }, + { + name: "plain HTTP referer with debug off", + referer: "http://example.com/", + wantCause: causePlainHTTP, + wantReason: csrf.ErrBadReferer, + }, + { + name: "origin on another host", + origin: "https://proxy.example.net", + wantCause: causeOtherHost, + wantReason: csrf.ErrBadOrigin, + }, + { + name: "referer on another host", + referer: "https://proxy.example.net/", + wantCause: causeOtherHost, + wantReason: csrf.ErrBadReferer, + }, + { + name: "HTTPS origin with debug on", + debug: true, + origin: "https://example.com", + wantCause: causeHTTPSWithDebug, + wantReason: csrf.ErrBadOrigin, + }, + { + name: "no origin or referer with debug off", + wantCause: causeNoReferer, + wantReason: csrf.ErrNoReferer, + }, + { + name: "missing token", + debug: true, + origin: "http://example.com", + wantCause: causeStaleToken, + wantReason: csrf.ErrNoToken, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec, logLine := postRefusedForm(t, tt.debug, tt.origin, tt.referer, + nil, "") + checkFormRefused(t, rec, logLine, tt.wantCause, tt.wantReason) + }) + } +} + +// TestFormRefusedNamesCauseForForeignToken verifies that a form whose token +// does not match its CSRF cookie answers 403 with a page naming +// causeStaleToken, and logs a warning with the request ID and the +// middleware's reason. +func TestFormRefusedNamesCauseForForeignToken(t *testing.T) { + t.Parallel() + + // The router and postRefusedForm derive their CSRF key from + // testSigningKey, so cookiesA is valid there and only tokenB is wrong. + _, router := newCSRFTestRouter(t) + cookiesA, _ := csrfCredentials(t, router, nil) + _, tokenB := csrfCredentials(t, router, nil) + + rec, logLine := postRefusedForm(t, true, "http://example.com", "", + cookiesA, tokenB) + checkFormRefused(t, rec, logLine, causeStaleToken, csrf.ErrBadToken) +} + +// postRefusedForm sends a POST with the given cookies to the CSRF +// middleware built for debug. It adds the form token and the Origin and +// Referer headers when they are not empty. It returns the response and the +// one JSON log line the middleware wrote. +func postRefusedForm( + t *testing.T, debug bool, origin, referer string, + cookies []*http.Cookie, token string, +) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + + keys, err := seal.DeriveKeys(testSigningKey) + if err != nil { + t.Fatalf("seal.DeriveKeys() error = %v", err) + } + + var logs bytes.Buffer + + log := slog.New(slog.NewJSONHandler(&logs, nil)) + reached := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + t.Error("the refused form reached the handler behind the middleware") + }) + srv := newCSRFProtect(keys.CSRFKey, debug, log)(reached) + + form := url.Values{loginKeyField: {"x"}} + if token != "" { + form.Set(csrfTokenField, token) + } + + ctx := context.WithValue(context.Background(), + middleware.RequestIDKey, refusedFormRequestID) + req := httptest.NewRequestWithContext(ctx, http.MethodPost, + "http://example.com/", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + for _, c := range cookies { + req.AddCookie(c) + } + + if origin != "" { + req.Header.Set("Origin", origin) + } + + if referer != "" { + req.Header.Set("Referer", referer) + } + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + var logLine map[string]any + + err = json.Unmarshal(logs.Bytes(), &logLine) + if err != nil { + t.Fatalf("decoding log line %q: %v", logs.String(), err) + } + + return rec, logLine +} + +// checkFormRefused checks that rec is a 403 page naming wantCause and that +// logLine is a warning carrying the request ID and wantReason. +func checkFormRefused( + t *testing.T, rec *httptest.ResponseRecorder, logLine map[string]any, + wantCause string, wantReason error, +) { + t.Helper() + + if rec.Code != http.StatusForbidden { + t.Errorf("status = %d, want %d", rec.Code, http.StatusForbidden) + } + + body := rec.Body.String() + t.Logf("body: %s", body) + + if !strings.Contains(body, template.HTMLEscapeString(wantCause)) { + t.Errorf("page does not name the cause %q", wantCause) + } + + t.Logf("log line: %v", logLine) + + if logLine["level"] != "WARN" || logLine["msg"] != "form refused" { + t.Errorf("log line = %v, want a WARN \"form refused\"", logLine) + } + + if logLine["request_id"] != refusedFormRequestID { + t.Errorf("logged request_id = %v, want %q", + logLine["request_id"], refusedFormRequestID) + } + + if logLine["reason"] != wantReason.Error() { + t.Errorf("logged reason = %v, want %q", logLine["reason"], wantReason) + } +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 0f4e134..6b68345 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -56,14 +56,16 @@ type Handlers struct { // New creates a new Handlers instance. func New(_ fx.Lifecycle, params Params) *Handlers { + log := params.Logger.Get() + return &Handlers{ - log: params.Logger.Get(), + log: log, hc: params.Healthcheck, config: params.Config, imgSvc: params.ImageService, sessMgr: params.SessionManager, encGen: params.EncryptedURLGenerator, - csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug), + csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug, log), signer: signature.New(params.Config.SigningKey), allowlist: allowlist.New(params.Config.AllowlistHosts), refererBlocklist: allowlist.New(params.Config.RefererBlocklist), diff --git a/internal/templates/form_refused.html b/internal/templates/form_refused.html new file mode 100644 index 0000000..76a86dc --- /dev/null +++ b/internal/templates/form_refused.html @@ -0,0 +1,22 @@ + + + + + + Pixa - Form Refused + + + +
+

Form Refused

+ +
+ {{.Cause}} +
+ +

+ Back to the start page +

+
+ + From 20874ba2af8c8d7e207db9850987fff90eb91f30 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 12 Nov 2026 00:25:45 +0100 Subject: [PATCH 130/131] Answer 502 for an upstream host with no address or a failed fetch (closes #367) The image routes answered 500 and logged at error level when the upstream host had no address, when the fetch failed before the upstream answered, or when the fetcher refused a redirect. Such a failure is now ErrInvalidHost or ErrUpstreamRequestFailed, which respondImageError answers 502 "upstream error" and logs at warn level. A TLS handshake, an address lookup or a connection that times out is now ErrUpstreamTimeout and answers 504; its log line keeps the cause. A cancelled fetch or address lookup is neither ErrUpstreamRequestFailed nor ErrInvalidHost. README Routes lists these answers. Model: opus-5-5 --- README.md | 11 + internal/handlers/image.go | 4 +- .../handlers/image_errors_internal_test.go | 8 + internal/httpfetcher/httpfetcher.go | 76 ++++++- .../request_failed_internal_test.go | 213 ++++++++++++++++++ 5 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 internal/httpfetcher/request_failed_internal_test.go diff --git a/README.md b/README.md index 786deb9..7839810 100644 --- a/README.md +++ b/README.md @@ -337,8 +337,19 @@ to any method the list does not give for it. neither the image nor its source in its cache. - 502 likewise for 5 minutes after a body whose first bytes are not those of its `Content-Type`. + - 502 when the upstream host, or a host it redirects to, has no address. + - 502 when the upstream redirects to a URL that is not `https`, or redirects + 10 times. + - 502 when the fetch fails before the upstream answers, such as on a refused + connection or a failed TLS handshake. - 503 when pixa is busy (see Limits). - 503 in maintenance mode. + - 504 when the address lookup of the upstream host, or of a host it + redirects to, times out. + - 504 when the connection to the upstream host, or to a host it redirects + to, times out. + - 504 when the upstream has not finished the TLS handshake within 10 + seconds. - 504 when the upstream has not sent its response headers within `upstream_fetch_timeout`. - 500 when that time runs out while the image itself is still arriving. diff --git a/internal/handlers/image.go b/internal/handlers/image.go index ff11b6e..108e3e6 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -254,7 +254,9 @@ func (s *Handlers) respondImageError( switch { case errors.Is(err, httpfetcher.ErrSSRFBlocked): status, message = http.StatusForbidden, "forbidden" - case errors.Is(err, httpfetcher.ErrUpstreamError): + case errors.Is(err, httpfetcher.ErrUpstreamError), + errors.Is(err, httpfetcher.ErrInvalidHost), + errors.Is(err, httpfetcher.ErrUpstreamRequestFailed): status, message = http.StatusBadGateway, "upstream error" case errors.Is(err, httpfetcher.ErrUpstreamTimeout): status, message = http.StatusGatewayTimeout, "upstream timeout" diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index c8c2a33..360a6f1 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "slices" "strings" + "syscall" "testing" "testing/fstest" "time" @@ -461,6 +462,13 @@ func TestImageRoutes_SameErrorAnswers(t *testing.T) { {"upstream error status", fmt.Errorf("%w: status 404", httpfetcher.ErrUpstreamError), http.StatusBadGateway, slog.LevelWarn}, + {"upstream host with no address", + fmt.Errorf("%w: cdn.example.com", httpfetcher.ErrInvalidHost), + http.StatusBadGateway, slog.LevelWarn}, + {"connection refused", + fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamRequestFailed, + syscall.ECONNREFUSED), + http.StatusBadGateway, slog.LevelWarn}, {"no response headers within the fetch timeout", httpfetcher.ErrUpstreamTimeout, http.StatusGatewayTimeout, slog.LevelWarn}, diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 7f8a8f7..4a01128 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -69,7 +69,11 @@ var ( ErrInvalidContentType = errors.New("invalid or unsupported content type") ErrUpstreamError = errors.New("upstream server error") ErrUpstreamTimeout = errors.New("upstream request timeout") - ErrTooManyConnections = errors.New("too many concurrent upstream connections") + // ErrUpstreamRequestFailed is a fetch that failed before the upstream + // answered, other than by timeout or cancellation: a failed connection + // or TLS handshake, or a redirect the fetcher refuses. + ErrUpstreamRequestFailed = errors.New("upstream request failed") + ErrTooManyConnections = errors.New("too many concurrent upstream connections") ) // Internal fetcher errors. @@ -283,11 +287,7 @@ func (f *HTTPFetcher) Fetch( fetchDuration := time.Since(startTime) if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return nil, ErrUpstreamTimeout - } - - return nil, fmt.Errorf("upstream request failed: %w", err) + return nil, upstreamRequestError(err) } // The deferred release gives the connection back, as no body follows @@ -316,6 +316,33 @@ func (f *HTTPFetcher) Fetch( return result, nil } +// upstreamRequestError returns the error Fetch reports when its upstream +// request fails with err: +// - err wrapped in ErrUpstreamTimeout for a timeout, so the log names what +// timed out. +// - err, still matching context.Canceled, when the caller cancelled the +// fetch. +// - err wrapped in ErrUpstreamRequestFailed otherwise. +func upstreamRequestError(err error) error { + var netErr net.Error + + switch { + case errors.Is(err, context.DeadlineExceeded), + errors.Is(err, ErrUpstreamTimeout), + errors.As(err, &netErr) && netErr.Timeout(): + // A timeout other than the fetch timeout, such as the TLS + // handshake timeout, does not match context.DeadlineExceeded. An + // address lookup or a connection that timed out is + // ErrUpstreamTimeout already, so its log line names that error twice. + return fmt.Errorf("%w: %w", ErrUpstreamTimeout, err) + case errors.Is(err, context.Canceled): + // The caller cancelled the fetch; the upstream did not fail + return fmt.Errorf("upstream request cancelled: %w", err) + default: + return fmt.Errorf("%w: %w", ErrUpstreamRequestFailed, err) + } +} + // requestHeaders returns the headers of a fetch: User-Agent, Accept, the // ID of the request the fetch serves, so the fetch can be found in the // upstream host's logs, and a non-empty ifNoneMatch or ifModifiedSince. @@ -505,7 +532,7 @@ func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) erro // Resolve the host to check IP addresses addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { - return fmt.Errorf("%w: %s", ErrInvalidHost, host) + return addressLookupError(host, err) } private := slices.ContainsFunc(addrs, func(addr net.IPAddr) bool { @@ -518,6 +545,23 @@ func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) erro return nil } +// addressLookupError returns the error for a failed address lookup of host: +// - ErrUpstreamTimeout when the lookup timed out. +// - err, still matching context.Canceled, when the lookup was cancelled. +// - ErrInvalidHost otherwise. +func addressLookupError(host string, err error) error { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return fmt.Errorf("%w: address lookup of %s", ErrUpstreamTimeout, host) + } + + if errors.Is(err, context.Canceled) { + return fmt.Errorf("address lookup cancelled: %w", err) + } + + return fmt.Errorf("%w: %s", ErrInvalidHost, host) +} + // isLocalhost checks if the host is localhost. func isLocalhost(host string) bool { host = strings.ToLower(host) @@ -646,7 +690,7 @@ func dialSSRFSafe( // Resolve the address ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host) if err != nil { - return nil, fmt.Errorf("%w: %s", ErrInvalidHost, host) + return nil, addressLookupError(host, err) } // Check all resolved IPs @@ -657,7 +701,10 @@ func dialSSRFSafe( } // Connect using the first valid IP - var dialer net.Dialer + var ( + dialer net.Dialer + lastDialErr error + ) for _, ip := range ips { addr := net.JoinHostPort(ip.String(), port) @@ -666,9 +713,18 @@ func dialSSRFSafe( if err == nil { return conn, nil } + + lastDialErr = err } - return nil, fmt.Errorf("%w to %s", errConnectFailed, host) + // Only the last address tried decides whether the connection timed out + var netErr net.Error + if errors.As(lastDialErr, &netErr) && netErr.Timeout() { + return nil, fmt.Errorf("%w: connect to %s: %w", + ErrUpstreamTimeout, host, lastDialErr) + } + + return nil, fmt.Errorf("%w to %s: %w", errConnectFailed, host, lastDialErr) } // limitedReader wraps a reader and refuses a body longer than remaining diff --git a/internal/httpfetcher/request_failed_internal_test.go b/internal/httpfetcher/request_failed_internal_test.go new file mode 100644 index 0000000..b71dac6 --- /dev/null +++ b/internal/httpfetcher/request_failed_internal_test.go @@ -0,0 +1,213 @@ +package httpfetcher + +import ( + "context" + "errors" + "net" + "os" + "syscall" + "testing" + "time" +) + +// TestFetchFailedBeforeAnswer checks the error of a fetch whose connection +// fails: +// - refused: ErrUpstreamRequestFailed. +// - timed out: ErrUpstreamTimeout, still matching the dial's error, even for +// a timeout error that does not match context.DeadlineExceeded, such as +// the TLS handshake timeout. +// - cancelled by its caller: context.Canceled, not ErrUpstreamRequestFailed. +func TestFetchFailedBeforeAnswer(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + dial func(cancel context.CancelFunc) error + wantErrs []error + wantNotErr error + }{ + { + name: "connection refused", + dial: func(context.CancelFunc) error { + return syscall.ECONNREFUSED + }, + wantErrs: []error{ErrUpstreamRequestFailed}, + wantNotErr: context.Canceled, + }, + { + name: "timed out while connecting", + dial: func(context.CancelFunc) error { + return os.ErrDeadlineExceeded + }, + wantErrs: []error{ErrUpstreamTimeout, os.ErrDeadlineExceeded}, + wantNotErr: ErrUpstreamRequestFailed, + }, + { + name: "cancelled while connecting", + dial: func(cancel context.CancelFunc) error { + cancel() + + return context.Canceled + }, + wantErrs: []error{context.Canceled}, + wantNotErr: ErrUpstreamRequestFailed, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(testContext(t)) + defer cancel() + + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + AllowHTTP: true, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + DialContext: func(context.Context, string, string) (net.Conn, error) { + return nil, tt.dial(cancel) + }, + }) + + _, err := f.Fetch(ctx, upstreamSource("/image"), "", "") + t.Logf("Fetch() error: %v", err) + + for _, wantErr := range tt.wantErrs { + if !errors.Is(err, wantErr) { + t.Errorf("Fetch() error = %v, want %v", err, wantErr) + } + } + + if errors.Is(err, tt.wantNotErr) { + t.Errorf("Fetch() error = %v, must not be %v", err, tt.wantNotErr) + } + }) + } +} + +// TestFetchAddressLookupTimedOut checks that a fetch whose address lookup +// times out fails with ErrUpstreamTimeout, both for the lookup of the source +// host and for the lookup while connecting. The error must match neither +// ErrInvalidHost nor ErrUpstreamRequestFailed, which the image routes answer +// 502. A context whose deadline has passed makes the lookup time out. +func TestFetchAddressLookupTimedOut(t *testing.T) { + t.Parallel() + + checkTimedOut := func(t *testing.T, err error) { + t.Helper() + t.Logf("Fetch() error: %v", err) + + if !errors.Is(err, ErrUpstreamTimeout) { + t.Errorf("Fetch() error = %v, want ErrUpstreamTimeout", err) + } + + if errors.Is(err, ErrInvalidHost) || errors.Is(err, ErrUpstreamRequestFailed) { + t.Errorf("Fetch() error = %v, must match neither ErrInvalidHost "+ + "nor ErrUpstreamRequestFailed", err) + } + } + + t.Run("lookup of the source host", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(testContext(t), 0) + defer cancel() + + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + }) + + _, err := f.Fetch(ctx, imageSourceOn("cdn.example.com"), "", "") + checkTimedOut(t, err) + }) + + t.Run("lookup while connecting", func(t *testing.T) { + t.Parallel() + + // The source host is an address, which validateURL does not look up; + // the dialer looks up a name + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + ctx, cancel := context.WithTimeout(ctx, 0) + defer cancel() + + return dialSSRFSafe(ctx, network, "img.example.com:443", nil) + }, + }) + + _, err := f.Fetch(testContext(t), upstreamSource("/image"), "", "") + checkTimedOut(t, err) + }) +} + +// TestFetchConnectionTimedOut checks that a fetch whose connection in +// dialSSRFSafe times out fails with ErrUpstreamTimeout, which the image +// routes answer 504. The error must match neither ErrUpstreamRequestFailed +// nor ErrInvalidHost, which they answer 502. A context whose deadline has +// passed makes the connection time out without touching the network. +func TestFetchConnectionTimedOut(t *testing.T) { + t.Parallel() + + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + ctx, cancel := context.WithTimeout(ctx, 0) + defer cancel() + + return dialSSRFSafe(ctx, network, addr, nil) + }, + }) + + _, err := f.Fetch(testContext(t), upstreamSource("/image"), "", "") + t.Logf("Fetch() error: %v", err) + + if !errors.Is(err, ErrUpstreamTimeout) { + t.Errorf("Fetch() error = %v, want ErrUpstreamTimeout", err) + } + + if errors.Is(err, ErrUpstreamRequestFailed) || errors.Is(err, ErrInvalidHost) { + t.Errorf("Fetch() error = %v, must match neither "+ + "ErrUpstreamRequestFailed nor ErrInvalidHost", err) + } +} + +// TestFetchAddressLookupCancelled checks that a fetch whose address lookup +// its caller cancelled fails with an error matching context.Canceled, not +// ErrInvalidHost, which the image routes answer 502. +func TestFetchAddressLookupCancelled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(testContext(t)) + cancel() + + f := New(&Config{ + Timeout: 30 * time.Second, + MaxResponseSize: 50 << 20, + MaxConnectionsPerHost: 20, + MaxConnections: 64, + }) + + _, err := f.Fetch(ctx, imageSourceOn("cdn.example.com"), "", "") + t.Logf("Fetch() error: %v", err) + + if !errors.Is(err, context.Canceled) { + t.Errorf("Fetch() error = %v, want context.Canceled", err) + } + + if errors.Is(err, ErrInvalidHost) { + t.Errorf("Fetch() error = %v, must not be ErrInvalidHost", err) + } +} From c2e32d987b18e569148321e69fff664e8aaedcc8 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 12 Nov 2026 04:41:25 +0100 Subject: [PATCH 131/131] Support SVG sources (closes #68) The fetcher accepts image/svg+xml; internal/magic detects SVG. orig of an SVG at the orig size is the SVG unchanged, cached, with a Content-Security-Policy allowing only its styles and data: images; at another size, ErrSVGNotOriginalSize (400). Any other format renders it at the least size covering the output, at most 8192 on a side. Rendering refuses an SVG with a side over 8192 times the other (ErrSVGTooThin, 415), and the orig size of one above 8192 on a side (ErrSVGTooLargeForOrigSize, 400). Child processes of pixad read an SVG's size and render it, killed after 20 seconds and limited to 1 GiB; any failure there is ErrSVGRenderFailed (415). Tests pinning the SVG refusal now expect SVG. Model: opus-5-5 --- README.md | 65 ++- cmd/pixad/main.go | 15 +- internal/handlers/image.go | 59 +- .../handlers/image_errors_internal_test.go | 2 +- .../unsupported_source_internal_test.go | 70 ++- internal/httpfetcher/httpfetcher.go | 1 + .../httpfetcher/httpfetcher_internal_test.go | 2 +- internal/httpfetcher/mock.go | 10 +- internal/imageprocessor/imageprocessor.go | 336 ++++++++++-- .../imageprocessor_internal_test.go | 5 + .../source_formats_internal_test.go | 2 +- internal/imageprocessor/svg_child_process.go | 255 +++++++++ internal/imageprocessor/svg_internal_test.go | 503 ++++++++++++++++++ internal/imagerequest/imagerequest.go | 6 +- internal/imgcache/cache.go | 16 +- internal/imgcache/cache_hit_internal_test.go | 2 - .../imgcache/source_content_internal_test.go | 2 - internal/imgcache/svg_internal_test.go | 54 ++ internal/magic/magic.go | 28 +- internal/magic/magic_internal_test.go | 6 +- internal/magic/svg_internal_test.go | 85 +++ internal/server/svg_internal_test.go | 132 ++++- 22 files changed, 1525 insertions(+), 131 deletions(-) create mode 100644 internal/imageprocessor/svg_child_process.go create mode 100644 internal/imageprocessor/svg_internal_test.go create mode 100644 internal/imgcache/svg_internal_test.go create mode 100644 internal/magic/svg_internal_test.go diff --git a/README.md b/README.md index 7839810..e089fc7 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,8 @@ run under. Its app for pixa needs: is half the CPUs pixa sees times 4.2 GiB, plus 64 times 50 MiB. On a host where pixa sees 48 CPUs, that is 24 times 4.2 GiB plus 3.1 GiB, about 104 GiB. Set `PIXA_MAX_CONCURRENT_PROCESSING` and `PIXA_UPSTREAM_CONNECTIONS` so that - this fits the memory the container may use. + this fits the memory the container may use. An SVG source is rendered in a + process of its own, which can use up to 1 GiB more. - **Temporary files:** while pixa processes a large source, libvips writes the decoded pixels to a file in `/tmp`. The file is removed from the directory once it is written, so a listing of `/tmp` does not show it. It keeps its @@ -312,6 +313,7 @@ to any method the list does not give for it. - 304 when `If-None-Match` matches the image's `ETag`. - 400 for a URL or parameter that is not valid. - 400 when the format is `auto` and the `Accept` header is not valid. + - 400 for `orig` of an SVG source at a size other than `orig`. - 406 when the format is `auto` and `Accept` allows none of the formats it chooses from. - 401 for a missing or wrong signature. @@ -332,6 +334,10 @@ to any method the list does not give for it. format its `Content-Type` names. - 415 when the upstream sends an image whose first bytes libvips reads as another format. + - 415 for an SVG source with one side more than 8192 times the other, in any + format but `orig`. + - 415 for an SVG source that is not valid, or that takes more than 20 + seconds or 1 GiB of memory to render. - 502 when the upstream answers with an error status. - 502 for 5 minutes after that for the same source URL, when pixa has neither the image nor its source in its cache. @@ -360,6 +366,8 @@ to any method the list does not give for it. - 304 when `If-None-Match` matches the image's `ETag`. - 400 for a token that does not decrypt. - 400 for a token that asks for a size or fit that is not valid. + - 400 for a token that asks for `orig` of an SVG source at a size other than + `orig`. - 410 once the token has expired. - 400 when the token's format is `auto` and the `Accept` header is not valid. @@ -502,10 +510,11 @@ outside ASCII. pixa signs and caches the query in that form. So `img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are the same source query, `name=my%20cat`. -**Sizes.** The largest width or height is 8192, for the source and for the -output. A size with one side `0` has that side worked out from the other side -and the source's proportions. The side is rounded down, but is at least 1. When -it would come out above 8192, the request is refused with 400. The fit modes: +**Sizes.** The largest width or height is 8192, for the output and for a source +in any format but SVG. A size with one side `0` has that side worked out from +the other side and the source's proportions. The side is rounded down, but is at +least 1. When it would come out above 8192, the request is refused with 400. The +fit modes: - `cover`: scale the image to cover the size, keeping its proportions, and cut out the centre. The output is exactly the size. @@ -525,9 +534,9 @@ applies to the format `orig` when the source is AVIF, and to `auto` when it chooses AVIF. So a client whose `Accept` names `image/avif` but not `image/jxl` is refused such a size. -**Source formats.** The source may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The -upstream's `Content-Type` must name its format, and the image's first bytes must -match it. An upstream answering with any other `Content-Type`, SVG included, is +**Source formats.** The source may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or +SVG. The upstream's `Content-Type` must name its format, and the image's first +bytes must match it. An upstream answering with any other `Content-Type` is refused with 415. So is a body whose first bytes are not those of the format its `Content-Type` names: a JPEG sent as `image/png`, text sent as an image, or an empty body. pixa then does not fetch that source URL again for 5 minutes, as @@ -536,6 +545,20 @@ need that fetch. An image whose first bytes libvips reads as another format is refused with 415 too. An SVG whose first bytes pass the check for AVIF is one example. +**SVG sources.** `orig` of an SVG source is the SVG itself, unchanged, and is +cached like any other image. It is given only at the `orig` size, since pixa +does not resize an SVG as SVG. `orig` of an SVG at any other size is refused +with 400. A browser that opens an SVG from pixa applies its styles and draws its +embedded `data:` images, but runs none of its scripts and loads nothing else. + +In any format but `orig`, pixa renders an SVG source at the requested size, so +an enlarged SVG stays sharp. Rendering fetches no URL and reads no file that the +SVG refers to. In any format but `orig`, an SVG at the `orig` size is rendered +at its own size. In any format but `orig`, an SVG above 8192 on a side is +refused with 400 at the `orig` size. In any format but `orig`, an SVG with one +side more than 8192 times the other is refused with 415. Rendered with no side +above 8192, its shorter side would be under 1 pixel. + **The format `auto`.** With `auto`, pixa chooses the format for each request from its `Accept` header, in this order: @@ -746,8 +769,9 @@ written percent-encoded. ### Image Metadata -pixa decodes and re-encodes every image it serves, and removes all metadata from -the output. This cannot be turned off. The metadata removed is: +pixa decodes and re-encodes every image it serves, except `orig` of an SVG +source, which is the SVG unchanged. It removes all metadata from every image it +re-encodes. This cannot be turned off. The metadata removed is: - EXIF, which can hold the GPS position, the camera make, model and serial number, the capture time and an embedded thumbnail; @@ -756,14 +780,16 @@ the output. This cannot be turned off. The metadata removed is: The rest of the rules: -- The `orig` format means the source's own format, not the source's bytes. An - `orig` image is re-encoded and stripped like any other. +- For a source in any format but SVG, the `orig` format means the source's own + format, not the source's bytes. An `orig` image is re-encoded and stripped + like any other. For an SVG source, `orig` is the SVG's own bytes, its metadata + included. - An image with an EXIF orientation is turned upright first, so it displays the same without the tag. A requested size applies to the upright image. - An image with an ICC profile is first converted to sRGB, the standard colour space, since clients show an image with no profile as sRGB. Colours outside sRGB are clipped: the most saturated ones in a Display P3 photo, for example. -- JPEG XL output is the one exception, with libvips 8.16 and later. libvips then +- JPEG XL output is an exception, with libvips 8.16 and later. libvips then writes an EXIF block of its own, which pixa cannot stop. The block holds the image's size and fixed values: an orientation of 1 and a resolution of 72 dpi. Nothing in it comes from the source. The Docker image has libvips 8.16, so its @@ -798,11 +824,11 @@ through, gives no way to stop an encode once it has begun. So an encode keeps its slot until it finishes, which can be after `downstream_timeout`. **Encode times.** The limits on image size bound how long a conversion can take. -They are 8192 on each side of the source and of the output, and 16,777,216 -pixels for an AVIF output. Decoding an 8192x8192 source takes up to about 11 -seconds. The WebP and JPEG XL encoders run below libvips' default effort. AVIF -runs at the lowest effort `govips` can set. With one libvips thread, as pixa -runs it, encoding 8192x8192 random pixels takes up to about: +They are 8192 on each side of the output and of a source in any format but SVG, +and 16,777,216 pixels for an AVIF output. Decoding an 8192x8192 source takes up +to about 11 seconds. The WebP and JPEG XL encoders run below libvips' default +effort. AVIF runs at the lowest effort `govips` can set. With one libvips +thread, as pixa runs it, encoding 8192x8192 random pixels takes up to about: - 1 second as JPEG; - 15 seconds as PNG; @@ -815,7 +841,8 @@ about 34 seconds with an alpha channel, and 22 without. At 8192x8192 and quality 85 it would take about 96 seconds with an alpha channel, and 52 without. The 34 seconds hold at any quality and at 8 or 16 bits per sample. No encode within the limits takes more than about 40 seconds. The default `downstream_timeout` is 60 -seconds. +seconds. An SVG source is rendered in a process of its own, which pixa stops +after 20 seconds and limits to 1 GiB of memory. **Processing and connections.** `max_concurrent_processing` images are decoded and encoded at once. A request that finds every slot in use waits up to 10 diff --git a/cmd/pixad/main.go b/cmd/pixad/main.go index 7806619..16a5a48 100644 --- a/cmd/pixad/main.go +++ b/cmd/pixad/main.go @@ -1,10 +1,21 @@ -// Package main is the entry point for the pixad image proxy server. +// Package main is the entry point for the pixad image proxy server. pixad +// also runs as a child process of itself, to read the size of an SVG source +// or to render it (see imageprocessor.RunAsSVGChildProcess). package main -import "sneak.berlin/go/pixa/internal/app" +import ( + "os" + + "sneak.berlin/go/pixa/internal/app" + "sneak.berlin/go/pixa/internal/imageprocessor" +) var Version string //nolint:gochecknoglobals // set by ldflags func main() { + if imageprocessor.IsSVGChildProcess(os.Args[1:]) { + os.Exit(imageprocessor.RunAsSVGChildProcess(os.Args[1:])) + } + app.Run(Version) } diff --git a/internal/handlers/image.go b/internal/handlers/image.go index 108e3e6..792fb14 100644 --- a/internal/handlers/image.go +++ b/internal/handlers/image.go @@ -234,12 +234,8 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc { func (s *Handlers) respondImageError( w http.ResponseWriter, req *imagerequest.Request, err error, ) { - // An upstream body whose first bytes are not those of the format its - // Content-Type names is the upstream's error, not pixa's: the image - // service has logged it as a refused source. - if errors.Is(err, magic.ErrMagicByteMismatch) || - errors.Is(err, magic.ErrUnknownFormat) || - errors.Is(err, magic.ErrNotEnoughData) { + // The image service has logged this as a refused source + if isFirstBytesMismatch(err) { s.respondError(w, "unsupported source format", http.StatusUnsupportedMediaType) @@ -269,11 +265,29 @@ func (s *Handlers) respondImageError( errors.Is(err, imageprocessor.ErrUnsupportedInputFormat): status = http.StatusUnsupportedMediaType message = "unsupported source format" + case errors.Is(err, imageprocessor.ErrSVGTooThin): + status = http.StatusUnsupportedMediaType + message = fmt.Sprintf("SVG source too thin to render: one side is "+ + "more than %d times the other", imageprocessor.MaxInputDimension) + case errors.Is(err, imageprocessor.ErrSVGRenderFailed): + status = http.StatusUnsupportedMediaType + message = fmt.Sprintf("SVG source could not be rendered: it is not "+ + "valid, or it takes more than %d seconds or %d GiB of memory to "+ + "render", int(imageprocessor.SVGRenderTimeout.Seconds()), + imageprocessor.MaxSVGRenderMemory/bytesPerGiB) case errors.Is(err, imageprocessor.ErrOutputTooLarge): status = http.StatusBadRequest message = fmt.Sprintf("size too large for the source: the side "+ "computed from its proportions is above %d", imageprocessor.MaxOutputDimension) + case errors.Is(err, imageprocessor.ErrSVGNotOriginalSize): + status = http.StatusBadRequest + message = "an SVG source is returned as SVG only at the orig size; " + + "ask for another format to resize it" + case errors.Is(err, imageprocessor.ErrSVGTooLargeForOrigSize): + status = http.StatusBadRequest + message = fmt.Sprintf("SVG source above %d on a side: give a size "+ + "to render it at", imageprocessor.MaxOutputDimension) case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge): status = http.StatusBadRequest message = fmt.Sprintf("size too large for AVIF: the output "+ @@ -297,6 +311,18 @@ func (s *Handlers) respondImageError( s.respondError(w, message, status) } +// bytesPerGiB is the number of bytes in a GiB. +const bytesPerGiB = 1 << 30 + +// isFirstBytesMismatch reports whether err refuses an upstream body whose +// first bytes are not those of the format its Content-Type names. That is the +// upstream's error, not pixa's. +func isFirstBytesMismatch(err error) bool { + return errors.Is(err, magic.ErrMagicByteMismatch) || + errors.Is(err, magic.ErrUnknownFormat) || + errors.Is(err, magic.ErrNotEnoughData) +} + // cacheControl returns the Cache-Control header for an image served through a // URL that expires at expires, or never when expires is the zero time. A cache // may keep the image for a year, but not past the URL's expiry, after which @@ -315,6 +341,26 @@ func cacheControl(expires time.Time) string { return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second)) } +// svgContentSecurityPolicyValue is the Content-Security-Policy header value +// of an SVG response, in place of the middleware's. A browser that opens the +// SVG applies its styles and draws its embedded data: images, but runs none +// of its scripts and loads nothing else. The styles need unsafe-inline, as an +// SVG keeps them inline, in style elements and attributes. +const svgContentSecurityPolicyValue = "default-src 'none'; " + + "style-src 'unsafe-inline'; " + + "img-src data:; " + + "frame-ancestors 'none'; " + + "sandbox" + +// setSVGContentSecurityPolicy sets the Content-Security-Policy in header to +// svgContentSecurityPolicyValue when contentType is that of SVG. Any other +// response keeps the middleware's policy. +func setSVGContentSecurityPolicy(header http.Header, contentType string) { + if contentType == imagerequest.FormatSVG.MIMEType() { + header.Set("Content-Security-Policy", svgContentSecurityPolicyValue) + } +} + // refuseBlockedReferer answers 403 with a JSON error when the request's Referer // names a host on referer_blocklist, and reports whether it answered. A request // with no Referer, or one that does not parse as a URL with a host, is not @@ -397,6 +443,7 @@ func (s *Handlers) serveImage( // Set response headers w.Header().Set("Content-Type", resp.ContentType) + setSVGContentSecurityPolicy(w.Header(), resp.ContentType) if resp.ContentLength > 0 { w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10)) diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go index 360a6f1..e398e15 100644 --- a/internal/handlers/image_errors_internal_test.go +++ b/internal/handlers/image_errors_internal_test.go @@ -475,7 +475,7 @@ func TestImageRoutes_SameErrorAnswers(t *testing.T) { {"no free upstream connection", httpfetcher.ErrTooManyConnections, http.StatusServiceUnavailable, slog.LevelWarn}, {"Content-Type not a source format", - fmt.Errorf("%w: image/svg+xml", httpfetcher.ErrInvalidContentType), + fmt.Errorf("%w: image/tiff", httpfetcher.ErrInvalidContentType), http.StatusUnsupportedMediaType, slog.LevelWarn}, {"any other failure", io.ErrUnexpectedEOF, http.StatusInternalServerError, slog.LevelError}, diff --git a/internal/handlers/unsupported_source_internal_test.go b/internal/handlers/unsupported_source_internal_test.go index dbb05b3..fb92c91 100644 --- a/internal/handlers/unsupported_source_internal_test.go +++ b/internal/handlers/unsupported_source_internal_test.go @@ -15,7 +15,7 @@ import ( // TestUnsupportedSourceFormatAnswers415 checks that respondImageError, which // answers the errors of both image routes, answers 415 with a clear error when -// the upstream answers with a content type pixa does not accept, such as SVG, +// the upstream answers with a content type pixa does not accept, such as TIFF, // and when libvips does not read a source as the format its first bytes // show, wrapped as the service wraps them. func TestUnsupportedSourceFormatAnswers415(t *testing.T) { @@ -23,11 +23,11 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) { h := &Handlers{log: slog.New(slog.DiscardHandler)} req := &imagerequest.Request{ - Source: imagerequest.Source{Host: "img.example.com", Path: "/a.svg"}, + Source: imagerequest.Source{Host: "img.example.com", Path: "/a.tiff"}, } for _, err := range []error{ - fmt.Errorf("upstream fetch failed: %w: image/svg+xml", + fmt.Errorf("upstream fetch failed: %w: image/tiff", httpfetcher.ErrInvalidContentType), fmt.Errorf("image processing failed: %w: libvips does not read the input as avif", imageprocessor.ErrUnsupportedInputFormat), @@ -47,3 +47,67 @@ func TestUnsupportedSourceFormatAnswers415(t *testing.T) { } } } + +// TestSVGTooThinAnswers415 checks that respondImageError answers 415 with a +// clear error for an SVG source too thin to render, wrapped as the service +// wraps it. +func TestSVGTooThinAnswers415(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + req := &imagerequest.Request{ + Source: imagerequest.Source{Host: "svg.example.com", Path: "/a.svg"}, + } + err := fmt.Errorf("image processing failed: %w: 81930x10", + imageprocessor.ErrSVGTooThin) + + rec := httptest.NewRecorder() + h.respondImageError(rec, req, err) + t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String()) + + if rec.Code != http.StatusUnsupportedMediaType { + t.Errorf("status = %d, want %d", rec.Code, + http.StatusUnsupportedMediaType) + } + + if !strings.Contains(rec.Body.String(), "SVG source too thin to render") { + t.Errorf("body = %q, want the SVG too thin error", rec.Body.String()) + } +} + +// TestSVGRenderFailedAnswers415 checks that respondImageError answers 415, +// naming the limits, for an SVG source that a child process could not read +// or render, wrapped as the service wraps it. +func TestSVGRenderFailedAnswers415(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + err := fmt.Errorf("image processing failed: %w: it took more than 20s", + imageprocessor.ErrSVGRenderFailed) + + rec := httptest.NewRecorder() + h.respondImageError(rec, &imagerequest.Request{}, err) + t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String()) + + checkErrorBody(t, rec, http.StatusUnsupportedMediaType, + "SVG source could not be rendered: it is not valid, or it takes more "+ + "than 20 seconds or 1 GiB of memory to render") +} + +// TestSVGTooLargeForOrigSizeAnswers400 checks that respondImageError answers +// 400, asking for a size, for an SVG source above 8192 on a side asked for at +// the orig size in a raster format, wrapped as the service wraps it. +func TestSVGTooLargeForOrigSizeAnswers400(t *testing.T) { + t.Parallel() + + h := &Handlers{log: slog.New(slog.DiscardHandler)} + err := fmt.Errorf("image processing failed: %w: 10000x10 is above 8192", + imageprocessor.ErrSVGTooLargeForOrigSize) + + rec := httptest.NewRecorder() + h.respondImageError(rec, &imagerequest.Request{}, err) + t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String()) + + checkErrorBody(t, rec, http.StatusBadRequest, + "SVG source above 8192 on a side: give a size to render it at") +} diff --git a/internal/httpfetcher/httpfetcher.go b/internal/httpfetcher/httpfetcher.go index 4a01128..ddeb743 100644 --- a/internal/httpfetcher/httpfetcher.go +++ b/internal/httpfetcher/httpfetcher.go @@ -128,6 +128,7 @@ func allowedContentTypes() []string { imagerequest.FormatWebP.MIMEType(), imagerequest.FormatAVIF.MIMEType(), imagerequest.FormatJXL.MIMEType(), + imagerequest.FormatSVG.MIMEType(), } } diff --git a/internal/httpfetcher/httpfetcher_internal_test.go b/internal/httpfetcher/httpfetcher_internal_test.go index 20d8968..2ee7821 100644 --- a/internal/httpfetcher/httpfetcher_internal_test.go +++ b/internal/httpfetcher/httpfetcher_internal_test.go @@ -279,7 +279,7 @@ func TestDetectContentTypeFromPath(t *testing.T) { {"foo/bar.gif", imagerequest.FormatGIF.MIMEType()}, {"foo/bar.webp", imagerequest.FormatWebP.MIMEType()}, {"foo/bar.avif", imagerequest.FormatAVIF.MIMEType()}, - {"foo/bar.svg", contentTypeSVG}, + {"foo/bar.svg", imagerequest.FormatSVG.MIMEType()}, {"foo/bar.bin", contentTypeOctetStream}, {"foo/bar", contentTypeOctetStream}, } diff --git a/internal/httpfetcher/mock.go b/internal/httpfetcher/mock.go index 5101cfc..e4312b6 100644 --- a/internal/httpfetcher/mock.go +++ b/internal/httpfetcher/mock.go @@ -67,11 +67,9 @@ func (m *MockFetcher) Fetch( }, nil } -// The content types the mock fetcher gives files in no format pixa reads. -const ( - contentTypeSVG = "image/svg+xml" - contentTypeOctetStream = "application/octet-stream" -) +// contentTypeOctetStream is the content type the mock fetcher gives files in +// no format pixa reads. +const contentTypeOctetStream = "application/octet-stream" // detectContentTypeFromPath returns the MIME type based on file extension. func detectContentTypeFromPath(path string) string { @@ -91,7 +89,7 @@ func detectContentTypeFromPath(path string) string { case strings.HasSuffix(path, ".jxl"): return imagerequest.FormatJXL.MIMEType() case strings.HasSuffix(path, ".svg"): - return contentTypeSVG + return imagerequest.FormatSVG.MIMEType() default: return contentTypeOctetStream } diff --git a/internal/imageprocessor/imageprocessor.go b/internal/imageprocessor/imageprocessor.go index 18fdd12..8294d29 100644 --- a/internal/imageprocessor/imageprocessor.go +++ b/internal/imageprocessor/imageprocessor.go @@ -1,4 +1,7 @@ // Package imageprocessor provides image format conversion and resizing using libvips. +// It reads the size of an SVG source and renders it in child processes, each +// the running program started again (see RunAsSVGChildProcess). pixa kills +// them after SVGRenderTimeout, and each may use at most MaxSVGRenderMemory. package imageprocessor import ( @@ -7,6 +10,8 @@ import ( "errors" "fmt" "io" + "math" + "strconv" "sync" "time" @@ -113,10 +118,43 @@ var ErrUnsupportedOutputFormat = errors.New("unsupported output format") // ErrUnsupportedInputFormat is returned, before the input is decoded, when // libvips reads the input's first bytes as another format than its source // format, or as none. An SVG whose first bytes pass the first-bytes check for -// AVIF is one example. A source format other than the six pixa reads is also -// refused with it. +// AVIF is one example. A source format other than the seven pixa reads is +// also refused with it. var ErrUnsupportedInputFormat = errors.New("unsupported input format") +// ErrSVGNotOriginalSize is returned when orig is asked of an SVG source at a +// size other than orig. pixa returns an SVG only as it is; a raster format +// gives it a size. +var ErrSVGNotOriginalSize = errors.New( + "an SVG is returned as SVG only at the orig size") + +// ErrSVGTooThin is returned, before rendering, when an SVG source has one +// side more than MaxInputDimension times the other. Rendered with its longer +// side at most MaxInputDimension, its shorter side would be under 1 pixel. +var ErrSVGTooThin = errors.New("SVG source too thin to render") + +// ErrSVGTooLargeForOrigSize is returned, before rendering, when a format other +// than orig asks for an SVG source at the orig size and the SVG has a side +// above MaxOutputDimension. +var ErrSVGTooLargeForOrigSize = errors.New( + "SVG source too large to render at its own size") + +// SVGRenderTimeout is how long the child processes that read the size of one +// SVG source and render it may run, counted from the start of the first, +// before pixa kills them. +const SVGRenderTimeout = 20 * time.Second + +// MaxSVGRenderMemory is the most memory, in bytes, that each child process +// reading the size of an SVG source or rendering it may use. It is the +// child's RLIMIT_DATA, which limits its heap and its other private writable +// memory. +const MaxSVGRenderMemory = 1 << 30 // 1 GiB + +// ErrSVGRenderFailed is returned when a child process fails to read the size +// of an SVG source or to render it: librsvg cannot read the SVG, or the +// child takes more than SVGRenderTimeout or MaxSVGRenderMemory. +var ErrSVGRenderFailed = errors.New("SVG source could not be rendered") + // ErrTooManyImages is returned when MaxConcurrentProcessing images are being // processed and none finishes within ProcessingWaitTimeout. var ErrTooManyImages = errors.New("too many images being processed at once") @@ -134,6 +172,10 @@ type ImageProcessor struct { processingSemaphore chan struct{} // processingWaitTimeout is ProcessingWaitTimeout; tests shorten it. processingWaitTimeout time.Duration + // svgRenderTimeout is SVGRenderTimeout; tests shorten it. + svgRenderTimeout time.Duration + // maxSVGRenderMemory is MaxSVGRenderMemory; tests lower it. + maxSVGRenderMemory int } // Params holds configuration for creating an ImageProcessor. Both fields @@ -153,6 +195,8 @@ func New(params Params) *ImageProcessor { maxInputBytes: params.MaxInputBytes, processingSemaphore: make(chan struct{}, params.MaxConcurrentProcessing), processingWaitTimeout: ProcessingWaitTimeout, + svgRenderTimeout: SVGRenderTimeout, + maxSVGRenderMemory: MaxSVGRenderMemory, } } @@ -164,7 +208,10 @@ func New(params Params) *ImageProcessor { // encodes, it returns ctx's error and gives the slot back. govips gives pixa // no way to stop libvips once an encode has begun, so an encode keeps its // slot until it finishes. An AVIF output of more than MaxAVIFOutputPixels is -// refused with ErrAVIFOutputTooLarge before anything is decoded. +// refused with ErrAVIFOutputTooLarge before anything is decoded. orig of an +// SVG source is the SVG as it is, since libvips cannot save SVG; any other +// format renders it at the output size. Child processes read an SVG's size +// and render it, as processSVG describes. func (p *ImageProcessor) Process( ctx context.Context, input io.Reader, @@ -183,35 +230,18 @@ func (p *ImageProcessor) Process( return nil, err } - // Input libvips would not load as sourceFormat is refused before it is - // decoded - err = checkInputFormat(data, sourceFormat) + if sourceFormat == imagerequest.FormatSVG { + return p.processSVG(ctx, data, transform) + } + + img, err := loadImage(data, sourceFormat) if err != nil { return nil, err } - - // Decode image - img, err := vips.NewImageFromBuffer(data) - if err != nil { - return nil, fmt.Errorf("failed to decode image: %w", err) - } defer img.Close() - // Turn the image upright now: encode strips the EXIF orientation tag, - // and sizes below must be worked out on the upright image. - err = img.AutoRotate() - if err != nil { - return nil, fmt.Errorf("failed to auto-rotate: %w", err) - } - - // Get original dimensions - origWidth := img.Width() - origHeight := img.Height() - - // Validate input dimensions to prevent DoS via decompression bombs - if origWidth > MaxInputDimension || origHeight > MaxInputDimension { - return nil, ErrInputTooLarge - } + origWidth, origHeight := img.Width(), img.Height() + outputFormat := resolveOutputFormat(transform.Format, sourceFormat) // Determine target dimensions; too large ones are refused before resizing targetWidth, targetHeight, err := targetDimensions(transform.Size, @@ -220,12 +250,6 @@ func (p *ImageProcessor) Process( return nil, err } - // orig is the source's own format; encode refuses an empty format - outputFormat := transform.Format - if outputFormat == imagerequest.FormatOriginal { - outputFormat = sourceFormat - } - // Determine output dimensions; too large ones are refused before // resizing, which with FitCover already decodes the input outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight, @@ -242,8 +266,104 @@ func (p *ImageProcessor) Process( } } - // Encode to target format - output, err := p.encode(ctx, img, outputFormat, transform.Quality) + return p.encodeResult(ctx, img, outputFormat, transform.Quality, + origWidth, origHeight) +} + +// WaitForProcessing waits until no image is being processed, or until ctx +// ends, and returns how many images were still being processed then. It +// waits by taking each slot in processingSemaphore as it frees up until it +// holds them all, or until ctx ends, then gives back the slots it took. +func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int { + taken := 0 + + defer func() { + for range taken { + <-p.processingSemaphore + } + }() + + for taken < cap(p.processingSemaphore) { + select { + case p.processingSemaphore <- struct{}{}: + taken++ + case <-ctx.Done(): + return len(p.processingSemaphore) - taken + } + } + + return 0 +} + +// processSVG is Process for an SVG source, data. Apart from the first-bytes +// check, pixad hands the SVG to libvips only in child processes (see +// runSVGChildProcess). One reads the SVG's size. For a raster format a second +// renders it as PNG, which is then resized and encoded as any other image. +// pixa kills them once svgRenderTimeout has passed since the first one +// started. +func (p *ImageProcessor) processSVG( + ctx context.Context, data []byte, transform *imagerequest.Transform, +) (*Result, error) { + err := checkInputFormat(data, imagerequest.FormatSVG) + if err != nil { + return nil, err + } + + deadline := time.Now().Add(p.svgRenderTimeout) + + origWidth, origHeight, err := p.readSVGSize(ctx, deadline, data) + if err != nil { + return nil, err + } + + outputFormat := resolveOutputFormat(transform.Format, imagerequest.FormatSVG) + if outputFormat == imagerequest.FormatSVG { + return svgAsItIs(data, transform.Size, origWidth, origHeight) + } + + err = checkSVGOrigSize(transform.Size, origWidth, origHeight) + if err != nil { + return nil, err + } + + targetWidth, targetHeight, err := targetDimensions(transform.Size, + origWidth, origHeight) + if err != nil { + return nil, err + } + + outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight, + targetWidth, targetHeight, transform.FitMode, outputFormat) + if err != nil { + return nil, err + } + + // The SVG is rendered at about the output size, to stay sharp + img, err := p.renderSVG(ctx, deadline, data, origWidth, origHeight, + outputWidth, outputHeight) + if err != nil { + return nil, err + } + defer img.Close() + + if targetWidth != origWidth || targetHeight != origHeight { + err := p.resize(img, outputWidth, outputHeight, transform.FitMode) + if err != nil { + return nil, fmt.Errorf("failed to resize: %w", err) + } + } + + return p.encodeResult(ctx, img, outputFormat, transform.Quality, + origWidth, origHeight) +} + +// encodeResult encodes img as format at quality and returns it as the result +// for a source of origWidth x origHeight. +func (p *ImageProcessor) encodeResult( + ctx context.Context, img *vips.ImageRef, format imagerequest.Format, + quality, origWidth, origHeight int, +) (*Result, error) { + output, err := p.encode(ctx, img, format, quality) if err != nil { return nil, fmt.Errorf("failed to encode: %w", err) } @@ -251,7 +371,7 @@ func (p *ImageProcessor) Process( return &Result{ Content: io.NopCloser(bytes.NewReader(output)), ContentLength: int64(len(output)), - ContentType: outputFormat.MIMEType(), + ContentType: format.MIMEType(), Width: img.Width(), Height: img.Height(), InputWidth: origWidth, @@ -290,29 +410,125 @@ func targetDimensions( return width, height, nil } -// WaitForProcessing waits until no image is being processed, or until ctx -// ends, and returns how many images were still being processed then. It -// waits by taking each slot in processingSemaphore as it frees up until it -// holds them all, or until ctx ends, then gives back the slots it took. -func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int { - taken := 0 - - defer func() { - for range taken { - <-p.processingSemaphore - } - }() - - for taken < cap(p.processingSemaphore) { - select { - case p.processingSemaphore <- struct{}{}: - taken++ - case <-ctx.Done(): - return len(p.processingSemaphore) - taken - } +// resolveOutputFormat returns the format of the output for the requested +// format: sourceFormat for orig, and format itself for any other. encode +// refuses an empty format. +func resolveOutputFormat( + format, sourceFormat imagerequest.Format, +) imagerequest.Format { + if format == imagerequest.FormatOriginal { + return sourceFormat } - return 0 + return format +} + +// loadImage refuses data that libvips would not load as sourceFormat, then +// decodes it and turns the image upright: encode strips the EXIF orientation +// tag, and sizes must be worked out on the upright image. An image with a +// side above MaxInputDimension is ErrInputTooLarge, which guards against +// decompression bombs. +func loadImage( + data []byte, sourceFormat imagerequest.Format, +) (*vips.ImageRef, error) { + err := checkInputFormat(data, sourceFormat) + if err != nil { + return nil, err + } + + img, err := vips.NewImageFromBuffer(data) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + err = img.AutoRotate() + if err != nil { + img.Close() + + return nil, fmt.Errorf("failed to auto-rotate: %w", err) + } + + if img.Width() > MaxInputDimension || img.Height() > MaxInputDimension { + img.Close() + + return nil, ErrInputTooLarge + } + + return img, nil +} + +// svgAsItIs returns the SVG data unchanged, which is orig of an SVG source +// at the orig size. width x height is the SVG's own size. Any other size is +// ErrSVGNotOriginalSize. +func svgAsItIs( + data []byte, size imagerequest.Size, width, height int, +) (*Result, error) { + if !size.OriginalSize() { + return nil, fmt.Errorf("%w: asked for %dx%d", ErrSVGNotOriginalSize, + size.Width, size.Height) + } + + return &Result{ + Content: io.NopCloser(bytes.NewReader(data)), + ContentLength: int64(len(data)), + ContentType: imagerequest.FormatSVG.MIMEType(), + Width: width, + Height: height, + InputWidth: width, + InputHeight: height, + }, nil +} + +// checkSVGOrigSize returns ErrSVGTooLargeForOrigSize when size is the orig +// size and the SVG source, of width x height, has a side above +// MaxOutputDimension. processSVG calls it after returning orig of the SVG, so +// the SVG would be rendered at its own size. +func checkSVGOrigSize(size imagerequest.Size, width, height int) error { + if size.OriginalSize() && max(width, height) > MaxOutputDimension { + return fmt.Errorf("%w: %dx%d is above %d", ErrSVGTooLargeForOrigSize, + width, height, MaxOutputDimension) + } + + return nil +} + +// svgRenderSize returns the least size in the proportions of width x height, +// an SVG's own size, that covers outputWidth x outputHeight, so that resize +// hardly changes it. Neither side is above MaxInputDimension. +func svgRenderSize(width, height, outputWidth, outputHeight int) (int, int) { + scale := min( + max(float64(outputWidth)/float64(width), + float64(outputHeight)/float64(height)), + float64(MaxInputDimension)/float64(max(width, height))) + + return int(math.Round(float64(width) * scale)), + int(math.Round(float64(height) * scale)) +} + +// renderSVG renders the SVG data, whose own size is width x height, at +// svgRenderSize for outputWidth x outputHeight, in a child process that pixa +// kills at deadline (see runSVGChildProcess). It returns the rendered image, +// read from the child's PNG. An SVG with one side more than +// MaxInputDimension times the other is ErrSVGTooThin. +func (p *ImageProcessor) renderSVG( + ctx context.Context, deadline time.Time, data []byte, + width, height, outputWidth, outputHeight int, +) (*vips.ImageRef, error) { + if max(width, height) > MaxInputDimension*min(width, height) { + return nil, fmt.Errorf("%w: %dx%d", ErrSVGTooThin, width, height) + } + + renderWidth, renderHeight := svgRenderSize(width, height, + outputWidth, outputHeight) + + png, err := p.runSVGChildProcess(ctx, deadline, data, + renderSVGAsPNGArgument, strconv.Itoa(renderWidth), + strconv.Itoa(renderHeight)) + if err != nil { + return nil, err + } + + return loadImage(png, imagerequest.FormatPNG) } // acquireSlot takes a slot in processingSemaphore, waiting at most @@ -397,6 +613,8 @@ func vipsImageType(format imagerequest.Format) vips.ImageType { return vips.ImageTypeAVIF case imagerequest.FormatJXL: return vips.ImageTypeJXL + case imagerequest.FormatSVG: + return vips.ImageTypeSVG case imagerequest.FormatOriginal, imagerequest.FormatAuto: return vips.ImageTypeUnknown default: @@ -575,7 +793,9 @@ func (p *ImageProcessor) encode( case imagerequest.FormatJXL: return exportJXL(img, quality) - case imagerequest.FormatOriginal, imagerequest.FormatAuto: + // libvips cannot save SVG; Process returns an SVG source as it is + case imagerequest.FormatSVG, imagerequest.FormatOriginal, + imagerequest.FormatAuto: return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format) default: diff --git a/internal/imageprocessor/imageprocessor_internal_test.go b/internal/imageprocessor/imageprocessor_internal_test.go index dca5645..6d21fc6 100644 --- a/internal/imageprocessor/imageprocessor_internal_test.go +++ b/internal/imageprocessor/imageprocessor_internal_test.go @@ -19,6 +19,11 @@ import ( ) func TestMain(m *testing.M) { + // The SVG tests start this test binary again as SVG child processes + if IsSVGChildProcess(os.Args[1:]) { + os.Exit(RunAsSVGChildProcess(os.Args[1:])) + } + initVips() code := m.Run() diff --git a/internal/imageprocessor/source_formats_internal_test.go b/internal/imageprocessor/source_formats_internal_test.go index ac0255a..4e65aad 100644 --- a/internal/imageprocessor/source_formats_internal_test.go +++ b/internal/imageprocessor/source_formats_internal_test.go @@ -51,7 +51,7 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { } // TestImageProcessor_NotASourceFormatRefused verifies that a source format -// other than the six, such as orig, auto or none, is refused with +// other than the seven, such as orig, auto or none, is refused with // ErrUnsupportedInputFormat. The input is text, which libvips reads as no // format. func TestImageProcessor_NotASourceFormatRefused(t *testing.T) { diff --git a/internal/imageprocessor/svg_child_process.go b/internal/imageprocessor/svg_child_process.go new file mode 100644 index 0000000..0842aeb --- /dev/null +++ b/internal/imageprocessor/svg_child_process.go @@ -0,0 +1,255 @@ +package imageprocessor + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "syscall" + "time" + + "github.com/davidbyttow/govips/v2/vips" +) + +// An SVG child process is the running program started again with one of +// these arguments first, naming what it does. Its command line, after the +// program's name, is one of: +// +// read-svg-size MEMORY_LIMIT +// render-svg-as-png MEMORY_LIMIT WIDTH HEIGHT +// +// MEMORY_LIMIT is in bytes. The child reads the SVG on standard input. It +// writes the SVG's size as WIDTH HEIGHT, or the SVG rendered at WIDTH x +// HEIGHT as PNG, to standard output. +const ( + readSVGSizeArgument = "read-svg-size" + renderSVGAsPNGArgument = "render-svg-as-png" +) + +// errSVGChildProcessArguments is returned for a command line that is not +// that of an SVG child process. +var errSVGChildProcessArguments = errors.New( + "not the command line of an SVG child process") + +// IsSVGChildProcess reports whether args, a process's command line after the +// program's name, are those of an SVG child process. +func IsSVGChildProcess(args []string) bool { + return len(args) > 0 && + (args[0] == readSVGSizeArgument || args[0] == renderSVGAsPNGArgument) +} + +// RunAsSVGChildProcess runs this process as the SVG child process that args, +// its command line after the program's name, ask for. It returns the status +// to exit with: 0 once it has written its result, and 1 after writing an +// error to standard error, which the parent discards. +func RunAsSVGChildProcess(args []string) int { + err := runAsSVGChildProcess(args, os.Stdin, os.Stdout) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + + return 1 + } + + return 0 +} + +// runAsSVGChildProcess does the work of RunAsSVGChildProcess, reading the SVG +// from stdin and writing the result to stdout. It sets the process's limits +// before it reads the SVG. +func runAsSVGChildProcess( + args []string, stdin io.Reader, stdout io.Writer, +) error { + numbers, err := parseSVGChildProcessArguments(args) + if err != nil { + return err + } + + err = setSVGChildProcessLimits(numbers[0]) + if err != nil { + return err + } + + initVips() + + svg, err := io.ReadAll(stdin) + if err != nil { + return fmt.Errorf("failed to read the SVG: %w", err) + } + + if args[0] == readSVGSizeArgument { + return writeSVGSize(svg, stdout) + } + + return writeSVGAsPNG(svg, numbers[1], numbers[2], stdout) +} + +// parseSVGChildProcessArguments returns the numbers on the command line of an +// SVG child process, args: the memory limit, then the width and height for +// render-svg-as-png. +func parseSVGChildProcessArguments(args []string) ([]int, error) { + wantArgs := 2 + if len(args) > 0 && args[0] == renderSVGAsPNGArgument { + wantArgs = 4 + } + + if !IsSVGChildProcess(args) || len(args) != wantArgs { + return nil, fmt.Errorf("%w: %q", errSVGChildProcessArguments, args) + } + + numbers := make([]int, 0, len(args)-1) + + for _, arg := range args[1:] { + number, err := strconv.Atoi(arg) + if err != nil { + return nil, fmt.Errorf("%w: %q: %w", errSVGChildProcessArguments, + args, err) + } + + numbers = append(numbers, number) + } + + return numbers, nil +} + +// setSVGChildProcessLimits sets this process's RLIMIT_DATA to memoryLimit +// bytes, so that allocating heap or other private writable memory past it +// fails. RLIMIT_AS would also count the address space the Go runtime reserves +// but does not use. It also sets RLIMIT_CORE to 0: librsvg crashes when an +// allocation fails, and the crash would write a core file of up to +// memoryLimit bytes into pixad's working directory. +func setSVGChildProcessLimits(memoryLimit int) error { + if memoryLimit < 1 { + return fmt.Errorf("%w: memory limit %d", errSVGChildProcessArguments, + memoryLimit) + } + + err := syscall.Setrlimit(syscall.RLIMIT_DATA, &syscall.Rlimit{ + Cur: uint64(memoryLimit), + Max: uint64(memoryLimit), + }) + if err != nil { + return fmt.Errorf("failed to limit memory to %d bytes: %w", + memoryLimit, err) + } + + err = syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{}) + if err != nil { + return fmt.Errorf("failed to turn off core files: %w", err) + } + + return nil +} + +// writeSVGSize writes the size of svg to stdout as WIDTH HEIGHT. librsvg +// reads the whole SVG for it. +func writeSVGSize(svg []byte, stdout io.Writer) error { + img, err := vips.NewImageFromBuffer(svg) + if err != nil { + return fmt.Errorf("failed to read the SVG: %w", err) + } + + defer img.Close() + + _, err = fmt.Fprintf(stdout, "%d %d\n", img.Width(), img.Height()) + + return err +} + +// writeSVGAsPNG renders svg at width x height and writes it to stdout as PNG. +// libvips' thumbnail has librsvg render the SVG at that size directly, at +// any scale: govips takes a density only in whole dots per inch, which cannot +// render an SVG below 1/72 of its own size. Given the SVG as bytes with no +// file name, librsvg loads no file or URL the SVG refers to. +func writeSVGAsPNG(svg []byte, width, height int, stdout io.Writer) error { + img, err := vips.NewThumbnailWithSizeFromBuffer(svg, width, height, + vips.InterestingNone, vips.SizeBoth) + if err != nil { + return fmt.Errorf("failed to render the SVG: %w", err) + } + + defer img.Close() + + png, err := exportPNG(img) + if err != nil { + return fmt.Errorf("failed to save the SVG as PNG: %w", err) + } + + _, err = stdout.Write(png) + + return err +} + +// readSVGSize reads the size of the SVG data in a child process that pixa +// kills at deadline (see runSVGChildProcess). +func (p *ImageProcessor) readSVGSize( + ctx context.Context, deadline time.Time, data []byte, +) (int, int, error) { + output, err := p.runSVGChildProcess(ctx, deadline, data, + readSVGSizeArgument) + if err != nil { + return 0, 0, err + } + + var width, height int + + _, err = fmt.Sscan(string(output), &width, &height) + if err != nil { + return 0, 0, fmt.Errorf("failed to read the size %q an SVG child "+ + "process wrote: %w", output, err) + } + + return width, height, nil +} + +// runSVGChildProcess starts the running program again as an SVG child +// process, with argument, p.maxSVGRenderMemory and extraArgs as its command +// line, and waits for it. It gives the child data on standard input, +// discards its standard error, and returns its standard output. pixa kills +// the child at deadline. A child that fails or is killed is +// ErrSVGRenderFailed, unless ctx has ended: runSVGChildProcess then returns +// ctx's error. Failing to start the child is another error. +func (p *ImageProcessor) runSVGChildProcess( + ctx context.Context, deadline time.Time, data []byte, + argument string, extraArgs ...string, +) ([]byte, error) { + executable, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("failed to find the running program: %w", err) + } + + childCtx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + + args := append([]string{argument, strconv.Itoa(p.maxSVGRenderMemory)}, + extraArgs...) + + //nolint:gosec // the running program, with arguments pixa makes + cmd := exec.CommandContext(childCtx, executable, args...) + cmd.Stdin = bytes.NewReader(data) + + var stdout bytes.Buffer + + cmd.Stdout = &stdout + + err = cmd.Run() + + var exitError *exec.ExitError + + switch { + case err == nil: + return stdout.Bytes(), nil + case ctx.Err() != nil: + return nil, ctx.Err() + case childCtx.Err() != nil: + return nil, fmt.Errorf("%w: it took more than %v", ErrSVGRenderFailed, + p.svgRenderTimeout) + case errors.As(err, &exitError): + return nil, fmt.Errorf("%w: %w", ErrSVGRenderFailed, err) + default: + return nil, fmt.Errorf("failed to run an SVG child process: %w", err) + } +} diff --git a/internal/imageprocessor/svg_internal_test.go b/internal/imageprocessor/svg_internal_test.go new file mode 100644 index 0000000..957c299 --- /dev/null +++ b/internal/imageprocessor/svg_internal_test.go @@ -0,0 +1,503 @@ +package imageprocessor + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/davidbyttow/govips/v2/vips" + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// testSVG is a 16x12 SVG, black on its left half and white on its right. +const testSVG = `` + + `` + + `` + +// newSVGTestProcessor returns a processor for the SVG tests. +func newSVGTestProcessor() *ImageProcessor { + return New(Params{MaxInputBytes: 1 << 20, MaxConcurrentProcessing: 1}) +} + +// processSVG processes svg, as an SVG source, with transform. +func processSVG( + t *testing.T, svg string, transform imagerequest.Transform, +) (*Result, []byte, error) { + t.Helper() + + result, err := newSVGTestProcessor().Process(t.Context(), + bytes.NewReader([]byte(svg)), imagerequest.FormatSVG, &transform) + if err != nil { + return nil, nil, err + } + + output, err := io.ReadAll(result.Content) + if err != nil { + t.Fatalf("reading the output: %v", err) + } + + return result, output, nil +} + +// TestImageProcessor_SVGOrigIsTheSVG checks that orig of an SVG at the orig +// size is the SVG itself, never another format, and that orig at another +// size is ErrSVGNotOriginalSize. +func TestImageProcessor_SVGOrigIsTheSVG(t *testing.T) { + t.Parallel() + + result, output, err := processSVG(t, testSVG, + imagerequest.Transform{Format: imagerequest.FormatOriginal}) + if err != nil { + t.Fatalf("orig of the SVG: %v", err) + } + + t.Logf("orig of the SVG: %s, %dx%d", result.ContentType, result.Width, + result.Height) + + if result.ContentType != "image/svg+xml" || string(output) != testSVG { + t.Errorf("orig of the SVG = %s %q, want image/svg+xml %q", + result.ContentType, output, testSVG) + } + + _, _, err = processSVG(t, testSVG, imagerequest.Transform{ + Size: imagerequest.Size{Width: 32, Height: 24}, + Format: imagerequest.FormatOriginal, + }) + if !errors.Is(err, ErrSVGNotOriginalSize) { + t.Errorf("orig of the SVG at 32x24: error = %v, want %v", err, + ErrSVGNotOriginalSize) + } +} + +// TestImageProcessor_RendersSVGToEachRasterFormat checks that an SVG is +// rendered to each raster format at the requested size. +func TestImageProcessor_RendersSVGToEachRasterFormat(t *testing.T) { + t.Parallel() + + for _, format := range []imagerequest.Format{ + imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, + imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, + } { + result, output, err := processSVG(t, testSVG, imagerequest.Transform{ + Size: imagerequest.Size{Width: 40, Height: 30}, + Format: format, + }) + if err != nil { + t.Errorf("SVG as %s: %v", format, err) + + continue + } + + t.Logf("SVG as %s: %s, %dx%d, %d bytes", format, result.ContentType, + result.Width, result.Height, len(output)) + + if result.ContentType != format.MIMEType() || + vips.DetermineImageType(output) != vipsImageType(format) || + result.Width != 40 || result.Height != 30 { + t.Errorf("SVG as %s = %s, %dx%d; want %s, 40x30", format, + result.ContentType, result.Width, result.Height, format.MIMEType()) + } + } +} + +// TestImageProcessor_RendersSVGSharpWhenEnlarged checks that a 16x12 SVG +// asked for at 1600x1200 is rendered at that size, not rendered at 16x12 and +// enlarged: 10 pixels either side of the edge between its black and white +// halves are black and white, where an enlarged image is grey there. +func TestImageProcessor_RendersSVGSharpWhenEnlarged(t *testing.T) { + t.Parallel() + + _, output, err := processSVG(t, testSVG, imagerequest.Transform{ + Size: imagerequest.Size{Width: 1600, Height: 1200}, + Format: imagerequest.FormatPNG, + }) + if err != nil { + t.Fatalf("SVG as PNG at 1600x1200: %v", err) + } + + img, err := vips.NewImageFromBuffer(output) + if err != nil { + t.Fatalf("decoding the PNG: %v", err) + } + + t.Cleanup(img.Close) + + for x, want := range map[int]float64{790: 0, 810: 255} { + point, err := img.GetPoint(x, 600) + if err != nil { + t.Fatalf("GetPoint(%d, 600): %v", x, err) + } + + t.Logf("pixel at %d,600: %v", x, point) + + if point[0] != want { + t.Errorf("pixel at %d,600 = %v, want %v", x, point[0], want) + } + } +} + +// TestImageProcessor_SVGFetchesNothing checks that rendering an SVG draws no +// image it refers to: one at a URL on a local server, by href or xlink:href, +// or a local file that exists. Each SVG renders without error, white where +// the image would be. An SVG that uses an external entity at that server is +// a separate case. No SVG makes a request to the server. +func TestImageProcessor_SVGFetchesNothing(t *testing.T) { + t.Parallel() + + var requests atomic.Int64 + + // Blue is 128 in every pixel of this PNG. + png := createTestPNG(t, 4, 4) + + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + t.Logf("request for %s", r.URL) + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(png) + })) + t.Cleanup(server.Close) + + pngPath := filepath.Join(t.TempDir(), "image.png") + + err := os.WriteFile(pngPath, png, 0o600) + if err != nil { + t.Fatalf("writing the PNG file: %v", err) + } + + for _, reference := range []string{ + `href="` + server.URL + `/image.png"`, + `xlink:href="` + server.URL + `/xlink.png"`, + `href="file://` + pngPath + `"`, + } { + svg := `` + + `` + + `` + + _, output, err := processSVG(t, svg, imagerequest.Transform{ + Format: imagerequest.FormatPNG, + }) + if err != nil { + t.Errorf("SVG with image %s as PNG: %v", reference, err) + + continue + } + + img, err := vips.NewImageFromBuffer(output) + if err != nil { + t.Fatalf("decoding the PNG: %v", err) + } + + point, err := img.GetPoint(8, 6) + img.Close() + + if err != nil { + t.Fatalf("GetPoint(8, 6): %v", err) + } + + t.Logf("SVG with image %s: pixel at 8,6 = %v", reference, point) + + if point[2] != 255 { + t.Errorf("SVG with image %s: blue at 8,6 = %v, want 255: the "+ + "image is drawn", reference, point[2]) + } + } + + svg := `` + + `]>` + + `` + + `&outside;` + + // librsvg loads no external entity, so it refuses this SVG: outside is + // not defined. + _, _, err = processSVG(t, svg, imagerequest.Transform{ + Format: imagerequest.FormatPNG, + }) + t.Logf("SVG with an external entity as PNG: error = %v", err) + + if requests.Load() != 0 { + t.Errorf("rendering the SVGs made %d requests, want none", + requests.Load()) + } +} + +// TestImageProcessor_RendersSVGLargerThanMaxInputDimension checks that an +// SVG whose own size is above MaxInputDimension is limited only by the +// output checks: it is rendered at a raster size, orig of it is the SVG +// itself, and a raster format at its own size is ErrSVGTooLargeForOrigSize. +// The 1000000x1000000 one is rendered at a ten-thousandth of its own size. +func TestImageProcessor_RendersSVGLargerThanMaxInputDimension(t *testing.T) { + t.Parallel() + + for _, svg := range []string{ + `` + + ``, + ``, + } { + result, _, err := processSVG(t, svg, imagerequest.Transform{ + Size: imagerequest.Size{Width: 100, Height: 100}, + Format: imagerequest.FormatPNG, + }) + if err != nil { + t.Errorf("%s as PNG at 100x100: %v", svg, err) + } else { + t.Logf("%s as PNG at 100x100: %dx%d", svg, result.Width, result.Height) + + if result.Width != 100 || result.Height != 100 { + t.Errorf("%s as PNG at 100x100 = %dx%d, want 100x100", svg, + result.Width, result.Height) + } + } + + result, output, err := processSVG(t, svg, + imagerequest.Transform{Format: imagerequest.FormatOriginal}) + if err != nil || result.ContentType != imagerequest.FormatSVG.MIMEType() || + string(output) != svg { + t.Errorf("orig of %s: error = %v, want the SVG itself", svg, err) + } + + _, _, err = processSVG(t, svg, + imagerequest.Transform{Format: imagerequest.FormatPNG}) + if !errors.Is(err, ErrSVGTooLargeForOrigSize) { + t.Errorf("%s as PNG at its own size: error = %v, want %v", svg, + err, ErrSVGTooLargeForOrigSize) + } + } +} + +// TestImageProcessor_RefusesSVGTooThin checks that an SVG with one side more +// than MaxInputDimension times the other is ErrSVGTooThin at a raster size, +// that one at exactly that proportion is rendered, and that orig of either +// is the SVG itself. +func TestImageProcessor_RefusesSVGTooThin(t *testing.T) { + t.Parallel() + + tooThin := `` + atTheLimit := `` + size := imagerequest.Size{Width: 100, Height: 100} + + _, _, err := processSVG(t, tooThin, + imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG}) + if !errors.Is(err, ErrSVGTooThin) { + t.Errorf("81930x10 SVG as PNG at 100x100: error = %v, want %v", err, + ErrSVGTooThin) + } + + result, _, err := processSVG(t, atTheLimit, + imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG}) + if err != nil { + t.Errorf("81920x10 SVG as PNG at 100x100: %v", err) + } else if result.Width != 100 || result.Height != 100 { + t.Errorf("81920x10 SVG as PNG at 100x100 = %dx%d, want 100x100", + result.Width, result.Height) + } + + for _, svg := range []string{tooThin, atTheLimit} { + result, output, err := processSVG(t, svg, + imagerequest.Transform{Format: imagerequest.FormatOriginal}) + if err != nil || result.ContentType != imagerequest.FormatSVG.MIMEType() || + string(output) != svg { + t.Errorf("orig of %s: error = %v, want the SVG itself", svg, err) + } + } +} + +// testSVGRenderTimeout is the time limit of the SVG time limit tests, short +// so that they finish in a few seconds. Under the race detector a child +// process takes about a second to start and read an SVG's size. +const testSVGRenderTimeout = 3 * time.Second + +// processSVGTimed has processor process svg, as an SVG source, with transform, +// and returns the error and how long it took. +func processSVGTimed( + t *testing.T, processor *ImageProcessor, svg string, + transform imagerequest.Transform, +) (time.Duration, error) { + t.Helper() + + start := time.Now() + _, err := processor.Process(t.Context(), strings.NewReader(svg), + imagerequest.FormatSVG, &transform) + + return time.Since(start), err +} + +// checkProcessesAnotherImage checks that processor, after refusing an SVG, +// processes a PNG. +func checkProcessesAnotherImage(t *testing.T, processor *ImageProcessor) { + t.Helper() + + _, err := processor.Process(t.Context(), + bytes.NewReader(createTestPNG(t, 64, 48)), imagerequest.FormatPNG, + &imagerequest.Transform{ + Size: imagerequest.Size{Width: 32, Height: 24}, + Format: imagerequest.FormatJPEG, + }) + if err != nil { + t.Errorf("a PNG after the SVG: %v", err) + } +} + +// TestImageProcessor_StopsSlowSVGRender checks that an SVG that takes about +// 12 seconds to render, 600 shapes each drawn with an feTurbulence filter +// effect, is ErrSVGRenderFailed soon after a time limit of +// testSVGRenderTimeout, not when the render would end. The processor then +// processes another image. +func TestImageProcessor_StopsSlowSVGRender(t *testing.T) { + t.Parallel() + + svg := `` + + `` + + `` + + strings.Repeat(``, + 600) + + `` + + processor := newSVGTestProcessor() + processor.svgRenderTimeout = testSVGRenderTimeout + + elapsed, err := processSVGTimed(t, processor, svg, imagerequest.Transform{ + Size: imagerequest.Size{Width: 256, Height: 256}, + Format: imagerequest.FormatPNG, + }) + t.Logf("slow SVG as PNG: error %v after %v", err, elapsed) + + if !errors.Is(err, ErrSVGRenderFailed) { + t.Errorf("slow SVG as PNG: error = %v, want %v", err, + ErrSVGRenderFailed) + } + + if elapsed > testSVGRenderTimeout+2*time.Second { + t.Errorf("slow SVG as PNG: refused after %v, want soon after %v", + elapsed, testSVGRenderTimeout) + } + + checkProcessesAnotherImage(t, processor) +} + +// TestImageProcessor_StopsSlowSVGSizeRead checks that orig of an SVG whose +// size takes about 10 seconds to read is ErrSVGRenderFailed soon after a time +// limit of testSVGRenderTimeout. To read the size, librsvg matches each of the +// SVG's 2500 CSS rules against each of its 2500 shapes, walking up the 75 +// groups around it. The processor then processes another image. +func TestImageProcessor_StopsSlowSVGSizeRead(t *testing.T) { + t.Parallel() + + var rules strings.Builder + + for i := range 2500 { + fmt.Fprintf(&rules, ".group%d rect { fill: red }", i) + } + + svg := `` + + `` + strings.Repeat(``, 75) + + strings.Repeat(``, 2500) + + strings.Repeat(``, 75) + `` + + processor := newSVGTestProcessor() + processor.svgRenderTimeout = testSVGRenderTimeout + + elapsed, err := processSVGTimed(t, processor, svg, + imagerequest.Transform{Format: imagerequest.FormatOriginal}) + t.Logf("orig of the SVG: error %v after %v", err, elapsed) + + if !errors.Is(err, ErrSVGRenderFailed) { + t.Errorf("orig of the SVG: error = %v, want %v", err, + ErrSVGRenderFailed) + } + + if elapsed > testSVGRenderTimeout+2*time.Second { + t.Errorf("orig of the SVG: refused after %v, want soon after %v", + elapsed, testSVGRenderTimeout) + } + + checkProcessesAnotherImage(t, processor) +} + +// TestImageProcessor_StopsSVGRenderOverMemoryLimit checks that an SVG that +// needs about 500 MiB more than a child process's own memory to render, a +// chain of 2000 blurs that each keep their result, is ErrSVGRenderFailed with +// a memory limit of 512 MiB, and that a plain SVG renders under the same +// limit. A child process needs about 130 MiB of its own, and about 350 MiB +// under the race detector, so the chain also renders under MaxSVGRenderMemory. +func TestImageProcessor_StopsSVGRenderOverMemoryLimit(t *testing.T) { + t.Parallel() + + var blurs strings.Builder + + blurs.WriteString(``) + + for i := 1; i < 2000; i++ { + fmt.Fprintf(&blurs, + ``, + i-1, i) + } + + svg := `` + + `` + + blurs.String() + `` + + `` + size := imagerequest.Size{Width: 256, Height: 256} + + processor := newSVGTestProcessor() + processor.maxSVGRenderMemory = 512 << 20 + + elapsed, err := processSVGTimed(t, processor, svg, + imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG}) + t.Logf("SVG of 2000 blurs as PNG: error %v after %v", err, elapsed) + + if !errors.Is(err, ErrSVGRenderFailed) { + t.Errorf("SVG of 2000 blurs as PNG: error = %v, want %v", err, + ErrSVGRenderFailed) + } + + _, err = processSVGTimed(t, processor, testSVG, + imagerequest.Transform{Size: size, Format: imagerequest.FormatPNG}) + if err != nil { + t.Errorf("plain SVG as PNG after it: %v", err) + } +} + +// TestSVGRenderSize checks the size at which an SVG is rendered to cover an +// output size, and that it is held to MaxInputDimension. +func TestSVGRenderSize(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + width, height int + outputWidth, outputHeight int + wantWidth, wantHeight int + }{ + {"own size", 16, 12, 16, 12, 16, 12}, + {"100 times larger", 16, 12, 1600, 1200, 1600, 1200}, + {"wider than the SVG's proportions", 16, 12, 160, 12, 160, 120}, + {"10 times smaller", 100, 100, 10, 10, 10, 10}, + {"held to MaxInputDimension", 8192, 1, 8192, 8192, 8192, 1}, + {"held to MaxInputDimension when enlarged", 1024, 1, 8192, 8192, 8192, 8}, + {"far above MaxInputDimension", 1000000, 1000000, 100, 100, 100, 100}, + } + + for _, tt := range tests { + gotWidth, gotHeight := svgRenderSize(tt.width, tt.height, + tt.outputWidth, tt.outputHeight) + if gotWidth != tt.wantWidth || gotHeight != tt.wantHeight { + t.Errorf("%s: svgRenderSize(%d, %d, %d, %d) = %dx%d, want %dx%d", + tt.name, tt.width, tt.height, tt.outputWidth, tt.outputHeight, + gotWidth, gotHeight, tt.wantWidth, tt.wantHeight) + } + } +} diff --git a/internal/imagerequest/imagerequest.go b/internal/imagerequest/imagerequest.go index 3ae3c6c..af73905 100644 --- a/internal/imagerequest/imagerequest.go +++ b/internal/imagerequest/imagerequest.go @@ -14,7 +14,8 @@ import ( var ErrInvalidFitMode = errors.New("invalid fit mode") // Format is an image format. pixa reads and writes the formats from JPEG to -// GIF below. In a request, orig and auto stand for one of them. +// GIF below. It reads SVG and writes it only as orig of an SVG source. In a +// request, orig and auto stand for another format. type Format string // Supported image formats. @@ -26,6 +27,7 @@ const ( FormatAVIF Format = "avif" FormatJXL Format = "jxl" FormatGIF Format = "gif" + FormatSVG Format = "svg" // FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each // request from its Accept header once the URL's signature or token has @@ -50,6 +52,8 @@ func (f Format) MIMEType() string { return "image/jxl" case FormatGIF: return "image/gif" + case FormatSVG: + return "image/svg+xml" case FormatOriginal, FormatAuto: return "" default: diff --git a/internal/imgcache/cache.go b/internal/imgcache/cache.go index e0e4748..0e03b5c 100644 --- a/internal/imgcache/cache.go +++ b/internal/imgcache/cache.go @@ -735,16 +735,18 @@ func (c *Cache) finishOpeningVariant( }, nil } -// readFormat reads the first bytes of a source or variant file, without -// moving its offset, and returns the format they show. Sources and variants -// are in the same six formats. +// readFormat reads up to magic.MaxMagicBytes first bytes of a source or +// variant file, without moving its offset, and returns the format they show. +// Sources and variants are in the same seven formats; a variant is SVG only +// as orig of an SVG source. A file shorter than magic.MinMagicBytes is +// io.EOF. func readFormat(content io.ReaderAt) (imagerequest.Format, error) { - firstBytes := make([]byte, magic.MinMagicBytes) + firstBytes := make([]byte, magic.MaxMagicBytes) - _, err := content.ReadAt(firstBytes, 0) - if err != nil { + n, err := content.ReadAt(firstBytes, 0) + if err != nil && (!errors.Is(err, io.EOF) || n < magic.MinMagicBytes) { return "", fmt.Errorf("failed to read the first bytes: %w", err) } - return magic.DetectFormat(firstBytes) + return magic.DetectFormat(firstBytes[:n]) } diff --git a/internal/imgcache/cache_hit_internal_test.go b/internal/imgcache/cache_hit_internal_test.go index 3d7edb2..9033315 100644 --- a/internal/imgcache/cache_hit_internal_test.go +++ b/internal/imgcache/cache_hit_internal_test.go @@ -98,8 +98,6 @@ func TestVariant_RefusesFileOfNoOutputFormat(t *testing.T) { wantErr error }{ {"text", []byte("this is text, not an image"), magic.ErrUnknownFormat}, - {"SVG", []byte(``), - magic.ErrUnknownFormat}, {"JPEG cut to 10 bytes", jpegImage[:10], io.EOF}, {"empty", nil, io.EOF}, } diff --git a/internal/imgcache/source_content_internal_test.go b/internal/imgcache/source_content_internal_test.go index d3b3d70..4e66530 100644 --- a/internal/imgcache/source_content_internal_test.go +++ b/internal/imgcache/source_content_internal_test.go @@ -216,8 +216,6 @@ func TestSource_RefusesFileOfNoSourceFormat(t *testing.T) { wantErr error }{ {"text", []byte("this is text, not an image"), magic.ErrUnknownFormat}, - {"SVG", []byte(``), - magic.ErrUnknownFormat}, {"JPEG cut to 10 bytes", jpegImage[:10], io.EOF}, {"empty file", nil, io.EOF}, } diff --git a/internal/imgcache/svg_internal_test.go b/internal/imgcache/svg_internal_test.go new file mode 100644 index 0000000..1c0263f --- /dev/null +++ b/internal/imgcache/svg_internal_test.go @@ -0,0 +1,54 @@ +package imgcache + +import ( + "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// testSVG is an SVG with a comment before its root element, so that its +// first magic.MinMagicBytes bytes do not show its format. +const testSVG = `` + +// TestSourceAndVariant_ReadSVG checks that a cached SVG source is read as +// SVG, and a cached SVG variant (orig of an SVG source) as image/svg+xml. +func TestSourceAndVariant_ReadSVG(t *testing.T) { + t.Parallel() + + cache, _ := newEvictionTestCache(t, 1<<30) + + storeEvictionTestSource(t, cache, testHostCDN, "/drawing.svg", []byte(testSVG)) + + reader, _, format, err := cache.Source(t.Context(), + imagerequest.Source{Host: testHostCDN, Path: "/drawing.svg"}) + t.Logf("Source() format = %q, error = %v", format, err) + + if err != nil || format != imagerequest.FormatSVG { + t.Errorf("Source() = format %q, error %v; want %q", format, err, + imagerequest.FormatSVG) + } + + if reader != nil { + _ = reader.Close() + } + + key := imagerequest.VariantKey("aabbccdd3001") + + err = cache.variants.Write(string(key), []byte(testSVG)) + if err != nil { + t.Fatalf("storing the variant file: %v", err) + } + + variant, err := cache.Variant(key) + t.Logf("Variant() content type = %q, error = %v", variant.ContentType, err) + + if err != nil || variant.ContentType != "image/svg+xml" { + t.Errorf("Variant() = content type %q, error %v; want image/svg+xml", + variant.ContentType, err) + } + + if variant.Content != nil { + _ = variant.Content.Close() + } +} diff --git a/internal/magic/magic.go b/internal/magic/magic.go index 4d069ba..1e4de2c 100644 --- a/internal/magic/magic.go +++ b/internal/magic/magic.go @@ -20,6 +20,10 @@ var ( // MinMagicBytes is the minimum number of bytes needed to detect format. const MinMagicBytes = 12 +// MaxMagicBytes is the most bytes DetectFormat looks at. An SVG's root +// element must start within them, as libvips requires. +const MaxMagicBytes = 1024 + // Magic byte signatures for supported formats. // These are effectively constants but Go doesn't support const slices. // @@ -39,6 +43,12 @@ var ( } ) +// utf8BOM is the UTF-8 byte order mark, which a text file such as an SVG may +// start with. +// +//nolint:gochecknoglobals // immutable lookup data +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + // WebP identifier appears at offset 8 after RIFF header. // //nolint:gochecknoglobals // immutable lookup data @@ -53,7 +63,7 @@ var ( ) // DetectFormat detects the image format from magic bytes. Data in none of -// the formats pixa reads, SVG included, is ErrUnknownFormat. +// the formats pixa reads is ErrUnknownFormat. func DetectFormat(data []byte) (imagerequest.Format, error) { if len(data) < MinMagicBytes { return "", ErrNotEnoughData @@ -96,9 +106,25 @@ func DetectFormat(data []byte) (imagerequest.Format, error) { return imagerequest.FormatJXL, nil } + if isSVG(data) { + return imagerequest.FormatSVG, nil + } + return "", ErrUnknownFormat } +// isSVG reports whether data starts as an SVG does: with a "<", after an +// optional UTF-8 byte order mark and white space, and with "`), - wantErr: ErrUnknownFormat, + name: "SVG", + data: []byte(``), + wantFormat: imagerequest.FormatSVG, }, { name: "unknown format", diff --git a/internal/magic/svg_internal_test.go b/internal/magic/svg_internal_test.go new file mode 100644 index 0000000..fdbf0e3 --- /dev/null +++ b/internal/magic/svg_internal_test.go @@ -0,0 +1,85 @@ +package magic + +import ( + "errors" + "strings" + "testing" + + "sneak.berlin/go/pixa/internal/imagerequest" +) + +// TestDetectFormat_SVG checks that DetectFormat names SVG text with what may +// come before the root element, and does not name as SVG text that does not +// start with "<" or has no "` + + tests := []struct { + name string + data string + wantErr error + }{ + {name: "root element only", data: root}, + {name: "XML declaration", data: `` + "\n" + root}, + {name: "byte order mark and white space", data: "\xEF\xBB\xBF \n\t" + root}, + {name: "comment", data: "" + root}, + { + name: "DOCTYPE", + data: `` + root, + }, + { + name: "root element after the first bytes", + data: "" + root, + wantErr: ErrUnknownFormat, + }, + {name: "text naming ", wantErr: ErrUnknownFormat}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := DetectFormat([]byte(tt.data)) + t.Logf("DetectFormat() = %q, %v", got, err) + + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Errorf("DetectFormat() error = %v, want %v", err, tt.wantErr) + } + + return + } + + if err != nil || got != imagerequest.FormatSVG { + t.Errorf("DetectFormat() = %q, %v; want %q", got, err, + imagerequest.FormatSVG) + } + }) + } +} + +// TestValidateMagicBytes_SVG checks that SVG data is SVG for the +// Content-Type image/svg+xml, with a charset too, and a mismatch for another. +func TestValidateMagicBytes_SVG(t *testing.T) { + t.Parallel() + + data := []byte(``) + + for _, declared := range []string{"image/svg+xml", "image/svg+xml; charset=utf-8"} { + got, err := ValidateMagicBytes(data, declared) + if err != nil || got != imagerequest.FormatSVG { + t.Errorf("ValidateMagicBytes(%q) = %q, %v; want %q", declared, got, + err, imagerequest.FormatSVG) + } + } + + _, err := ValidateMagicBytes(data, "image/png") + if !errors.Is(err, ErrMagicByteMismatch) { + t.Errorf("ValidateMagicBytes(image/png) error = %v, want %v", err, + ErrMagicByteMismatch) + } +} diff --git a/internal/server/svg_internal_test.go b/internal/server/svg_internal_test.go index 95b46fa..c623b84 100644 --- a/internal/server/svg_internal_test.go +++ b/internal/server/svg_internal_test.go @@ -4,38 +4,134 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "sync/atomic" "testing" + + "sneak.berlin/go/pixa/internal/imageprocessor" + "sneak.berlin/go/pixa/internal/imageproxy" + "sneak.berlin/go/pixa/internal/imagerequest" + "sneak.berlin/go/pixa/internal/magic" + "sneak.berlin/go/pixa/internal/middleware" ) -// TestImageProxyRefusesSVG requests an image the upstream answers as SVG, -// in its own format (orig) and resized as PNG, through pixa's router, -// upstream fetcher and image processor. pixa does not accept SVG as a -// source, so each request is answered 415, never with an image. -func TestImageProxyRefusesSVG(t *testing.T) { - t.Parallel() +// TestMain runs this test binary as an SVG child process when the image +// processor starts it again as one, as the SVG tests below make it do. +func TestMain(m *testing.M) { + if imageprocessor.IsSVGChildProcess(os.Args[1:]) { + os.Exit(imageprocessor.RunAsSVGChildProcess(os.Args[1:])) + } + + os.Exit(m.Run()) +} + +// testSVG is the SVG the upstream serves in the SVG tests below. +const testSVG = `` + + `` + +// startSVGUpstream starts an upstream that answers every request with +// testSVG as image/svg+xml, and returns it with its count of requests. +func startSVGUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + + var requests atomic.Int64 upstream := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) w.Header().Set("Content-Type", "image/svg+xml") - _, _ = io.WriteString(w, ``) + _, _ = io.WriteString(w, testSVG) })) t.Cleanup(upstream.Close) + return upstream, &requests +} + +// TestImageProxyServesSVGAsItIs requests an SVG source in its own format +// (orig) at the orig size through pixa's router, upstream fetcher, image +// processor and cache, twice. Both answers are the SVG itself, never another +// format, the second from the cache; the upstream is asked once. Both carry +// the Content-Security-Policy of an SVG response in place of the +// middleware's. orig at another size is refused with 400. +func TestImageProxyServesSVGAsItIs(t *testing.T) { + t.Parallel() + + const wantPolicy = "default-src 'none'; style-src 'unsafe-inline'; " + + "img-src data:; frame-ancestors 'none'; sandbox" + + upstream, requests := startSVGUpstream(t) + s, _, _ := startImageProxy(t, upstream) + target := "/v1/image/" + upstreamHost + "/drawing.svg/orig.orig" + + for _, wantCache := range []imageproxy.CacheStatus{ + imageproxy.CacheMiss, imageproxy.CacheHit, + } { + rec := getImage(t, s, target) + + if rec.Header().Get("Content-Type") != "image/svg+xml" || + rec.Body.String() != testSVG { + t.Errorf("GET %s: %s %q, want image/svg+xml %q", target, + rec.Header().Get("Content-Type"), rec.Body.String(), testSVG) + } + + if got := rec.Header().Get("X-Pixa-Cache"); got != string(wantCache) { + t.Errorf("GET %s: X-Pixa-Cache = %q, want %q", target, got, wantCache) + } + + policy := rec.Header().Get("Content-Security-Policy") + if policy != wantPolicy { + t.Errorf("GET %s: Content-Security-Policy = %q, want %q", target, + policy, wantPolicy) + } + } + + if requests.Load() != 1 { + t.Errorf("upstream requests = %d, want 1", requests.Load()) + } + + resized := "/v1/image/" + upstreamHost + "/drawing.svg/32x24.orig" + + rec := httptest.NewRecorder() + s.ServeHTTP(rec, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, resized, nil)) + t.Logf("GET %s: %d %s", resized, rec.Code, rec.Body.String()) + + if rec.Code != http.StatusBadRequest { + t.Errorf("GET %s: %d %s, want 400", resized, rec.Code, rec.Body.String()) + } +} + +// TestImageProxyRendersSVG requests an SVG source as each raster format +// through pixa's router, upstream fetcher and image processor. Each answer +// is an image in that format, with the middleware's Content-Security-Policy. +func TestImageProxyRendersSVG(t *testing.T) { + t.Parallel() + + upstream, _ := startSVGUpstream(t) s, _, _ := startImageProxy(t, upstream) - for _, sizeFormat := range []string{"64x48.orig", "32x24.png"} { - target := "/v1/image/" + upstreamHost + "/drawing.svg/" + sizeFormat + for _, format := range []imagerequest.Format{ + imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, + imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL, + } { + target := "/v1/image/" + upstreamHost + "/drawing.svg/32x24." + + string(format) - rec := httptest.NewRecorder() - s.ServeHTTP(rec, httptest.NewRequestWithContext( - t.Context(), http.MethodGet, target, nil)) - t.Logf("GET %s: %d, Content-Type %s, body %s", - target, rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) + rec := getImage(t, s, target) + detected, err := magic.DetectFormat(rec.Body.Bytes()) + t.Logf("GET %s: %s, body in %q (%v)", target, + rec.Header().Get("Content-Type"), detected, err) - if rec.Code != http.StatusUnsupportedMediaType { - t.Errorf("GET %s status = %d, want %d", - target, rec.Code, http.StatusUnsupportedMediaType) + if rec.Header().Get("Content-Type") != format.MIMEType() || + err != nil || detected != format { + t.Errorf("GET %s: %s, body in %q (%v); want %s", target, + rec.Header().Get("Content-Type"), detected, err, format.MIMEType()) + } + + policy := rec.Header().Get("Content-Security-Policy") + if policy != middleware.ContentSecurityPolicyValue { + t.Errorf("GET %s: Content-Security-Policy = %q, want %q", target, + policy, middleware.ContentSecurityPolicyValue) } } }