2 Commits
Author SHA1 Message Date
sneak 73f4cf03c1 Serve a cached variant while its source URL is negative-cached (closes #245)
check / check (push) Canceled after 0s
Get checked the negative cache before it looked for the variant on
disk, so after one failed fetch of a source every cached variant of it
answered 502 for 5 minutes. Get now serves a cached variant first and
checks the negative cache only on a miss, before processing. The 502
line under /v1/image/ in README.md and TODO.md say so.

Model: opus-5-5
2026-10-15 05:33:05 +00:00
sneak 2c8ac2b7c7 Test that a cached variant is served while its URL is negative-cached
check / check (push) Canceled after 0s
Get refuses a cached variant with the negative-cache error while its
source URL has a negative-cache row. The new test caches a variant,
stores a row for its URL, and expects the variant to be served from the
cache, and a variant not in the cache to be refused as before. It fails
until Get checks the negative cache only on a miss
(#245).

Model: opus-5-5
2026-10-15 04:56:23 +00:00
180 changed files with 11840 additions and 16207 deletions
+9 -18
View File
@@ -13,21 +13,12 @@
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and # `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
# deletes the package directory from the context. # deletes the package directory from the context.
# .git is sent without its config. Without a VERSION build argument the # Unlike the standard file, which leaves out all of .git, pixa sends
# stage that compiles runs `git describe --tags --always` on .git, which # .git without its config. Without a VERSION build argument the stage
# does not need .git/config; that file can hold a credential, such as a # that compiles runs `git describe --tags --always` on .git, which does
# not need .git/config; that file can hold a credential, such as a
# password in a remote URL or the token the CI checkout step stores there. # password in a remote URL or the token the CI checkout step stores there.
# Each submodule keeps a config with the same exposure in its git directory .git/config
# under .git/modules/, nested again for a submodule's own submodules, or in
# its own .git directory when it keeps one.
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
# `deploy/config`, `config/lib`) loses its whole git directory, because
# `**/.git/modules/**/config` also matches that segment's directory
# under .git/modules/. Go's version stamping then fails the build;
# nothing leaks. Name such a submodule without that segment:
# `git submodule add --name`.
**/.git/config
**/.git/modules/**/config
# Agent scratch: one full checkout of the repo per in-flight agent. # Agent scratch: one full checkout of the repo per in-flight agent.
# Anchored because it occurs once where agents run at the repo root. # Anchored because it occurs once where agents run at the repo root.
@@ -51,9 +42,7 @@
**/[iI][dD]_[rR][sS][aA] **/[iI][dD]_[rR][sS][aA]
**/[iI][dD]_[dD][sS][aA] **/[iI][dD]_[dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA] **/[iI][dD]_[eE][cC][dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
**/[iI][dD]_[eE][dD]25519 **/[iI][dD]_[eE][dD]25519
**/[iI][dD]_[eE][dD]25519_[sS][kK]
# Dependencies: restored inside the image, never copied in. # Dependencies: restored inside the image, never copied in.
**/node_modules **/node_modules
@@ -71,8 +60,10 @@
**/.vscode **/.vscode
**/*.sublime-* **/*.sublime-*
# pixa's own entries. On the host, `make build` writes bin/pixad, and the # pixa's own entries. Nothing in the build reads .gitignore. On the
# example config keeps its state directory in data/. # host, `make build` writes bin/pixad, and the example config keeps its
# state directory in data/.
.gitignore
/bin /bin
/data /data
-3
View File
@@ -13,6 +13,3 @@ indent_style = tab
[*.go] [*.go]
indent_style = tab indent_style = tab
# This repository's own sections, such as one for another language it
# uses, go below this comment, and a re-vendor keeps them.
+4 -9
View File
@@ -1,20 +1,15 @@
name: check name: check
on: [push] on: [push]
# Free the shared runner: a new push cancels only the same branch's older run.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs: jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Free the shared runner from a hung build.
timeout-minutes: 20
steps: steps:
# actions/checkout v4.2.2, 2026-02-22 # actions/checkout v4.2.2, 2026-02-22
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
# script/cibuild needs no token, so none is left in .git/config. # The default clone is shallow and has no tags, so the
# version the build takes from `git describe` would be a
# bare commit; this fetches the whole history with its tags.
with: with:
persist-credentials: false
# All history and tags, so git describe finds the version tag.
fetch-depth: 0 fetch-depth: 0
- run: script/cibuild - run: script/cibuild
- run: script/docker-smoke
+6 -37
View File
@@ -17,46 +17,15 @@ Thumbs.db
# entry and must not be given a `**/` prefix on the way into one. # entry and must not be given a `**/` prefix on the way into one.
.claude/ .claude/
# Node # Environment / secrets
node_modules/ .env
.env.*
# Secrets. Unanchored like every entry above, so each matches at every *.pem
# depth. Matching is case-sensitive on Linux, so names use character *.key
# ranges rather than a lowercase form that misses `Server.Key`.
# Environment files. `*.env` covers bare `.env` and the `prod.env`
# convention. Only the templates `example.env` and `sample.env` are
# re-included below. A repository that commits any other template adds
# its own negation at the end of this file, for example `!.env.example`.
*.[eE][nN][vV]
.[eE][nN][vV].*
.[eE][nN][vV][rR][cC]
!example.env
!sample.env
# Private keys and the bundles carrying them.
*.[pP][eE][mM]
*.[kK][eE][yY]
*.[pP]12
*.[pP][fF][xX]
[iI][dD]_[rR][sS][aA]
[iI][dD]_[dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
[iI][dD]_[eE][dD]25519
[iI][dD]_[eE][dD]25519_[sS][kK]
# This repository's own entries, such as its build outputs, go below
# this comment, and a re-vendor keeps them. Anchor a binary built at the
# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
# Go
*.log
*.out
*.test
# Dependencies # Dependencies
vendor/ vendor/
node_modules/
# Build output # Build output
/bin/ /bin/
-3
View File
@@ -17,7 +17,6 @@ linters:
disable: disable:
# Genuinely incompatible with project patterns # Genuinely incompatible with project patterns
- exhaustruct # Requires all struct fields - exhaustruct # Requires all struct fields
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
- godot # Requires comments to end with periods - godot # Requires comments to end with periods
- wrapcheck # Too verbose for internal packages - wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go - varnamelen # Short names like db, id are idiomatic Go
@@ -25,8 +24,6 @@ linters:
# silenced by disabling that name, not by enabling the successor. # silenced by disabling that name, not by enabling the successor.
- wsl # Deprecated, replaced by wsl_v5 - wsl # Deprecated, replaced by wsl_v5
- gomodguard # Deprecated, replaced by gomodguard_v2 - gomodguard # Deprecated, replaced by gomodguard_v2
# Misses findings at random in v2.14.0; back once a pinned release fixes it
- canonicalheader
settings: settings:
lll: lll:
line-length: 88 line-length: 88
+5
View File
@@ -1,2 +1,7 @@
node_modules/ node_modules/
yarn.lock yarn.lock
# A byte-for-byte copy of the one in sneak/prompts.
REPO_POLICIES.md
vendor/
-76
View File
@@ -1,76 +0,0 @@
# Repository Rules
Last Updated 2026-10-27
These rules MUST be followed at all times. This is very important.
- Never use `git add -A`. Stage specific changes for a deliberate commit. A
commit holds one change. After each change, make a commit with a good one-line
summary.
- NEVER modify the linter config without asking first.
- NEVER modify a test to exclude special cases or otherwise make it pass without
asking first. In almost all cases, change the code, NOT the test. You need
explicit user approval to modify an existing test. If you think a test needs
to change, make your case and ask for permission to proceed. Then stop. A bug
fix may change an existing test without asking only when that test pins
exactly the behavior the filed bug calls wrong. The change must then move the
test to the fixed behavior and nothing else. Any other change to an existing
test needs explicit user approval. Writing NEW tests needs no approval.
- When linting, assume the linter config is CORRECT. Treat each item the linter
reports as something that legitimately needs fixing in the code.
- When running tests, use `make test`.
- Before committing, run `make check`. It runs `make test`, `make lint` and
`make fmt-check`. Any issue it finds MUST be resolved before committing,
unless you are explicitly told otherwise.
- When fixing a bug, write a failing test for the bug FIRST. Add appropriate
logging to the test, to make sure the test is written correctly. Commit that.
Then work on the fix until the test passes, without changing the test further.
Then commit the fix.
- When adding a new feature, do the same: write a test first (test-driven
development, TDD). The test does not need to be very complex. Commit the test,
then commit the feature.
- When adding a new feature, use a feature branch. The feature branch reaches
`next` through a PR that passes review. It is merged only when the feature is
completely finished and the code is up to standards. The code is up to
standards when it passes `make check`. The branch is deleted after the merge.
Only the user merges `next` into `main`.
- Write `godoc` comments for all exported types and functions as you go along.
- ALWAYS be consistent in naming. If you name something one way in one place,
use the EXACT SAME name in every other place.
- Be descriptive and specific in naming. `al` is bad; `SourceHostAllowlist` is
good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good.
- This is not prototype or teaching code. It is designed for production. Any
security issue (such as denial of service) or other web vulnerability is a P1
bug. It must be added at the top of `TODO.md`.
- Do not stub an implementation unless you are specifically instructed to. This
is production code, and we need working implementations.
- NEVER silently fall back to a different setting when a user's parameter
explicitly specifies a value. For example, if a user requests `format=webp`
and WebP encoding is not supported, return an error. Do NOT silently output
PNG instead. If a user specifies `fit=invalid` and that fit mode does not
exist, return an error. Do NOT silently default to `cover`. A silent fallback
violates the principle of least surprise and masks bugs. Defaults are
acceptable only for OMITTED parameters, never for INVALID explicit values.
- Avoid vendoring dependencies unless specifically instructed to. NEVER commit
the `vendor/` directory. NEVER commit compiled binaries. If such a directory
or file exists and `.gitignore` does not list it, add it to `.gitignore`.
Commit that `.gitignore` change. Keep the entire git repository, history
included, under 20 MiB. Go over 20 MiB only when you specifically must commit
a large file, such as a media file used as a test fixture. Only OUR source
code and its immediately supporting files, such as test examples, go into the
repository and its history.
+71
View File
@@ -0,0 +1,71 @@
# Repository Rules
Last Updated 2026-01-08
These rules MUST be followed at all times, it is very important.
- Never use `git add -A` - add specific changes to a deliberate commit. A commit
should contain one change. After each change, make a commit with a good
one-line summary.
- NEVER modify the linter config without asking first.
- NEVER modify tests to exclude special cases or otherwise get them to pass
without asking first. In almost all cases, the code should be changed, NOT the
tests. If you think the test needs to be changed, make your case for that and
ask for permission to proceed, then stop. You need explicit user approval to
modify existing tests. (You do not need user approval for writing NEW tests.)
- When linting, assume the linter config is CORRECT, and that each item output
by the linter is something that legitimately needs fixing in the code.
- When running tests, use `make test`.
- Before commits, run `make check`. This runs `make lint` and `make test` and
`make check-fmt`. Any issues discovered MUST be resolved before committing
unless explicitly told otherwise.
- When fixing a bug, write a failing test for the bug FIRST. Add appropriate
logging to the test to ensure it is written correctly. Commit that. Then go
about fixing the bug until the test passes (without modifying the test
further). Then commit that.
- When adding a new feature, do the same - implement a test first (TDD). It
doesn't have to be super complex. Commit the test, then commit the feature.
- When adding a new feature, use a feature branch. When the feature is
completely finished and the code is up to standards (passes `make check`) then
and only then can the feature branch be merged into `main` and the branch
deleted.
- Write godoc documentation comments for all exported types and functions as you
go along.
- ALWAYS be consistent in naming. If you name something one thing in one place,
name it the EXACT SAME THING in another place.
- Be descriptive and specific in naming. `wl` is bad; `SourceHostWhitelist` is
good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good.
- This is not prototype or teaching code - this is designed for production. Any
security issues (such as denial of service) or other web vulnerabilities are
P1 bugs and must be added to TODO.md at the top.
- As this is production code, no stubbing of implementations unless specifically
instructed. We need working implementations.
- NEVER silently fall back to a different setting when a user's parameter
explicitly specifies a value. If a user requests format=webp and WebP encoding
is not supported, return an error - do NOT silently output PNG instead. If a
user specifies fit=invalid and that fit mode doesn't exist, return an error -
do NOT silently default to "cover". Silent fallbacks violate the principle of
least surprise and mask bugs. The only acceptable defaults are for OMITTED
parameters, never for INVALID explicit values.
- Avoid vendoring deps unless specifically instructed to. NEVER commit the
vendor directory, NEVER commit compiled binaries. If these directories or
files exist, add them to .gitignore (and commit the .gitignore) if they are
not already in there. Keep the entire git repository (with history) small -
under 20MiB, unless you specifically must commit larger files (e.g. test
fixture example media files). Only OUR source code and immediately supporting
files (such as test examples) goes into the repo/history.
+41 -68
View File
@@ -1,16 +1,7 @@
# In every stage ARG CACHEBUST comes right after the steps that install
# system packages and download Go's modules, or right after FROM in a
# stage with neither, and every step that lints, tests or compiles
# comes after it. script/ passes a new value on every build, which
# reruns every RUN step after it. Docker keeps the layers above it
# until the base image, their own lines or go.mod and go.sum change.
# The steps that compile mount Go's build cache, and golangci-lint run
# also mounts golangci-lint's; both survive from build to build.
# Lint phase. script/lint builds it alone. The linter is run directly: # Lint phase. script/lint builds it alone. The linter is run directly:
# `make lint` and script/lint are themselves a docker build. # `make lint` and script/lint are themselves a docker build.
# golangci/golangci-lint:v2.14.0, 2026-10-15 # golangci/golangci-lint:v2.12.2, 2026-10-04
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
# The linter compiles every package, and govips needs the libvips # The linter compiles every package, and govips needs the libvips
# headers for that. REPO_POLICIES.md has the lint phase install them # headers for that. REPO_POLICIES.md has the lint phase install them
@@ -22,87 +13,70 @@ RUN apt-get update \
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN golangci-lint run --config .golangci.yml ./...
--mount=type=cache,target=/root/.cache/golangci-lint \
golangci-lint run --config .golangci.yml ./...
# Test phase. script/test builds it alone. -race needs cgo and so a C # Test phase. script/test builds it alone.
# compiler, which the Debian Go image ships and the alpine one does not. # golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
# golang:1.25.4-trixie, 2026-10-15 FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test
FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test
# govips compiles against the libvips and libheif headers, and the tests
# need libvips' JPEG XL support; libvips-dev brings all three. The tests
# also encode AVIF, which needs libheif's AVIF encoder,
# libheif-plugin-aomenc: Debian only recommends it, so it is named here.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev libheif-plugin-aomenc \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /src WORKDIR /src
# script/bootstrap --cgo installs the build dependencies (a C compiler,
# the libvips and libheif headers, and libvips' JPEG XL support, which
# the tests need) and downloads the Go modules.
COPY script/ ./script/
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN script/bootstrap --cgo
ARG CACHEBUST
COPY . . COPY . .
# Without -v first; on a failure, again with -v for the details, and # Without -v first; on a failure, again with -v for the details, and
# the step fails even if the second run passes. -count=1 because the # the step fails even if the second run passes. -parallel 4: by default
# mounted cache also holds test results, which go test would otherwise # a package runs as many of its tests at once as the host has CPUs, and
# replay. -parallel 4: by default a package runs as many of its tests at # on a busy host they then wait so long to be scheduled that a test
# once as the host has CPUs, and on a busy host they then wait so long # that times a request can see it return a second late.
# to be scheduled that a test that times a request can see it return a RUN go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \
# second late.
RUN --mount=type=cache,target=/root/.cache/go-build \
go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; } go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; }
# Build stage. Nothing is wanted from the two phases above: these copies # Build stage. Nothing is wanted from the two phases above: these copies
# make BuildKit build them first, so nothing after them runs unless lint # make BuildKit build them first, so this stage runs only when lint and
# and test passed. # test passed.
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too # golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
# git for the version, and for govips a C compiler, pkg-config and the
# libvips and libheif headers
RUN apk add --no-cache git build-base pkgconfig vips-dev libheif-dev
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null
# A tar-stream context keeps the sender's file owners, which git refuses. WORKDIR /src
RUN git config --system --add safe.directory /src
# Build dependencies and Go modules, as in the test phase
COPY script/ ./script/
COPY go.mod go.sum ./
RUN script/bootstrap --cgo
# Copy source code # Copy source code
COPY . . COPY . .
# The VERSION build arg when one is given, otherwise # VERSION is declared here, not earlier: a new value reruns only the
# `git describe --tags --always` on the .git in the build context: the # build, not script/bootstrap. Given none, the version is
# tag on a tagged commit, tag-N-gHASH after one, the short commit when # `git describe --tags --always` of the .git in the build context (git
# no tag is reachable. With .git present, a version that is still empty, # comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH
# dev or unknown fails the build: git is missing or could not read the # after one, the short commit when no tag is reachable. A context that
# checkout. A context without .git, as from a source tarball, stamps an # carries .git and still yields no version fails the build; one without
# empty version. CGO stays enabled for govips; -trimpath keeps build # .git, as from a source tarball, stamps an empty version. CGO stays
# paths out of the binary, and -s -w leave out the symbol table and # enabled for govips; -trimpath keeps build paths out of the binary, and
# debug information. # -s -w leave out the symbol table and debug information.
ARG VERSION ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN version="${VERSION:-$(git describe --tags --always)}"; \
VERSION="${VERSION:-$(git describe --tags --always)}"; \ if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
if [ -e .git ]; then \ [ "$version" = unknown ]; }; then \
case "$VERSION" in ""|dev|unknown) \ echo "the build context carries .git but yields no version" >&2; \
echo "version is '$VERSION' although .git is present" >&2; \ exit 1; \
exit 1 ;; \
esac; \
fi; \ fi; \
CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \ CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \ -ldflags "-s -w -X main.Version=${version}" \
-o /pixad ./cmd/pixad -o /pixad ./cmd/pixad
# Runtime stage, and the last one: a plain `docker build .` builds this # Runtime stage, and the last one: a plain `docker build .` builds this
@@ -121,7 +95,6 @@ RUN apk add --no-cache \
ca-certificates \ ca-certificates \
tzdata \ tzdata \
su-exec su-exec
ARG CACHEBUST
# Copy binary from builder # Copy binary from builder
COPY --from=builder /pixad /usr/local/bin/pixad COPY --from=builder /pixad /usr/local/bin/pixad
+1 -7
View File
@@ -1,4 +1,4 @@
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest bench .PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
LDFLAGS := -X main.Version=$(VERSION) LDFLAGS := -X main.Version=$(VERSION)
@@ -64,12 +64,6 @@ docker-smoke:
loadtest: loadtest:
@script/loadtest @script/loadtest
# Serve cache hits in parallel through the image handler and report hits per
# second (needs Go on the host after `script/bootstrap --cgo`, not Docker; a
# benchmark, not part of check)
bench:
@script/bench
# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION) # Build Docker image as `make docker` does, and also tag it pixa:$(VERSION)
docker-versioned: docker-versioned:
@script/docker @script/docker
+151 -526
View File
@@ -71,12 +71,12 @@ The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the
database and the disk cache: database and the disk cache:
- It needs a persistent volume: without one, every restart starts with an empty - It needs a persistent volume: without one, every restart starts with an empty
cache. Losing the directory also ends every login session (see Encrypted cache. In the container, the startup script gives the directory to the user
URLs). In the container, the startup script gives the directory to the user
pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user pixa runs as (uid 65532) and sets its mode to `750`; outside it, that user
must be able to write the directory. must be able to write the directory.
- `cache_max_bytes` limits the source and transformed images together. The - `cache_max_bytes` limits the source and transformed images together. The
database and files still being written come on top, and eviction runs in the database, the metadata files, the `.meta` file beside each transformed image
and files still being written come on top, and eviction runs in the
background, so the cache can pass the limit for a while: leave room on the background, so the cache can pass the limit for a while: leave room on the
volume beyond it. volume beyond it.
- Set `cache_max_bytes` for a lasting deployment. Its default, worked out each - Set `cache_max_bytes` for a lasting deployment. Its default, worked out each
@@ -88,25 +88,21 @@ A load balancer's health check can request `/.well-known/healthcheck.json`,
which answers 200 whenever pixa is running, in maintenance mode too (see which answers 200 whenever pixa is running, in maintenance mode too (see
`maintenance_mode`). `maintenance_mode`).
On SIGTERM or SIGINT pixa stops accepting connections. The requests in progress On SIGTERM or SIGINT pixa stops accepting connections, gives the requests in
and the images being processed get 5 seconds to finish. A request not finished progress and the images being processed 5 seconds to finish, then writes to the
by then is cut off. pixa then writes the waiting last-used times of cached database the counts of cache hits, misses, fetches and conversions that requests
images to the database (see Storage) and exits. It exits with 1 when: could not write by their deadline, and exits: with 0, or with 1 when images were
still being processed after those 5 seconds, some of those counts could not be
- images were still being processed after those 5 seconds, written, or another part of pixa failed to stop. A request not finished by then
- some of those times could not be written, or is cut off. `docker stop` waits 10 seconds before it kills the container.
- another part of pixa failed to stop.
Otherwise it exits with 0. `docker stop` waits 10 seconds before it kills the
container.
Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as
it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL
support, which on Alpine is the `vips-jxl` package and which the nix and brew support, which on Alpine is the `vips-jxl` package and which the nix and brew
packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04 packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
on. Building pixa also needs the development files of libvips and libheif, on. Building pixa also needs the development files of libvips and libheif,
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these; `pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
the `Dockerfile` installs its own packages. Plain `script/bootstrap`, which as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
`script/setup` and `script/cibuild` run, installs git, make and Go, and Node, `script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but
none of the C libraries: the checks compile pixa in Docker. Docker itself must none of the C libraries: the checks compile pixa in Docker. Docker itself must
@@ -127,41 +123,9 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
comma-separated; default none. Every image on a listed host is served to comma-separated; default none. Every image on a listed host is served to
anyone without a signature, so list only hosts you control, where nobody anyone without a signature, so list only hosts you control, where nobody
else can put images. else can put images.
- `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it. Set - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it;
it: the volume is a directory on the host's filesystem, shared with the default 75% of (free space + what the cache holds)
host and every other app. The default is 75% of the sum of that - the rest are in the table under Configuration below
filesystem's free space and what the cache holds, and at least 500 MiB, so
the cache can grow to three quarters or more of the space they all have
left, and to all of it when that is 500 MiB or less.
- the rest are in the list under Configuration below
- **Memory:** pixa can use up to `max_concurrent_processing` times 4.2 GiB for
the images it processes at once, plus `upstream_connections` times
`upstream_max_response_size` for the source images it has fetched that wait
for processing. 4.2 GiB is the most resident memory was measured to grow while
pixa, with pixad's libvips settings, processed one 8192x8192 source, the
largest it accepts, saving it at full size to each output format in turn, one
at a time. The sources were a JPEG, a GIF, a WebP, an 8-bit and a 16-bit PNG,
an 8-bit and a 12-bit AVIF, and an 8-bit, a 16-bit and a 32-bit floating-point
JPEG XL; memory grew most for the 16-bit JPEG XL, of random pixels, saved as
WebP. At the defaults that is half the CPUs pixa sees times 4.2 GiB, plus 64
times 50 MiB: on a host where pixa sees 48 CPUs, 24 times 4.2 GiB plus 3.1
GiB, about 104 GiB. Set `PIXA_MAX_CONCURRENT_PROCESSING` and
`PIXA_UPSTREAM_CONNECTIONS` so that this fits the memory the container may
use.
- **Temporary files:** while pixa processes a large source, libvips writes its
decoded pixels to a file in `/tmp`. The file disappears from `/tmp` once it is
written, so a listing of `/tmp` does not show it, but it keeps its space, on
disk or, where `/tmp` is a tmpfs, in memory, until the image is done. For a
GIF, WebP, AVIF or JPEG XL source the decoded pixels are held in memory as
well as written to that file, and the 4.2 GiB above includes that memory. For
an 8192x8192 source the file was measured on pixad at 192 MiB for an 8-bit RGB
JPEG, 512 MiB for a 16-bit RGBA PNG or JPEG XL, and 1 GiB for a JPEG XL of
32-bit floating-point RGBA, the largest measured: up to 1 GiB per image
processed at once, 24 GiB at the defaults on a host where pixa sees 48 CPUs.
This `/tmp` is in the container's own filesystem, not on the volume at
`/var/lib/pixa`, and `cache_max_bytes` does not count it. The 4.2 GiB above
does not include these files; where `/tmp` is a tmpfs, they are memory on top
of it.
- **Health check:** the image's `HEALTHCHECK` requests - **Health check:** the image's `HEALTHCHECK` requests
`/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds `/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds
after a deploy and marks the deploy failed unless it is `healthy`. The probe after a deploy and marks the deploy failed unless it is `healthy`. The probe
@@ -178,102 +142,28 @@ allowlisted source hosts for open access.
## Design ## Design
### Concepts
These terms are the names the code uses:
- **Source**: the image at one upstream URL, named by its host, path and query.
- **Source key**: the SHA-256 of a source's host, path and query, which names
the source's file and its row in the database.
- **Transform**: the output size, format, quality and fit an image request asks
for; with the format `auto`, pixa chooses the output format from the `Accept`
header.
- **Image request**: a source and a transform, which each image route reads from
its URL.
- **Variant**: the image pixa serves for an image request, made from its source
with its transform.
- **Variant key**: the SHA-256 of a variant's source, size, format, quality and
fit, which names the variant's file and its row in the database, and gives its
`ETag`.
- **Signed URL**: a `/v1/image/` URL whose `sig` signs its image request and its
expiry `exp` with `signing_key`.
- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any
expiry, encrypted with a key derived from `signing_key`.
- **Login session**: an encrypted cookie that keeps a browser logged in to the
generator page for 30 days, or until a logout replaces the random value pixa
keeps in the `logout` table (see Encrypted URLs).
- **Negative cache**: the source URLs pixa does not fetch again for 5 minutes,
after one of these failed fetches:
- the upstream answered with an error status
- the body's first bytes were not those of the format its `Content-Type`
names
- **Image cache**: the part of pixa that keeps the sources, the variants and the
negative cache (see Storage).
- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached
sources and variants may reach together.
- **Image proxy**: the part of pixa that serves a variant from the image cache
when it can, and otherwise makes it from the source.
### Storage ### Storage
- **Sources**: `<state_dir>/cache/sources/<ab>/<cd>/<source key>`, one file for - **Source content**:
each source URL `<state_dir>/cache/sources/<ab>/<cd>/<sha256 of source content>`
- **Source metadata**:
`<state_dir>/cache/metadata/<hostname>/<sha256 of path and query>.json` (host,
path and query, content hash, upstream status and headers, fetch time)
- **Database**: `<state_dir>/state.sqlite3` (SQLite) - **Database**: `<state_dir>/state.sqlite3` (SQLite)
- **Variants**: `<state_dir>/cache/variants/<ab>/<cd>/<variant key>` - **Transformed images**:
`<state_dir>/cache/variants/<ab>/<cd>/<sha256 of host, path, query, size, format, quality and fit>`,
each with a `.meta` file beside it holding its content type
`<ab>` and `<cd>` are the first and second pairs of characters of the file's `<ab>` and `<cd>` are the first and second pairs of characters of the file's
name. name.
The image cache keeps these tables in the database: Multiple source paths may reference the same content blob; the database tracks
references rather than using filesystem refcounting.
- `source_content`: a row for each source, by source key, with its size and when
it was last used
- `variant_content`: a row for each variant, by variant key (column
`cache_key`), with its size and when it was last used
- `cache_usage`: the total size of the cached sources and variants, which
eviction reads
- `negative_cache`: a row for each source URL in the negative cache, with when
it expires
pixa reads a cached source image's format from its file's first bytes. When they
match none of the source formats, pixa fetches the source again. Two source URLs
that serve the same bytes are stored twice, once under each URL. A source image
is stored by writing its file and then its row; when the row cannot be written,
the file is removed again.
Each source and variant file is written under a temporary name and synced to
disk before it is renamed into place. So a crash never leaves part of a file
under its own name. When pixa starts, and every five minutes after, it removes
each source or variant whose file is empty, or not the size its row records. It
logs a warning naming the file. A later request for that image fetches the
source or makes the variant again. Looking for those files reads the database
once for every cached file, so pixa skips each five-minute look that would start
less than ten times as long after the last look ended as that look took. After a
look that took one minute, the next one starts 10 to 15 minutes after that look
ended.
Each look also drops the rows of variants whose files are missing. When pixa
cannot read a variant file's size for another reason, it keeps the variant's
row. It logs a warning naming the variant key and the error.
During a look, pixa skips a directory it cannot list and a file whose size it
cannot read. It logs a warning naming the path and the error, and goes on with
the rest of the look. A file renamed or removed after pixa listed its directory
gets no warning.
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
Test). Toward it, a cache hit opens the variant's file and nothing else, and Test). Toward it, pixa keeps in memory the content types of the 10,000
makes no database call. The content type comes from the file's first bytes, transformed images most recently cached or served, so a cache hit on one of them
which are those of the format pixa wrote it in; a file whose first bytes are not reads only the image file from disk and not the metadata file stored beside it.
those of a format pixa writes is not served, and the image is converted again. A
hit hands the time it was served to a background writer and does not wait for
the writer. Eviction orders by that time. The hit is counted in memory. A miss
that uses a cached source image also hands that writer the time it used the
image. Once a second that writer writes the waiting times in one transaction. Up
to 10,000 access times of variants, and as many of sources, can wait to be
written. A time handed over beyond that is dropped. Its image then only looks
less recently used to eviction. `script/bench` measures how many hits pixa
serves per second (see Load Test).
### Routes ### Routes
@@ -294,69 +184,34 @@ path under `/v1/` answers 200, in maintenance mode too.
session it answers 303 to `/`. Answers: 200 with the page showing the URL; 400 session it answers 303 to `/`. Answers: 200 with the page showing the URL; 400
with the page naming a field that is not valid; 500 when the URL cannot be with the page naming a field that is not valid; 500 when the URL cannot be
made. made.
- `POST /logout` — log out with the generator page's Logout button: end every - `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`.
login session, in every browser (see Encrypted URLs), and remove the session
cookie. Needs: a login session and the generator page's form (below); without
a login session it only removes the cookie. Answers: 303 to `/`; 500 when the
logout cannot be saved in the database.
- `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>`, or - `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>`, or
`/v1/image/<host>/<path>/<size>` with no format — an image, fetched, resized `/v1/image/<host>/<path>/<size>` with no format — an image, fetched, resized
and converted (below). Needs: a signature, unless the host is allowlisted (see and converted (below). Needs: a signature, unless the host is allowlisted (see
Source Hosts). Answers: Source Hosts). Answers: 200; 304 when `If-None-Match` matches the image's
- 200 with the image. `ETag`; 400 for a URL or parameter that is not valid, or for the format `auto`
- 304 when `If-None-Match` matches the image's `ETag`. an `Accept` header that is not valid; 406 for the format `auto` when `Accept`
- 400 for a URL or parameter that is not valid. allows none of the formats it chooses from; 401 for a missing or wrong
- 400 when the format is `auto` and the `Accept` header is not valid. signature, a missing `exp` or an `exp` in the past; 403 when the request's
- 406 when the format is `auto` and `Accept` allows none of the formats it `Referer` names a host in `referer_blocklist`, checked before the signature,
chooses from. the cache and the upstream fetch; 403 when the upstream host, or a host it
- 401 for a missing or wrong signature. redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an
- 401 for a missing `exp`. address in a blocked network (see `blocked_networks`); 429 over
- 401 for an `exp` in the past. `image_requests_per_minute` (below); 415 when the upstream answers with a
- 403 when the request's `Referer` names a host in `referer_blocklist`, `Content-Type` that is not one of the source formats (below), or with an image
checked before the signature, the cache and the upstream fetch. whose first bytes libvips reads as another format; 502 when the upstream
- 403 when the upstream host, or a host it redirects to, is `localhost` or answered with an error status, and for 5 minutes after that for the same
ends in `.localhost` or `.local`. source URL, though an image of it that is already in the cache is still
- 403 when the upstream host, or a host it redirects to, has an address in a served; 503 when pixa is busy or in maintenance mode; 500 for any other
blocked network (see `blocked_networks`). failure.
- 429 over `image_requests_per_minute` (below).
- 499, with no body, when the client goes away before its image is ready,
unless its request is the one making the image (below).
- 415 when the upstream answers with a `Content-Type` that is not one of the
source formats (below).
- 415 when the upstream sends a body whose first bytes are not those of the
format its `Content-Type` names.
- 415 when the upstream sends an image whose first bytes libvips reads as
another format.
- 502 when the upstream answers with an error status.
- 502 for 5 minutes after that for the same source URL, when pixa has
neither the image nor its source image in its cache.
- 502 likewise for 5 minutes after a body whose first bytes are not those of
its `Content-Type`.
- 503 when pixa is busy.
- 503 when pixa is in maintenance mode.
- 504 when the upstream has not sent its response headers within
`upstream_fetch_timeout`.
- 500 when that time runs out while the image itself is still arriving.
- 500 for any other failure.
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL - `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
(see Encrypted URLs). Needs: nothing but the URL. Answers: (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
- 200 with the image. `If-None-Match` matches the image's `ETag`; 400 for a token that does not
- 304 when `If-None-Match` matches the image's `ETag`. decrypt, or that asks for a size or fit that is not valid; 410 once it has
- 400 for a token that does not decrypt. expired; 504 when the upstream has not sent its response headers within
- 400 for a token that asks for a size or fit that is not valid. `upstream_fetch_timeout`, but 500 when that time runs out while the image
- 410 once the token has expired. itself is still arriving; 400 for an `Accept` header that is not valid, and
- 400 when the token's format is `auto` and the `Accept` header is not 406, 403, 415, 429, 502, 503 and 500, as for `/v1/image/`.
valid.
- These answers, for the same causes as on `/v1/image/`:
- 406
- 403
- 415
- 429
- 499, with no body
- 502
- 503
- 504
- 500
- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs: - `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs:
nothing. Answers: 200. nothing. Answers: 200.
- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`, - `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`,
@@ -377,7 +232,7 @@ the request carries the same ID as `request_id`, and so do the lines it logs
when it fetches, converts and serves an image; the fetch sends it to the when it fetches, converts and serves an image; the fetch sends it to the
upstream host as `X-Request-ID`. upstream host as `X-Request-ID`.
The `POST` routes accept only a form that pixa's own page served: the page puts Both `POST` routes accept only a form that pixa's own page served: the page puts
a token in the form and sets a cookie to match, and a request without both is a token in the form and sets a cookie to match, and a request without both is
refused with 403, so another site cannot submit the form from a visitor's refused with 403, so another site cannot submit the form from a visitor's
browser. The login and generator pages are meant to be opened over HTTPS: while browser. The login and generator pages are meant to be opened over HTTPS: while
@@ -416,57 +271,17 @@ once, is refused with 400.
both optional (values under Signature Specification). Both are part of what is both optional (values under Signature Specification). Both are part of what is
cached, so each value of either is a separate cached image. cached, so each value of either is a separate cached image.
A source URL's query follows its path after `%3F`, the encoded `?`:
`/v1/image/<host>/<path>%3F<query>/<size>.<format>`. pixa decodes the
percent-escapes in the image URL's path once, then reads the host up to the
first `/`, the size and format after the last `/`, and between them the source
path up to the first `?` and the source query after it. So the `?` before the
query, any `?` in the query, and every `%` in the source path or query must be
percent-encoded, as `%3F` and `%25`. Any other character may be written as
itself or percent-encoded: `/`, `;`, `,`, `=` and `&` work either way. Neither
the source path nor the source query may hold a `#`, and an image URL that
writes one as `%23` is refused with 400. For example,
`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg` at
800x600 WebP is
`/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp`,
signed as shown under Signature Specification.
The source path is the path with its percent-escapes decoded, and pixa fetches
it percent-encoded again: every character other than letters, digits, `-`, `.`,
`_`, `~`, `/`, `$`, `&`, `+`, `,`, `:`, `;`, `=` and `@` is sent as a
percent-escape. So `my%20cat.jpg` in the image URL is the source path
`/my cat.jpg`, fetched as `/my%20cat.jpg`; `my%2520cat.jpg` is the source path
`/my%20cat.jpg`, fetched as `/my%2520cat.jpg`; and `100%25.png` is fetched as
`/100%25.png`. The source query is the source URL's query as that URL writes it,
its percent-escapes included, and is sent as it is, except that the characters a
browser percent-encodes in a query (a space, `"`, `'`, `<`, `>`, control
characters and characters outside ASCII) and a `%` that does not begin a
percent-escape are percent-encoded. pixa signs and caches the query in that
form, so `img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are
the same source query, `name=my%20cat`.
A size with one side `0` has that side scaled to keep the source's proportions, A size with one side `0` has that side scaled to keep the source's proportions,
and is refused with 400 when that side would come out above 8192, the most and is refused with 400 when that side would come out above 8192, the most
either side may be; so is a size with `fit=outside`, which scales the image to either side may be; so is a size with `fit=outside`, which scales the image to
cover the size and so past it on one side, when that side would come out cover the size and so past it on one side, when that side would come out
above 8192. above 8192.
An AVIF image of more than 16,777,216 pixels, the number 4096x4096 and 8192x2048
have, is refused with 400 before anything is decoded, as its encode could take
more than 40 seconds (see below). That includes `orig` of an AVIF source, and
`auto` when it chooses AVIF, so a client whose `Accept` names `image/avif` but
not `image/jxl` is refused such a size.
The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's The source image may be JPEG, PNG, GIF, WebP, AVIF or JPEG XL. The upstream's
`Content-Type` must name its format, and the image's first bytes must match it. `Content-Type` must name its format, and the image's first bytes must match it.
An upstream answering with any other `Content-Type`, SVG included, is refused An upstream answering with any other `Content-Type`, SVG included, is refused
with 415. So is a body whose first bytes are not those of the format its with 415, and so is an image whose first bytes libvips reads as another format,
`Content-Type` names, such as a JPEG sent as `image/png`, text sent as an image, such as an SVG whose first bytes pass the check for AVIF.
or an empty body. pixa then does not fetch that source URL again for 5 minutes,
as after an upstream error status, and meanwhile answers 502 for an image that
would need that fetch. An image whose first bytes libvips reads as another
format, such as an SVG whose first bytes pass the check for AVIF, is refused
with 415 too.
With the format `auto`, pixa chooses the format for each request from its With the format `auto`, pixa chooses the format for each request from its
`Accept` header, in this order: `Accept` header, in this order:
@@ -499,30 +314,9 @@ cache at once, they share one upstream fetch (or one read of the cached source)
and one transcode: the first request does the work, and the others wait for its and one transcode: the first request does the work, and the others wait for its
image or its error, holding no upstream connection or processing slot of their image or its error, holding no upstream connection or processing slot of their
own. A waiting request stops waiting when its own client goes away. The work own. A waiting request stops waiting when its own client goes away. The work
goes on for the others even if the first request's client goes away. The shared goes on for the others even if the first request's client goes away, until that
fetch sends the first request's ID upstream, and the lines logged for the fetch request's `downstream_timeout` ends. The shared fetch sends the first request's
and the transcode carry that ID. ID upstream, and the lines logged for the fetch and the transcode carry that ID.
A transcode whose request's `downstream_timeout` has ended (for a shared one,
the first request's) stops waiting for a processing slot, or, once it has one,
gives it back without beginning to encode. A request waiting for a shared
transcode that stops this way is answered 503 with the error
`server busy, try again later`, though its own `downstream_timeout` may not have
ended. govips gives pixa no way to stop libvips once an encode has begun, so an
encode keeps its slot until it finishes, which can be after
`downstream_timeout`. The limit of 8192 on each side of the source and of the
output, and for AVIF the limit of 16,777,216 pixels, bound how long that takes.
With one libvips thread, as pixad runs it, encoding 8192x8192 random pixels
takes up to about 1 second as JPEG, 15 as PNG and 16 as GIF, the slowest of 8 or
16 bits per sample, with or without an alpha channel. The WebP and JPEG XL
encoders run below libvips' default effort, and AVIF at the lowest effort govips
can set, so that no encode takes more than 40 seconds of the default
`downstream_timeout` of 60: at any quality and 8 or 16 bits per sample,
8192x8192 random pixels with an alpha channel take up to about 23 seconds as
WebP and 27 as JPEG XL, and 4096x4096, 8192x2048 or 2048x8192 up to about 34 as
AVIF, which at quality 85 would take about 96 at 8192x8192. Without an alpha
channel they take up to about 16, 13 and 22 seconds. Decoding such a source adds
up to about 11 seconds.
The login form (`POST /`) is limited to 5 attempts per minute per client The login form (`POST /`) is limited to 5 attempts per minute per client
address, counting an IPv6 client by its /64; an attempt over the limit is address, counting an IPv6 client by its /64; an attempt over the limit is
@@ -559,28 +353,14 @@ nor change what it asks for.
1. Open `/` in a browser over HTTPS (or over plain HTTP while `debug` is on, see 1. Open `/` in a browser over HTTPS (or over plain HTTP while `debug` is on, see
Routes) and log in with the signing key (`signing_key`). The login session Routes) and log in with the signing key (`signing_key`). The login session
lasts 30 days, or until someone logs out with the generator page's Logout lasts 30 days, or until `/logout`.
button. Every login is made with the one signing key, so logging out ends
every login session, in every browser, a copy of the session cookie included.
pixa keeps a random value in its database, which it makes when the database
has none and replaces at each logout. A login session carries the value
current at its login and is refused once the value has changed, also after a
restart (see Deployment). The check for a logout compares no time, so a login
after a logout works whatever the host's clock says; the 30 days are still
measured by the host's clock.
2. On the generator page, give the source image's URL, the width and height, the 2. On the generator page, give the source image's URL, the width and height, the
format, quality and fit, and how long the URL lasts, then submit the form format, quality and fit, and how long the URL lasts, then submit the form
(`POST /generate`). The format is JPEG XL unless another is chosen; a form (`POST /generate`). The format is JPEG XL unless another is chosen; a form
sent with an empty format, or none, also makes a JPEG XL URL. Width and sent with an empty format, or none, also makes a JPEG XL URL. Width and
height both empty or `0` keep the original size; if only one of them is empty height both empty or `0` keep the original size; if only one of them is empty
or `0`, that side is scaled to keep the image's proportions. A URL whose or `0`, that side is scaled to keep the image's proportions. A URL whose
scaled side would come out above 8192 answers 400, and so does an AVIF URL of scaled side would come out above 8192 answers 400.
more than 16,777,216 pixels or a source URL whose path holds a `..` segment.
So does a source URL that the image URL would fetch written another way, and
the page then shows the URL pixa would fetch: one with a user name and
password, which pixa does not send, or one whose path holds `%2F`, fetched as
`/`, or `(`, fetched as `%28`, as pixa keeps the source path decoded and
fetches it percent-encoded again (see Routes).
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when 3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
it expires. `<host>` is the host the page was opened on, and the URL starts it expires. `<host>` is the host the page was opened on, and the URL starts
with `http` instead while `debug` is on. The name after the token is ignored with `http` instead while `debug` is on. The name after the token is ignored
@@ -588,11 +368,9 @@ nor change what it asks for.
`jxl` for a form with no format. `jxl` for a form with no format.
The token holds the source's host, path and query and the size, format, quality, The token holds the source's host, path and query and the size, format, quality,
fit and expiry, encrypted with a key derived from `signing_key`. The host is fit and expiry, encrypted with a key derived from `signing_key`. The source
kept in lower case and an empty path as `/`, as for `/v1/image/`, so URL's scheme is not kept: the image is fetched like any other (see Routes), and
`https://CDN.Example.com` and `https://cdn.example.com/` are one source image, the blocked networks still apply.
fetched and cached once. The source URL's scheme is not kept: the image is
fetched like any other (see Routes), and the blocked networks still apply.
How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never. How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never.
The expiry is fixed in the token when the URL is made and cannot be changed or The expiry is fixed in the token when the URL is made and cannot be changed or
@@ -641,23 +419,11 @@ HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit"
Where: Where:
- `host` — source host as the URL writes it up to the first `/`, in lower case, - `host` — source origin hostname (e.g. `cdn.example.com`)
keeping an IPv6 address's brackets and any `:` and port after it (e.g. - `path` — source path (e.g. `/photos/cat.jpg`)
`cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`) - `query` — source query string, empty string if none
- `path` — source path, with its percent-escapes decoded as under Routes (e.g. - `width` — requested width in pixels, `0` for original
`/photos/cat.jpg`, or `/photos/my cat.jpg` for `/photos/my%20cat.jpg`), with - `height` — requested height in pixels, `0` for original
each `%` written as `%25` and each `:` as `%3A`, so `/photos/a:b.jpg` is
signed as `/photos/a%3Ab.jpg`
- `query` — source query string as under Routes, empty string if none, with `%`
and `:` written as in `path`; neither then holds a `:`, so path `/a:b` with no
query and path `/a` with query `b:` are signed differently
- `width` — requested width in pixels, or `0`: the width is then worked out from
the source's proportions and the requested height, rounded down but at least
1, or is the source's own width when the height is `0` too (the `orig` size)
- `height` — requested height in pixels, or `0`: the height is then worked out
from the source's proportions and the requested width, rounded down but at
least 1, or is the source's own height when the width is `0` too (the `orig`
size)
- `format` — output format, one of those listed under Routes, with `original` - `format` — output format, one of those listed under Routes, with `original`
signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no
format has the signature of the same URL ending in `.jxl`; `auto` is signed as format has the signature of the same URL ending in `.jxl`; `auto` is signed as
@@ -670,10 +436,7 @@ Where:
with 400 with 400
- `fit` — the URL's `fit` query parameter (cover, contain, fill, inside, - `fit` — the URL's `fit` query parameter (cover, contain, fill, inside,
outside), or `cover` when the URL has no `fit`; a request whose `fit` is outside), or `cover` when the URL has no `fit`; a request whose `fit` is
anything else, an empty `fit=` included, is refused with 400. `outside` scales anything else, an empty `fit=` included, is refused with 400
the image to cover the requested size, keeping its proportions, so it can come
out larger than that size on one side: an 8192x1 source at `100x0`, which is
100x1 by its proportions, comes out at 8192x1
The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of
RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa
@@ -697,17 +460,6 @@ For the same image at quality 40 with fit `contain`, the input ends in
and the URL is and the URL is
`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`. `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`.
For a source URL with a query,
`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg`, at
800x600 WebP with the same key and expiration and the default quality and fit:
1. Build input, with the query's `:` written as `%3A` and its `%` as `%25`:
`cdn.example.com:/img:url=https%3A//origin.example.com/my%2520cat.jpg:800:600:webp:1704067200:85:cover`
2. The signature is `37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=`
3. URL, with the `?` before the query written as `%3F` and the query's `%` as
`%25` (see Routes):
`/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp?sig=37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=&exp=1704067200`
**Allowlist patterns:** **Allowlist patterns:**
- **Exact match**: `cdn.example.com` — matches only that host - **Exact match**: `cdn.example.com` — matches only that host
@@ -719,100 +471,59 @@ entry that is neither a host name (letters, digits, hyphens, underscores and
dots, with at most one leading dot) nor an IP address, such as one with a port dots, with at most one leading dot) nor an IP address, such as one with a port
or a `*.` wildcard, aborts startup. or a `*.` wildcard, aborts startup.
pixa lower-cases the host of a `/v1/image/` URL before anything else uses it, so
`CDN.Example.com` is matched, fetched, cached and counted against
`upstream_connections_per_host` as `cdn.example.com`. Such a URL whose host has
a port, as in `cdn.example.com:8443`, or a `:` with no port, needs a signature
even when the host is allowlisted. The host must be a host name or an IPv4
address (letters, digits, hyphens, underscores and dots) or an IPv6 address in
brackets, either one optionally followed by `:` and a port. Any other host, such
as one holding `@`, or a `?`, `#`, `%`, `\` or space written percent-encoded, is
refused with 400.
### Configuration ### Configuration
Every setting can be given as an environment variable, in a YAML config file Every setting can be given as an environment variable, in a YAML config file
(`--config`), or both. A variable present in the environment wins over the file, (`--config`), or both. A variable present in the environment wins over the file,
even when it is empty, and the file wins over the built-in default. The one even when it is empty, and the file wins over the built-in default. The one
exception is a variable set in the file's `env:` section. pixa sets that exception is a variable named in the file's `env:` section: it is set while the
variable while loading the file, so the variable wins over the environment pixa file loads, so it overrides both the environment the process was started with
started with. It also wins over the file's key for the same setting. A and the file's own key. A variable's value is parsed as the same text in the
variable's value is parsed as the same text in the file would be. A setting that file would be. The three lists take comma-separated entries, with the spaces
holds a list takes comma-separated entries, with the spaces around each trimmed. around each trimmed; an empty variable is an empty list. A value that does not
An empty variable is an empty list. A value that does not parse or is invalid parse or is invalid aborts startup, naming the variable. A variable whose name
aborts startup, naming the variable. pixa refuses to start when a variable starts with `PIXA_` but is not in the table below, such as a misspelled one or
starts with `PIXA_` but is not in the list below, such as a misspelled one or `PIXA_PORT`, aborts startup naming it, as an unknown config key does. The one
`PIXA_PORT`. The error names the variable, as it names an unknown config key. other accepted name is `PIXA_CONFIG_PATH`, the config file's path (like
pixa also accepts `PIXA_CONFIG_PATH`, which names the config file, like `--config`). The variables set by the file's `env:` section are checked the same
`--config`. The variables set by the file's `env:` section are checked the same
way. way.
pixa reads at most one config file. It reads the file given with `--config` (or pixa reads at most one config file: the one given with `--config` (or `-c`),
`-c`), or else the one `PIXA_CONFIG_PATH` names. pixa refuses to start when that otherwise the one `PIXA_CONFIG_PATH` names, otherwise the first of these that
file does not exist, cannot be read or does not parse. Without `--config` or pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`,
`PIXA_CONFIG_PATH`, pixa looks in `/etc/pixa/`, then in `~/.config/pixa/`, then `~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml` and
in the working directory. In each directory it looks for `config.yml`, then `config.yaml` in the working directory. A named file that does not exist, cannot
`config.yaml`. It reads the first file it finds. It skips a missing file without be read or does not parse aborts startup. Of the files pixa looks for on its
a message. pixa refuses to start when a file it finds cannot be read or does not own, only one that does not exist is passed over, without a message. One that
parse, and the error names the file. It also refuses to start when it may not pixa cannot read or parse aborts startup, naming the file. So does one in a
enter one of these directories, whether or not the file is there, since it directory pixa may not enter, whether or not it is there, since pixa cannot
cannot tell. With no file, pixa uses the environment and the defaults. tell. With no file, pixa uses the environment and the defaults.
Each item below is one setting, named by its config key: | Variable | Config key | Meaning |
| ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- |
- `signing_key`: variable `PIXA_SIGNING_KEY`, required. The secret for signed | `PIXA_SIGNING_KEY` | `signing_key` | Required: secret for signed and encrypted URLs and login, 32+ characters |
URLs, encrypted URLs and login. It must be 32 characters or longer. | `PORT` | `port` | Port to listen on; default `8080` |
- `port`: variable `PORT`, default `8080`. The port to listen on. | `PIXA_STATE_DIR` | `state_dir` | Directory for the database and the disk cache; default `/var/lib/pixa` |
- `state_dir`: variable `PIXA_STATE_DIR`, default `/var/lib/pixa`. The directory | `PIXA_DB_URL` | `db_url` | SQLite database URL; default `state.sqlite3` in the state directory |
for the database and the disk cache. | `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of (free + cached) |
- `db_url`: variable `PIXA_DB_URL`, default `state.sqlite3` in the state | `PIXA_ALLOWLIST_HOSTS` | `allowlist_hosts` | Upstream hosts served without a signature |
directory. The URL of the SQLite database. | `PIXA_REFERER_BLOCKLIST` | `referer_blocklist` | Hosts whose pages the image routes refuse with 403, by `Referer` |
- `cache_max_bytes`: variable `PIXA_CACHE_MAX_BYTES`, default 75% of the sum of | `PIXA_BLOCKED_NETWORKS` | `blocked_networks` | CIDR ranges never fetched from, on top of the built-in ones |
the free space and what the cache holds. The disk cache limit in bytes. `0` | `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 |
turns the disk cache off. | `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` |
- `allowlist_hosts`: variable `PIXA_ALLOWLIST_HOSTS`, default none. The upstream | `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` |
hosts whose images pixa serves without a signature. | `PIXA_UPSTREAM_CONNECTIONS` | `upstream_connections` | Concurrent connections to all upstream hosts together; default `64` |
- `referer_blocklist`: variable `PIXA_REFERER_BLOCKLIST`, default none. The | `PIXA_MAX_CONCURRENT_PROCESSING` | `max_concurrent_processing` | Images processed at once; default the number of CPUs |
image routes refuse a request with 403 when its `Referer` header names one of | `PIXA_IMAGE_REQUESTS_PER_MINUTE` | `image_requests_per_minute` | Image requests per client per minute; `0` turns it off; default `600` |
these hosts. | `PIXA_UPSTREAM_FETCH_TIMEOUT` | `upstream_fetch_timeout` | Time allowed for one fetch from an upstream host; default `30s` |
- `blocked_networks`: variable `PIXA_BLOCKED_NETWORKS`, default none. The CIDR | `PIXA_UPSTREAM_MAX_RESPONSE_SIZE` | `upstream_max_response_size` | Largest upstream response accepted, in bytes; default 50 MiB |
ranges pixa never fetches from, on top of the built-in ones. | `PIXA_DOWNSTREAM_TIMEOUT` | `downstream_timeout` | Time allowed for answering one client request; default `60s` |
- `trusted_proxies`: variable `PIXA_TRUSTED_PROXIES`, default the RFC 1918 | `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN` | `access_control_allow_origin` | CORS origin allowed to read image responses: `*` or one origin; default `*` |
ranges. The CIDR ranges of the proxies whose `X-Forwarded-For` header pixa | `PIXA_METRICS_USERNAME` | `metrics.username` | Username for `/metrics`, which is served only when both are set |
believes. | `PIXA_METRICS_PASSWORD` | `metrics.password` | Password for `/metrics`; set together with the username |
- `allow_http`: variable `PIXA_ALLOW_HTTP`, default `false`. Allows plain-HTTP | `PIXA_SENTRY_DSN` | `sentry_dsn` | Sentry DSN for error reporting; empty disables it |
upstream hosts. It is for testing only. | `PIXA_DEBUG` | `debug` | Debug logging and plain-HTTP local development; default `false` |
- `upstream_connections_per_host`: variable | `PIXA_MAINTENANCE_MODE` | `maintenance_mode` | Answer image requests with 503; the health check stays 200; default `false` |
`PIXA_UPSTREAM_CONNECTIONS_PER_HOST`, default `20`. The most connections to
one upstream host at once.
- `upstream_connections`: variable `PIXA_UPSTREAM_CONNECTIONS`, default `64`.
The most connections to all upstream hosts together at once.
- `max_concurrent_processing`: variable `PIXA_MAX_CONCURRENT_PROCESSING`,
default half the CPUs, at least 1. The most images processed at once.
- `image_requests_per_minute`: variable `PIXA_IMAGE_REQUESTS_PER_MINUTE`,
default `600`. How many image requests one client may send per minute. `0`
turns the limit off.
- `upstream_fetch_timeout`: variable `PIXA_UPSTREAM_FETCH_TIMEOUT`, default
`30s`. The time allowed for one fetch from an upstream host.
- `upstream_max_response_size`: variable `PIXA_UPSTREAM_MAX_RESPONSE_SIZE`,
default 50 MiB. The largest upstream response accepted, in bytes.
- `downstream_timeout`: variable `PIXA_DOWNSTREAM_TIMEOUT`, default `60s`. The
time allowed for answering one client request.
- `access_control_allow_origin`: variable `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN`,
default `*`. The CORS origin allowed to read image responses. It is `*` or one
origin.
- `metrics.username`: variable `PIXA_METRICS_USERNAME`, default empty. The
username for `/metrics`. pixa serves `/metrics` only when the username and the
password are both set.
- `metrics.password`: variable `PIXA_METRICS_PASSWORD`, default empty. The
password for `/metrics`. Set it together with the username.
- `sentry_dsn`: variable `PIXA_SENTRY_DSN`, default empty. The Sentry DSN for
error reporting. An empty DSN turns error reporting off.
- `debug`: variable `PIXA_DEBUG`, default `false`. Turns on debug logging and
plain-HTTP local development.
- `maintenance_mode`: variable `PIXA_MAINTENANCE_MODE`, default `false`. While
`true`, pixa answers image requests with 503. The health check still
answers 200.
Key settings in more detail: Key settings in more detail:
@@ -840,12 +551,7 @@ Key settings in more detail:
and generator pages. Default: empty and generator pages. Default: empty
- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added - `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added
to the always-enforced built-in ranges (loopback, private, link-local, CGNAT, to the always-enforced built-in ranges (loopback, private, link-local, CGNAT,
benchmark, NAT64, and the like); an invalid CIDR aborts startup. An IPv6 benchmark, NAT64, and the like); an invalid CIDR aborts startup
address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4
address it holds, so an IPv6 range such as `::/0` refuses neither it nor a
plain IPv4 address. An entry written in that form is read as the IPv4 range it
covers, so `::ffff:10.0.0.0/104` refuses what `10.0.0.0/8` refuses; such an
entry shorter than `/96` covers more than IPv4 addresses and aborts startup
- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of - `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of
pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one
of these ranges; the logged and login-recorded client address is then the of these ranges; the logged and login-recorded client address is then the
@@ -855,22 +561,16 @@ Key settings in more detail:
omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`, omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`,
`172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a
private network; an explicitly empty list (`[]`) trusts no one, and an private network; an explicitly empty list (`[]`) trusts no one, and an
explicit list replaces the default. An invalid CIDR aborts startup. An IPv6 explicit list replaces the default. An invalid CIDR aborts startup. Set this
address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4 to the address pixa sees for requests that come through your proxy, such as
address it holds, so an IPv6 range such as `::/0` trusts neither it nor a `172.17.0.1/32`, when the defaults do not cover it, or to trust nothing else
plain IPv4 address. An entry written in that form is read as the IPv4 range it (see the login limit and `image_requests_per_minute` under Routes). For a
covers, so `::ffff:10.0.0.0/104` trusts what `10.0.0.0/8` trusts; such an proxy on the Docker host that connects to pixa over `127.0.0.1`, that address
entry shorter than `/96` covers more than IPv4 addresses and aborts startup. is the gateway of the container's Docker network (`172.17.0.1` on the default
Set this to the address pixa sees for requests that come through your proxy, bridge), not the proxy's own address; a proxy that connects through another of
such as `172.17.0.1/32`, when the defaults do not cover it, or to trust the host's addresses is seen with that address. To be sure which address it
nothing else (see the login limit and `image_requests_per_minute` under is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request
Routes). For a proxy on the Docker host that connects to pixa over through the proxy, and read `remoteIP` in pixa's request log line for it
`127.0.0.1`, that address is the gateway of the container's Docker network
(`172.17.0.1` on the default bridge), not the proxy's own address; a proxy
that connects through another of the host's addresses is seen with that
address. To be sure which address it is, set this to `[]` (or
`PIXA_TRUSTED_PROXIES` to empty), send a request through the proxy, and read
`remoteIP` in pixa's request log line for it
- `upstream_fetch_timeout` — time allowed for one fetch from an upstream host, - `upstream_fetch_timeout` — time allowed for one fetch from an upstream host,
as a duration such as `30s` (the default) or `2m` as a duration such as `30s` (the default) or `2m`
- `upstream_max_response_size` — largest upstream response accepted, in bytes; - `upstream_max_response_size` — largest upstream response accepted, in bytes;
@@ -879,27 +579,14 @@ Key settings in more detail:
duration; default `60s`. The upstream fetch counts toward it, and so do the duration; default `60s`. The upstream fetch counts toward it, and so do the
waits for an upstream connection and for a processing slot (up to 10 seconds waits for an upstream connection and for a processing slot (up to 10 seconds
each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds
- `signing_key` — the secret that protects signed URLs, encrypted URLs and - `signing_key` — HMAC secret for URL signatures
logins. pixa uses it as it is for two things:
- the HMAC key of signed URLs (see Signature Specification)
- the password of the login page
pixa derives every other key it uses from it at startup, with HKDF-SHA256:
- a key that signs the session cookie
- a key that encrypts the session cookie
- a key that encrypts the token of an encrypted URL
- a key that signs the CSRF token cookie of the login and generator pages
- `db_url` — the SQLite database to open; omitted, it is - `db_url` — the SQLite database to open; omitted, it is
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the `file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
database in WAL mode. `state_dir` is percent-encoded in it, so a `%`, `?` or database in WAL mode. pixa opens one connection to it, so its own reads and
`#` in the name is written as `%25`, `%3F` or `%23`. For a `state_dir` of `/` writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to any
or one starting with `//` it starts `file:////`, so SQLite does not read the `db_url`, so a write that finds another program writing to the file waits up
first name in the path as a host. pixa opens one connection to it, so its own to five seconds for it instead of failing. WAL mode comes only from the URL:
reads and writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to keep `_pragma=journal_mode(WAL)` in one you set
any `db_url`, so a write that finds another program writing to the file waits
up to five seconds for it instead of failing. WAL mode comes only from the
URL: keep `_pragma=journal_mode(WAL)` in one you set
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk - `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk
cache entirely; omitted defaults to 75% of the sum of the free space on the cache entirely; omitted defaults to 75% of the sum of the free space on the
filesystem containing `<state_dir>/cache/` and the bytes of source and filesystem containing `<state_dir>/cache/` and the bytes of source and
@@ -912,12 +599,10 @@ Key settings in more detail:
does, and `downstream_timeout` has not ended first, the request is answered does, and `downstream_timeout` has not ended first, the request is answered
503 with the error `server busy, try again later` 503 with the error `server busy, try again later`
- `max_concurrent_processing` — the most images decoded and encoded at once; - `max_concurrent_processing` — the most images decoded and encoded at once;
default half the CPUs pixa can use (`GOMAXPROCS`, which follows a container's default the number of CPUs pixa can use (`GOMAXPROCS`), which follows a
CPU limit), rounded down, and at least 1. A value that is not a whole number container's CPU limit. A request that finds all of them in use waits up to 10
of at least 1 aborts startup. A request that finds all of them in use waits up seconds for one to free up; if none does, and `downstream_timeout` has not
to 10 seconds for one to free up; if none does, and `downstream_timeout` has ended first, it is answered 503 the same way
not ended first, it is answered 503 the same way. Each one can take a lot of
memory: see "Running under upaas"
- `image_requests_per_minute` — how many requests one client may send to the - `image_requests_per_minute` — how many requests one client may send to the
image routes per minute (see the limit on them under Routes); default `600`. image routes per minute (see the limit on them under Routes); default `600`.
`0` turns the limit off, and a negative or non-integer value aborts startup. `0` turns the limit off, and a negative or non-integer value aborts startup.
@@ -936,50 +621,15 @@ See `configs/config.example.yml` for all options with defaults.
### Architecture ### Architecture
- **Dependency injection**: Uber fx. `internal/app` puts every part of pixad in - **Dependency injection**: Uber fx
one fx graph. `internal/providers` builds the image cache, the image proxy and
the other parts that serve images and logins. Each setting's default is
declared once, in `internal/config`.
- **Packages**: these packages under `internal/` work with the concepts above:
- `handlers`: the routes, which read the image request from the URL, check
the signed or encrypted URL, and make the login session
- `imagerequest`: the image request, with the source key and the variant key
- `imageproxy`: the image proxy
- `imgcache`: the image cache, which keeps its sources in one `FileStore`
and its variants in another
- `httpfetcher`: fetching sources from upstream hosts
- `magic`: reading an image's format from its first bytes
- `imageprocessor`: making a variant from a source with libvips
- `signature`: checking the `sig` of a signed URL
- `encurl`: making and reading the tokens of encrypted URLs
- `seal`: the keys derived from `signing_key`, and the encryption of those
tokens
- `session`: the login session
- **HTTP router**: go-chi - **HTTP router**: go-chi
- **Image processing**: govips (CGO wrapper for libvips) - **Image processing**: govips (CGO wrapper for libvips)
- **Database**: SQLite via modernc.org/sqlite - **Database**: SQLite via modernc.org/sqlite
- **Static assets**: embedded via `//go:embed` - **Static assets**: embedded via `//go:embed`
- **Metrics**: Prometheus, at `/metrics`. pixa serves `/metrics` and measures - **Metrics**: Prometheus, at `/metrics`: generic HTTP request metrics
HTTP requests only when `metrics.username` and `metrics.password` are set. (duration, response size, requests in flight) and the Go runtime and process
`/metrics` holds: metrics; requests are measured and `/metrics` is served only when
- the duration of HTTP requests `metrics.username` and `metrics.password` are set
- the size of HTTP responses
- the HTTP requests in flight
- the Go runtime and process metrics
- `pixa_cache_hits_total`: image requests served from the cache
- `pixa_cache_misses_total`: image requests not served from the cache
- `pixa_upstream_fetches_total`: fetches of a source image that read any
bytes
- `pixa_upstream_fetch_bytes_total`: the bytes those fetches read
- `pixa_transforms_total`: images converted
A request's `handler` label is the pattern of the route it matches, such as
`/v1/image/*`, never its path. Requests that match no route share the label
`unmatched`: those the router answers 404 or 405, and CORS preflights. A
request's `method` label is its method when that is `GET`, `HEAD`, `POST`,
`PUT`, `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` or `TRACE`, and `other` for
any other method.
- **Logging**: stdlib slog - **Logging**: stdlib slog
## Entrypoints ## Entrypoints
@@ -998,9 +648,9 @@ provide:
install-precommit) install-precommit)
- `script/projectname` — output the project name ("pixa") - `script/projectname` — output the project name ("pixa")
- `script/test` — run the test suite: build the `test` phase of the - `script/test` — run the test suite: build the `test` phase of the
`Dockerfile`, writing no image `Dockerfile`, tagged `pixa-test`
- `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`, - `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`,
writing no image; the linter never runs on the host tagged `pixa-lint`; the linter never runs on the host
- `script/fmt` — format the Go code with gofmt and the markdown with prettier - `script/fmt` — format the Go code with gofmt and the markdown with prettier
(writes) (writes)
- `script/fmt-check` — check the same formatting (read-only), on the host - `script/fmt-check` — check the same formatting (read-only), on the host
@@ -1011,9 +661,6 @@ provide:
- `script/docker-smoke` — build the image, start it, wait for it to be healthy - `script/docker-smoke` — build the image, start it, wait for it to be healthy
- `script/loadtest` — measure pixad's throughput, latency and peak memory; a - `script/loadtest` — measure pixad's throughput, latency and peak memory; a
benchmark, not part of `script/check` (see Load Test) benchmark, not part of `script/check` (see Load Test)
- `script/bench` — run the Go benchmark that serves cache hits in parallel
through the image handler, with Go on the host after `script/bootstrap --cgo`;
a benchmark, not part of `script/check` (see Load Test)
- `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`), - `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`),
then `script/check`, then build the image as `script/docker` does then `script/check`, then build the image as `script/docker` does
- `script/precommit` — pre-commit checks (`go mod tidy` guard, then - `script/precommit` — pre-commit checks (`go mod tidy` guard, then
@@ -1021,26 +668,15 @@ provide:
- `script/install-precommit` — install the git pre-commit hook that runs - `script/install-precommit` — install the git pre-commit hook that runs
`script/precommit` `script/precommit`
Every `docker build` in these scripts passes a new value of the `CACHEBUST` Every `docker build` in these scripts passes `--no-cache`, so the lint and test
build argument, which every stage of the `Dockerfile` declares, so every `RUN` phases run on every build instead of coming from the build cache.
step after that line, the lint and test steps included, runs on every build `script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`,
instead of coming from the build cache. Above the line are the steps that `script/setup` and `script/install-precommit` are the standard copies from
install system packages and download the Go modules; Docker keeps their layers `sneak/prompts`, kept identical to them. `script/fmt` and `script/fmt-check` are
until the base image, the step itself, `go.mod` or `go.sum` changes. Go's build the standard copies with pixa's `gofmt` step kept before prettier. prettier
cache is kept from build to build: the steps that compile mount it, so a package formats the markdown only: not the HTML templates, as it cannot parse a Go
whose inputs are unchanged is not compiled again, and both `go test` runs pass template action inside a tag, and not `REPO_POLICIES.md` (see
`-count=1` so that no stored test result is replayed. The lint step also mounts `.prettierignore`), a copy of the one in `sneak/prompts`.
golangci-lint's cache, which reuses a package's findings only when its files,
those of the packages it imports, `go.mod`, the linters and their settings in
`.golangci.yml`, and the golangci-lint version are all unchanged. The `test`
phase runs on the Debian Go image, with Debian's libvips and libheif; the image
pixa ships is built and runs on Alpine. `script/check`, `script/cibuild`,
`script/docker`, `script/lint`, `script/test`, `script/setup` and
`script/install-precommit` are the standard copies from `sneak/prompts`, kept
identical to them. `script/fmt` and `script/fmt-check` are the standard copies
with pixa's `gofmt` step kept before prettier. prettier formats the markdown
only, not the HTML templates, as it cannot parse a Go template action inside a
tag.
## Load Test ## Load Test
@@ -1094,17 +730,6 @@ The numbers depend on the machine and on whatever else runs on it. The first
measurement, made on a shared machine with few clients, is in `TODO.md`; it says measurement, made on a shared machine with few clients, is in `TODO.md`; it says
nothing about the target. nothing about the target.
`script/bench` (or `make bench`) measures cache hits alone, without the network
and with no pixad process: a Go benchmark in `internal/handlers` serves one
cached image through the `/v1/image/` handler from as many goroutines at once as
the machine has CPUs, and reports the hits served per second as `hits/s`. Each
run starts with an empty cache and an empty database, a SQLite file of its own,
and puts the image in the cache before it measures. The benchmark fails if
serving the hits ran any statement on the database. Like the load test it is not
part of `script/check`. It runs Go on the host, as the load test does, and
compiles pixa there, so it needs the C compiler and libraries that
`script/bootstrap --cgo` installs, but not Docker.
## TODO ## TODO
See [TODO.md](TODO.md) for the full prioritized task list. See [TODO.md](TODO.md) for the full prioritized task list.
+107 -293
View File
@@ -1,6 +1,6 @@
--- ---
title: Repository Policies title: Repository Policies
last_modified: 2026-10-18 last_modified: 2026-09-08
--- ---
This document covers repository structure, tooling, and workflow standards. Code This document covers repository structure, tooling, and workflow standards. Code
@@ -54,39 +54,34 @@ style conventions are in separate documents:
`cibuild`. `script/bootstrap` installs all dependencies idempotently and `cibuild`. `script/bootstrap` installs all dependencies idempotently and
assumes nothing is present: base tools come from nix, apt, brew, or apk assumes nothing is present: base tools come from nix, apt, brew, or apk
(detected in that order; apt runs noninteractive). For node it uses the (detected in that order; apt runs noninteractive). For node it uses the
installed node only when its major version is the pinned one; otherwise (node installed node if present; otherwise it installs a PINNED node version via
missing, or another major, such as the node 24 the CI runner image ships) it nvm, first installing nvm itself if missing — from a hash-verified GitHub
installs the PINNED node version via nvm and installs yarn and the packages release archive (never `curl | sh`), with bash installed as an explicit
under it. nvm itself is installed first if missing, from a hash-verified prerequisite since nvm requires bash. yarn is then pinned via
GitHub release archive (never `curl | sh`), with bash installed as an explicit `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
prerequisite since nvm requires bash. Only the major version is compared always exact versions. `script/cibuild` runs the CI build: it changes to the
because the `Dockerfile` stages start from a node image whose exact version is repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned with the version; the Gitea workflow calls it. **`script/cibuild` runs
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
`script/bootstrap` first**, because the workflow checks out the repo and runs `script/bootstrap` first**, because the workflow checks out the repo and runs
nothing else, while `script/fmt-check` runs the formatter on the host: on a nothing else, while `script/fmt-check` runs the formatter on the host: on a
pristine checkout with nothing installed the run dies there, after the pristine checkout with nothing installed the run dies there, after the
containerised gates have passed. **The bootstrap alone is not enough**: when containerised gates have passed. **The bootstrap alone is not enough**:
`script/bootstrap` installs node and yarn under nvm it leaves neither on the `script/bootstrap` installs node and yarn under nvm and leaves neither on the
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
under another node. The host entrypoints that need yarn — `script/fmt` and entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when source nvm for the pinned node version before invoking it, exactly as
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`, `script/bootstrap`'s own install step does. A runner carrying nothing but
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the docker and git then gets through `script/check`. Four further scripts are our
pinned node. A runner carrying nothing but docker and git then gets through own extensions to the standard: `script/check` runs `script/test`,
`script/check`. Four further scripts are our own extensions to the standard: `script/lint` and `script/fmt-check`; `script/precommit` is what the git
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`; pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
`script/precommit` is what the git pre-commit hook runs, and it calls installs the git pre-commit hook (the `make hooks` target shims to it); and
`script/check`; `script/install-precommit` installs the git pre-commit hook `script/projectname` (literally that filename) simply outputs the project's
(the `make hooks` target shims to it); and `script/projectname` (literally name. Scripts that need the name call `script/projectname` — e.g.
that filename) simply outputs the project's name. Scripts that need the name `script/docker` assembles its image tag from it — so those scripts stay
call `script/projectname` — e.g. `script/docker` assembles its image tag from byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
it — so those scripts stay byte-identical across all repos. Repo-type-specific `go mod tidy` verification in Go repos) belong in `script/precommit`, not in
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in the hook itself. Model scripts are at
`script/precommit`, not in the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the must document the provided scripts in an **Entrypoints** section (see the
README requirements below). README requirements below).
@@ -109,24 +104,18 @@ style conventions are in separate documents:
`lint` phase and a `test` phase, with the final stage depending on both so the `lint` phase and a `test` phase, with the final stage depending on both so the
image cannot be built unless they pass. For non-server repos the final stage image cannot be built unless they pass. For non-server repos the final stage
brings up a development environment; for server repos it is the runtime image. brings up a development environment; for server repos it is the runtime image.
The gate phases and the build stage start from their pinned base images and Dockerfiles install development prerequisites by running `script/bootstrap`
install what those images lack either inline, as the canonical Go `Dockerfile` rather than duplicating installs inline; COPY `script/` and the dependency
below does for `git`, or by running `script/bootstrap`, as the `prompts` manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before
repo's own `Dockerfile` does for its yarn packages. The development running it.
environment stage installs development prerequisites by running
`script/bootstrap` rather than duplicating its installs inline. A stage that
runs `script/bootstrap` COPYs `script/` and the dependency manifests
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is - **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
no separate lint file. `script/lint` and `script/test` each build one phase no separate lint file. `script/lint` and `script/test` each build one phase
and nothing else: and nothing else:
```sh ```sh
docker build --build-arg CACHEBUST="$(date +%s)-$$" \ docker build --no-cache --target lint -t "$(script/projectname)-lint" .
--target lint --output type=cacheonly . docker build --no-cache --target test -t "$(script/projectname)-test" .
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--target test --output type=cacheonly .
``` ```
**A stage that is not the last one in the file is built only when the final **A stage that is not the last one in the file is built only when the final
@@ -136,15 +125,10 @@ style conventions are in separate documents:
plain `docker build .` builds the last stage alone and exits 0 having linted plain `docker build .` builds the last stage alone and exits 0 having linted
and tested nothing. and tested nothing.
**The gate builds write no image.** With `--output type=cacheonly` the phase **Every `docker build` in `script/` is tagged**, here and in
runs and a failing step fails the build, but the result is not exported. `script/cibuild` and `script/docker`. An untagged build leaves a dangling
Nothing uses those images, and writing one out is slow: a Go test phase's image behind on every invocation, on every developer host and every CI
image holds the toolchain. A build given neither `--output` nor `-t` writes runner; a tagged one replaces the previous image.
an untagged image and leaves it dangling, on every developer host and every
CI runner. `script/cibuild` and `script/docker` build the image that ships
and tag it, so each build replaces the previous image; each assigns the tag
on its own line before the build, so `set -e` stops it where
`script/projectname` fails.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are `eslint`, `prettier` — never through `make lint` or `script/test`, which are
@@ -159,122 +143,66 @@ style conventions are in separate documents:
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own `TMPDIR`, so the lock is org, in both directions. A container has its own cache, its own `TMPDIR` and
out of reach. The lint phase does keep golangci-lint's cache between check a digest-pinned binary, so neither is reachable.
builds (below), but it lints every checkout at `/src`, and a finding is
taken from that cache only for the same files, settings and golangci-lint
version.
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker - **Any build that runs checks is built with `--no-cache`.** Docker invalidates
invalidates a `COPY` layer only when the copied content changes, so on an a `COPY` layer only when the copied content changes, so on an unchanged tree
unchanged tree the check `RUN` is served from cache, nothing executes, and the the check `RUN` is served from cache, nothing executes, and the build still
build still exits 0. Every `docker build` in `script/` therefore passes exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
id, so no two builds share a value, not even the three in one four, and there is no fifth — `script/check` runs the two gate phases and
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and `script/fmt-check`, and builds no image of its own. A bare `docker build .` is
`script/docker` are the four, and there is no fifth — `script/check` runs the not evidence that anything ran: a sub-second build reporting success is a
two gate phases and `script/fmt-check`, and builds no image of its own. In cache hit, not a result. Never invalidate by pruning — `docker builder prune`
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps and friends destroy a build cache shared with every other build on the host.
that install system packages and download Go's modules, or right after `FROM`
in a stage with neither, and every step that lints, tests or compiles comes
after it. A step that installs anything else as well, such as a
`script/bootstrap` that also installs node packages, comes after it too. A new
value reruns every `RUN` step after that line; `COPY` steps are compared by
content and need nothing. Docker keeps the layers above the line until the
base image, their own lines or the files they copy change. A stage without the
`ARG` line is served from cache on an unchanged tree, so a repository
re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
not pass `--no-cache`: it reruns every step too, but it also builds the layers
above the line again and empties every cache mount, so Go's build cache and
golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
survive between check builds, would start empty every time. A bare
`docker build .` is not evidence that anything ran: a sub-second build
reporting success is a cache hit, not a result. Never invalidate by pruning —
`docker builder prune` and friends destroy a build cache shared with every
other build on the host. When a check is added or changed, prove it works by
planting a defect it must catch and watching the run fail on it, then revert
the defect. A green run alone shows neither that the check ran nor that it
covers what it should.
- **The gate phases are separate stages, and the build stage depends on both.** - **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by The lint phase is based on the `golangci/golangci-lint` image (pinned by
hash), so lint failures surface in seconds rather than after a full compile, hash), so lint failures surface in seconds rather than after a full compile,
and the test phase is based on the Debian Go image. The canonical Go repo and the test phase is based on the Go image. The canonical Go repo
`Dockerfile`: `Dockerfile`:
```dockerfile ```dockerfile
# In every stage ARG CACHEBUST comes right after the steps that install
# system packages and download Go's modules, or right after FROM in a
# stage with neither, and every step that lints, tests or compiles
# comes after it. script/ passes a new value on every build, which
# reruns every RUN step after it. Docker keeps the layers above it
# until the base image, their own lines or go.mod and go.sum change.
# The steps that compile mount Go's build cache, and golangci-lint run
# also mounts golangci-lint's; both survive from build to build.
# Lint phase # Lint phase
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN golangci-lint run --config .golangci.yml ./...
--mount=type=cache,target=/root/.cache/golangci-lint \
golangci-lint run --config .golangci.yml ./...
# Test phase. -race needs cgo and so a C compiler, which the Debian Go # Test phase
# image ships and the alpine one does not. -count=1 because the mounted # golang:1.x-alpine, YYYY-MM-DD
# cache also holds test results, which go test would otherwise replay.
# golang:1.x, YYYY-MM-DD
FROM golang@sha256:... AS test FROM golang@sha256:... AS test
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY . . COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \ RUN go test -timeout 90s -race -cover ./... || \
go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; } go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies # Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so nothing after them can # are what make BuildKit build them first, so this stage cannot run
# run unless lint and test passed. # unless lint and test passed.
# golang:1.x-alpine, YYYY-MM-DD # golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder FROM golang@sha256:... AS builder
RUN apk add --no-cache git COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
ARG CACHEBUST
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
COPY . . COPY . .
# The VERSION build arg when one is given, otherwise ARG VERSION=dev
# `git describe --tags --always` on the .git in the build context. With RUN CGO_ENABLED=0 go build -trimpath \
# .git present, a version that is still empty, dev or unknown fails the
# build: git is missing or could not read the checkout.
ARG VERSION
RUN --mount=type=cache,target=/root/.cache/go-build \
VERSION="${VERSION:-$(git describe --tags --always)}"; \
if [ -e .git ]; then \
case "$VERSION" in ""|dev|unknown) \
echo "version is '$VERSION' although .git is present" >&2; \
exit 1 ;; \
esac; \
fi; \
CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \ -ldflags="-s -w -X main.Version=${VERSION}" \
-o /app ./cmd/app/ -o /app ./cmd/app/
# Runtime stage, and the last one # Runtime stage, and the last one
FROM alpine@sha256:... FROM alpine@sha256:...
ARG CACHEBUST
COPY --from=builder /app /usr/local/bin/app COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"] ENTRYPOINT ["app"]
``` ```
@@ -289,106 +217,23 @@ style conventions are in separate documents:
- Keep the runtime stage last, and if you add a stage after it, give it the - Keep the runtime stage last, and if you add a stage after it, give it the
same two copies. A plain `docker build .` builds the last stage's chain same two copies. A plain `docker build .` builds the last stage's chain
and nothing else. and nothing else.
- Check builds keep these, and none of them can hand back a stale result:
- The layers above `ARG CACHEBUST`: the system packages and Go's
modules. Docker installs the packages again when the base image or the
install line changes, and downloads the modules again when `go.mod` or
`go.sum` changes. Go checks every module against `go.sum` as it
downloads it, and no later build can change a kept layer. The modules
are kept in a layer rather than a cache mount because a cache mount is
shared by every build on the host, and Go builds from whatever files
it holds without checking them again (only `go mod verify` does), so a
file changed there would go unnoticed.
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
root), mounted on the steps that compile: `golangci-lint run`, both
`go test` runs and `go build`. Go reuses a compiled package only when
all its inputs are unchanged. The cache also holds test results, which
is why both `go test` runs carry `-count=1`.
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
a package's findings only when the package's files and those of every
package it imports, `go.mod`, the linters and their settings in
`.golangci.yml`, and the golangci-lint version are all unchanged.
- If the project uses `//go:embed` directives that reference build artifacts - If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must (e.g. a web frontend compiled in a separate stage), the lint phase must
create placeholder files so the embed directives resolve. Example: create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
- If the project requires CGO or system libraries for linting, install them - If the project requires CGO or system libraries for linting (e.g.
in the lint phase. The `golangci/golangci-lint` image is Debian-based and `vips-dev`), install them in the lint phase with `apk add`.
has no `apk`, so install with `apt-get` under the Debian package name - `ARG VERSION=dev` is declared in the stage that compiles and supplied by
(`libvips-dev`, where alpine says `vips-dev`), and delete the package `script/docker` and `script/cibuild`; no stage may call `git describe`.
lists in the same `RUN`, so the layer does not keep them. The test phase,
on the Debian Go image, installs the libraries its tests need the same
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
Docker keeps it until the base image or the install line changes:
```dockerfile
FROM golangci/golangci-lint@sha256:... AS lint
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
ARG CACHEBUST
```
- `.dockerignore` lets `.git` into the build context. It keeps out every git
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
repository's own, each submodule's under `.git/modules/`, and that of a
submodule keeping its own `.git` directory. `git describe` does not need
them, and each can hold a credential: a password in a remote URL, or the
token the CI checkout step stores there. A submodule whose name has a
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
git directory to `**/.git/modules/**/config`, and Go's version stamping
then fails the build: give it a name without that segment
(`git submodule add --name`). The stage that compiles has `git` (the
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
takes the version from the `VERSION` build argument when one is given,
otherwise from `git describe --tags --always`. That gives the tag on a
tagged commit; on a later commit, the tag, the number of commits since it
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
tag is reachable. The stage that compiles also marks its working directory
safe for git (`git config --system --add safe.directory /src`): a context
sent as a tar stream keeps the sender's file owners, and git refuses a
checkout owned by another user, so the version would come out empty.
`ARG VERSION` has no default, and the build fails if the context carries
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
`docker build .` with no build arguments must succeed; a Dockerfile that
refuses an empty build argument drops that refusal and keeps the argument.
A checkout whose `.git` is a file (a linked worktree, or a repository
checked out as a submodule) is the exception: that file points to a git
directory outside the build context, so the build cannot read the version
and a plain `docker build .` fails; pass the version with
`--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already
do.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
runs `script/cibuild` on push, and checks out the repo as its only other step, runs `script/cibuild` on push, and checks out the repo as its only other step.
with `persist-credentials: false`: `script/cibuild` needs no token, and That script bootstraps, runs the gate phases, and then builds the image, so a
without it the checkout leaves the job's token in `.git/config` for every successful run means every check passed; a bare `docker build .` does not
later step. The checkout step also sets `fetch-depth: 0`, which fetches the
tags `git describe` needs: by default it clones shallow with no tags, and a
tagged repository's CI build would stamp a bare short commit id. The
workflow's `concurrency` block groups runs by workflow and branch
(`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`,
so a new push cancels the older run on the same branch, queued or running, and
no other: runs for replaced commits do not hold up the shared runner.
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
so a successful run means every check passed; a bare `docker build .` does not
carry the same guarantee, because its gate phases may come from the cache. The carry the same guarantee, because its gate phases may come from the cache. The
image build gets a `CACHEBUST` value of its own and so runs the gate phases a image build is uncached and so runs the gate phases a second time. That is the
second time. That is the price of the rule above, and it is worth paying: the price of the rule above, and it is worth paying: the image that ships is built
image that ships is built from a run of its own gates rather than from a cache from a run of its own gates rather than from a cache entry.
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
shared runner after 20 minutes. That allows for the three Docker builds
described above (the test phase, the lint phase, then the image), each held to
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
limited to `main` by a `branches` list under `on: push` cannot be checked by
review: to try a change to it, add the feature branch to that list and push,
then remove the branch from the list again before merging. Keep any job in it
that publishes behind `if: github.ref_name == 'main'`, so the run from the
feature branch publishes nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for - Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -441,18 +286,17 @@ style conventions are in separate documents:
``` ```
`-count=1` is required on both invocations: it defeats Go's test _result_ `-count=1` is required on both invocations: it defeats Go's test _result_
cache, so neither run can report a stored pass in place of running the cache, so the target cannot report a pass it did not earn, and the rerun
tests. It leaves the build cache alone, so it costs the runtime of the suite reproduces a failure instead of replaying it. It leaves the build cache
and no recompilation. alone, so it costs the runtime of the suite and no recompilation.
That cache is Go's own, separate from Docker's layer cache. Go stores a Note that this is a second, independent cache, stacked below the Docker
passing result in its cache directory (`GOCACHE`), and when the same tests layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26)
run again on unchanged code it prints that result, marked `(cached)`, addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes;
without running them. The directory lasts from one run to the next on a it does not guarantee `go test` inside that step does any work, because the
developer's machine, where this target runs, and in the `test` phase of the `GOCACHE` baked into earlier image layers survives into the re-executed
`Dockerfile`, which mounts it to keep compiled packages between check step. They are two separate defects requiring two separate fixes, and a fix
builds, so the phase's `go test` runs carry `-count=1` too. A new for one must not be recorded as covering the other.
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
Python example: Python example:
@@ -480,12 +324,9 @@ style conventions are in separate documents:
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
`node_modules/`, and the repo's own build outputs. Fetch the standard language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
`.gitignore` from from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up setting up a new repo. These patterns are written to `.gitignore`'s own
a new repo. A repo's `.gitignore` is the standard file followed by the repo's
own entries, such as its binaries; a re-vendor replaces the standard part and
keeps those entries. These patterns are written to `.gitignore`'s own
semantics, in which an unanchored pattern already matches at every depth; they semantics, in which an unanchored pattern already matches at every depth; they
are not a `.dockerignore` and must not be transplanted into one unmodified. are not a `.dockerignore` and must not be transplanted into one unmodified.
@@ -499,7 +340,7 @@ style conventions are in separate documents:
— which is more dangerous than a short file with no secret patterns at all, — which is more dangerous than a short file with no secret patterns at all,
because it reads as solved and stops anyone looking. Give every because it reads as solved and stops anyone looking. Give every
depth-independent pattern the `**/` prefix and leave only genuinely depth-independent pattern the `**/` prefix and leave only genuinely
root-anchored entries unprefixed: `.claude`, and the repo's own host-built root-anchored entries unprefixed: `.git`, and the repo's own host-built
binary, written `/myapp` and never `**/myapp`, which would also match binary, written `/myapp` and never `**/myapp`, which would also match
`cmd/myapp/` and delete the package directory from the context. Matching is `cmd/myapp/` and delete the package directory from the context. Matching is
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
@@ -524,24 +365,21 @@ style conventions are in separate documents:
directory, so a repo running agents in subdirectories still ships directory, so a repo running agents in subdirectories still ships
`services/api/.claude/` and must add its own anchored entry there. `services/api/.claude/` and must add its own anchored entry there.
- **A plain `docker build .` of a clone stamps the version that - **Excluding `.git` means `git describe` cannot run inside any build stage, and
`git describe --tags --always` gives**, derived from the `.git` in the build it fails quietly there.** In a build stage there is no repository, so
context as the canonical `Dockerfile` above shows. Without its failure check, `git describe` writes nothing to stdout, `-X main.Version=` comes out empty,
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty the binary reports no version at all, and the build still exits 0. Compute the
and the build would still exit 0. `script/docker` and `script/cibuild` pass version on the host and thread it in as a build arg. `script/docker` and
the version they compute on the host; it takes precedence. They do this `script/cibuild` do this, byte-identically across repos:
byte-identically across repos:
```sh ```sh
# The version and the tag each get their own line: a failing command # Own line: a failing command substitution inside an argument does not
# substitution inside an argument does not trip `set -e`, so the inline # trip `set -e`, so the inline form degrades to an empty constant.
# form degrades to an empty constant.
version="$(git describe --tags --always --dirty 2>/dev/null || true)" version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown" [ -n "$version" ] || version="unknown"
tag="$(script/projectname)" docker build --no-cache \
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
--build-arg VERSION="$version" \ --build-arg VERSION="$version" \
-t "$tag" . -t "$(script/projectname)" .
``` ```
`--always` makes an untagged repo yield an abbreviated commit hash rather `--always` makes an untagged repo yield an abbreviated commit hash rather
@@ -549,12 +387,12 @@ style conventions are in separate documents:
fallback is applied — a live check that fires on a build from an export with fallback is applied — a live check that fires on a build from an export with
no `.git` and on a repository with no commits yet. Do not fold it into the no `.git` and on a repository with no commits yet. Do not fold it into the
substitution as `|| echo unknown`, which makes the guard unreachable. The substitution as `|| echo unknown`, which makes the guard unreachable. The
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
the scripts stay byte-identical. One consequence for CI: the standard the scripts stay byte-identical. One consequence for CI: the standard
checkout action clones shallow and fetches no tags, so the canonical checkout action clones shallow and fetches no tags, so a repo that embeds a
`.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step. tag-derived version must set `fetch-depth: 0` on its checkout step.
- **Verify `.dockerignore` by enumerating the image, not by reading the - **Verify `.dockerignore` by enumerating the image, not by reading the
patterns.** Plant files at the root _and_ at least two directories deep, build patterns.** Plant files at the root _and_ at least two directories deep, build
@@ -588,18 +426,12 @@ style conventions are in separate documents:
`test-support` depguard rule, where a repo names its own test-support packages `test-support` depguard rule, where a repo names its own test-support packages
by full import path. A repo adds entries there and changes nothing else, and a by full import path. A repo adds entries there and changes nothing else, and a
re-vendor carries its entries forward. The canonical golangci-lint version is re-vendor carries its entries forward. The canonical golangci-lint version is
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base
image image
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`,
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the
directive must not name a newer Go minor version than the one golangci-lint only pin, since no repo installs golangci-lint on the host: bumping the
was built with, or golangci-lint refuses to lint it: this release lints version means changing it and nothing else.
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
installs golangci-lint on the host. A repo sets the lint phase digest to the
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
the two prompted the change: the canonical copy can name linters that an older
golangci-lint rejects, and a newer golangci-lint can add linters that
`default: all` switches on until the canonical copy disables them.
- **`script/bootstrap` installs a pinned tool by comparing versions, never by - **`script/bootstrap` installs a pinned tool by comparing versions, never by
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
@@ -621,18 +453,8 @@ style conventions are in separate documents:
function defined and never invoked has the same exit status and the same function defined and never invoked has the same exit status and the same
empty output as one that worked. empty output as one that worked.
Node is the exception to the whole-token comparison: the canonical
`script/bootstrap` compares only its major version, because the node 22
alpine image the `Dockerfile` stages start from is not the exact pin, and
nvm cannot install a prebuilt node on alpine.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
A Go tool a repo needs on the host is installed with `go install` pinned to
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
a `go.mod` tool dependency or through a `tools.go` file, either of which
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
- When pinning images or packages by hash, add a comment above the reference - When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD). with the version and date (YYYY-MM-DD).
@@ -742,17 +564,13 @@ style conventions are in separate documents:
Never edit existing migrations after release. Never edit existing migrations after release.
- All repos should have an `.editorconfig` enforcing the project's indentation - All repos should have an `.editorconfig` enforcing the project's indentation
settings: the standard file from settings.
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets
tabs for `Makefile` and Go files, followed by the repo's own sections, such as
one for another language it uses. A re-vendor replaces the standard part and
keeps those sections.
- Avoid putting files in the repo root unless necessary. Root should contain - Avoid putting files in the repo root unless necessary. Root should contain
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
and language-specific config). Everything else goes in a subdirectory. language-specific config). Everything else goes in a subdirectory. Canonical
Canonical subdirectory names: subdirectory names:
- `bin/` — executable scripts and tools - `bin/` — executable scripts and tools
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
body is a single call into `internal/` or `pkg/`, no project logic in body is a single call into `internal/` or `pkg/`, no project logic in
@@ -783,7 +601,3 @@ style conventions are in separate documents:
- Go: `go.mod`, `go.sum`, `.golangci.yml` - Go: `go.mod`, `go.sum`, `.golangci.yml`
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
- Python: `pyproject.toml` - Python: `pyproject.toml`
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
is never committed under a file or directory named after one agent tool, such
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
+10 -598
View File
@@ -18,7 +18,7 @@
pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work lands on pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work lands on
`next`, and `main` receives only the milestone PR that the owner merges. `next` `next`, and `main` receives only the milestone PR that the owner merges. `next`
is at the canonical `golangci-lint` v2.14.0 config and is green. Recent work is at the canonical `golangci-lint` v2.12.2 config and is green. Recent work
extracted the internal/magic, internal/allowlist, internal/httpfetcher, and extracted the internal/magic, internal/allowlist, internal/httpfetcher, and
internal/signature packages. The gosec findings from the 2026-07-06 survey are internal/signature packages. The gosec findings from the 2026-07-06 survey are
resolved. The disk cache is now size-bounded with LRU eviction resolved. The disk cache is now size-bounded with LRU eviction
@@ -30,592 +30,13 @@ P2: security: per-origin rate limiting
# Completed Steps # Completed Steps
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
whose info it cannot read, and goes on (closes #343). It stopped before its
later checks at a directory it could not list, or at a source or variant file
whose info it could not read.
- It logs the warning "skipped unreadable entry in cache directory", with
the path and the error.
- A file renamed or removed after its directory was listed gets no warning.
- `reconcileFiles` reads each file's info once, before the temp-file check,
and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
`reconcileSourceFile`.
- A cancelled context still stops the walk with its error.
- 2026-10-26 the reconciliation pass drops a variant's accounting row only when
its file is missing (closes #340). Any other error reading the file's size
keeps the row. It dropped the row on any error and logged the file as missing.
- The kept row gets the warning "kept accounting row for unreadable variant
file", with the variant key and the error.
- The pass checks for `ErrNotFound`, which `FileStore.Size` returns for a
missing file.
- 2026-10-26 on both image routes, a request is answered 499 and logged at info
level when its client goes away before its image is ready, unless it is the
request making the image (closes #339). It was answered 500 and logged at
error level, as pixa's own failure.
- The line is "client went away before the image was ready". It carries the
`request_id`, the host and the path.
- The 499 has no body, since nobody reads it.
- `serveImage` answers it before calling `respondImageError`, which cannot
see the request's context.
- An error wrapping `context.Canceled` while the request's context is live
is still answered 500 and logged at error level.
- 2026-10-26 `README.md`'s Design section starts with a Concepts list (closes
#321). Each concept gets one sentence, under the name the code gives it.
- Storage calls the transformed images variants, in shorter sentences.
- Storage's paths end in the source key and the variant key.
- Storage lists the image cache's tables in the database.
- Architecture names `internal/providers`, and lists the packages that work
with the concepts.
- 2026-10-26 one table in `internal/config` lists every setting once (closes
#320). `Config.settings` gives each setting's key, its environment variable
and the function that loads it.
- Most rows load through `field`, whose arguments give the setting's type,
its default and its check.
- `db_url` and `cache_max_bytes` have their own load functions. The default
of `db_url` comes from `state_dir`, and `cache_max_bytes` records whether
it is set.
- Loading, the unknown-key check, the unknown-variable check and the names
in error messages all read the table.
- `allowlist_hosts` loads as `referer_blocklist` does, through
`parseHostList`.
- The metrics username and password are checked together once the table has
loaded.
- When a config holds more than one invalid value, the error names the one
that comes first in the table.
- `README.md` lists the settings one item each, where it had a table. A test
checks that this list and `configs/config.example.yml` name exactly the
keys of the table.
- The key constants moved to the tests, their only users.
- Removed:
- `isKnownConfigKey`, `envVarNames`, `strictLoader` and `validate`
- `validateAllowlistHostsValue` and `getStringSlice`, with the three
tests of `getStringSlice`
- 2026-10-26 a fetched image's format is detected from its bytes once, as an
`imagerequest.Format` (closes #319). The upstream `Content-Type` check in the
fetcher stays as the filter before downloading.
- `magic.DetectFormat` returns an `imagerequest.Format`, and
`magic.ValidateMagicBytes` returns the format it checked against the
`Content-Type`.
- `Format.MIMEType` gives each format's MIME type. The fetcher, the mock
fetcher, the cache's read of a variant's type and the handler's `Accept`
matching use it.
- `Cache.Source` returns a source's format, read from its file's first bytes
as `Cache.Variant` reads a variant's. A file in no source format is an
error, and the proxy fetches the source again.
- The proxy hands the format to `ImageProcessor.Process`. The processor
refuses input that libvips would load as another format, as before, with
`ErrUnsupportedInputFormat`. It refuses a format that is not one of the
six source formats the same way.
- `internal/magic` no longer detects SVG, which pixa does not read. SVG data
is `ErrUnknownFormat`.
- Removed:
- `magic.MIMEType`, `magic.ImageFormat` and `magic.MIMEToImageFormat`
- `detectFormat`, `formatFromString`, `FormatToMIME` and
`Result.InputFormat` of the processor
- the fetcher's MIME constants of the six formats
- `errNotAnOutputFormat` of the cache
- 2026-10-26 the cache keeps its source images in one `imgcache.FileStore` and
its variants in another (closes #318). A `FileStore` is a directory of files,
each named by a hex key, at `<ab>/<cd>/<key>`.
- `ContentStorage` and `VariantStorage` are gone.
- `FileStore` has these methods:
- `Write`, which writes a file under a temp name, syncs it and renames
it into place
- `Open`, which returns the file and its size
- `Size`
- `Delete`
- `FileStore` has no `Exists`. The reconciliation pass checks that a
variant's file is there with `Size`. The tests look for the file on disk.
- `Cache.Variant` reads the content type from the variant file's first
bytes, as `VariantStorage.LoadWithContentType` did. A cache hit still
makes no database call.
- `storage.go` is now `filestore.go`. `internal/imgcache/testdata/red.avif`,
which no test used, is gone.
- 2026-10-26 `internal/imgcache` holds the cache only (closes #317). The new
package `internal/imageproxy` fetches, transforms, caches and serves images.
It imports `imgcache`, and `imgcache` does not import it.
- `imgcache.Service` is `imageproxy.Proxy`, built by `imageproxy.New` from
an `imageproxy.Config`.
- `ImageResponse`, `CacheStatus`, `ErrNegativeCached` and
`ErrProcessingTimedOut` moved with it, under their names.
- `imgcache.CacheConfig` is `imgcache.Config`.
- The cache has one read and one write for each kind it keeps. Each takes
the place of an earlier method:
- `Variant`, of `GetVariant`
- `PutVariant`, of `StoreVariant`
- `Source`, of `LookupSource` followed by `GetSourceContent`
- `PutSource`, of `StoreSource`
- `Failure`, of `checkNegativeCache`
- `PutFailure`, of `StoreNegative`
- `Variant` and `Source` each open the file and hand its access time to the
background writer. A cache hit still makes no database call.
- `imgcache.go` and `module.go` are gone.
- `imageproxy.New` makes the counters of upstream fetches and transforms,
and registers them on the registry it is given. The cache keeps only the
hit and miss counters.
- A cached source whose row cannot be read is logged as "failed to load
cached source, fetching". It was logged as "source lookup failed".
- The proxy's tests moved to `internal/imageproxy`. They reach the cache
through its database, its state directory and its metrics registry.
- The tests of an empty or cut cached file are split in two:
- `internal/imgcache` checks that the reconciliation pass removes the
file and its row, with a warning naming the file.
- `internal/imageproxy` removes the file and its row as the pass does,
and checks that `Get` fetches or converts again.
- 2026-10-26 both image routes serve an image with one function, `serveImage`,
and answer its errors with one, `respondImageError` (closes #316).
`HandleImage` parses the path and verifies the signature, and `HandleImageEnc`
decrypts the token. Each then calls `serveImage`.
- `handleImageError`, the encrypted route's own mapping, is gone.
- `/v1/image/` answers 504 when the upstream has not sent its response
headers within `upstream_fetch_timeout`, as `/v1/e/` did. It answered 500.
- `respondImageError` logs the image service's errors as "failed to get
image". Each answer gets the same level on `/v1/image/` and on `/v1/e/`:
- 500: error, as pixa's own failure
- 400: warn
- 403: warn
- 415: warn
- 502: warn
- 503: warn
- 504: warn
- Before, `/v1/image/` logged all of these answers at error level. `/v1/e/`
logged only its 500, as "image request failed", without the host and the
path.
- `respondImageError` does not log the 415 for a body refused for its first
bytes, on either route. The image service logs it as "source refused", at
warn level.
- The "image served" line carries the host, the path and `fetched_bytes` on
both routes. The debug line "encrypted image request" is gone.
- The debug line "failed to decrypt URL" is now "invalid encrypted URL". A
token that decrypts to a payload that does not decode also gets it.
- A failed write is logged once, as "failed to write response" at warn
level, since most often the client has gone away. It carries the
`request_id`, the host, the path, `served_bytes` and the error, and is not
also logged as "image served".
- 2026-10-26 every key derived from the signing key is derived in one place
(closes #315): `seal.DeriveKeys` returns them as one `seal.Keys`, with their
salts declared beside it. `internal/providers` calls it once at startup. Each
key is derived as before, with `seal.DeriveKey` and the same salt.
- The session manager, the encrypted URL generator and the CSRF protection
each take their own key instead of the signing key.
- `session.NewManager`, `encurl.NewGenerator`, `newCSRFProtect` and
`handlers.New` return no error, as nothing in them can fail now.
- The signing key itself stays the HMAC key of signed URLs and the login
password.
- `README.md` says under `signing_key` what it protects.
- 2026-10-26 the image request holds only what names the cached object (closes
#314): its source and its transform, which the variant key covers. The signed
URL format, the signing algorithm and every route's answers are unchanged.
- The `/v1/image/` handler checks `allowlist_hosts` and verifies the
signature itself, with `Signer.Verify(req, sig, expires)`.
- Each image handler keeps its URL's expiry as a value of its own, for
`Cache-Control`. `encurl`'s `Generate` takes the expiry and its `Parse`
returns it.
- The fetcher takes the source and builds its URL with `Source.URL`, over
http only when the fetcher's own `AllowHTTP` is set. `validateURL` takes
that URL, so no URL string is built and parsed again.
- `chooseAutoFormat` returns the format it chooses, and the handler puts it
into the request it parsed.
- The image service serves images through `Get`, and has `WaitForProcessing`
for shutdown.
- Removed:
- `Signature`, `Expires`, `AllowHTTP` and `SourceURL` of the image
request
- `Service.ValidateRequest`, and the service's signer, allowlist and
`AllowHTTP`
- `extractHost` of the fetcher and `urlToFSPath` of the mock fetcher
- 2026-10-26 pixad's fx graph builds the image pipeline (closes #313).
`database.New` now opens the database, so the parts below can read it while
they are built; its stop hook still runs after the cache's. The cache's
`MaxBytes` is the one value for the disk cache limit, and 0 turns the disk
cache off. For an omitted `cache_max_bytes`, `internal/providers` works out
`config.DefaultCacheMaxBytes` before it builds the cache. `internal/app` and
the server tests build the parts below with `providers.Constructors()`; the
image proxy test replaces only the fetcher's dial function.
- Built by the providers in `internal/providers`, and given to
`handlers.New`:
- the cache
- the fetcher
- the processor
- the image service
- the session manager
- the encrypted URL generator
- Removed:
- `initImageService` and `handlers.Params.Fetcher`
- `CacheConfig.UseDefaultMaxBytes`, `CacheConfig.DisableDiskCache` and
`Cache.disabled`
- `httpfetcher.DefaultConfig()` and the fetcher's copies of setting
defaults
- `imageprocessor.DefaultMaxInputBytes`, and the processor's defaults
for a zero `MaxInputBytes` or `MaxConcurrentProcessing`
- 2026-10-26 the code nothing in production called is gone (closes #312). The
handlers tests sign their image URLs with `generateSignedURL`, a test helper
that writes them as the removed generator did. Removed:
- `Service.Warm` and the `ImageCache` interface
- `Service.GenerateSignedURL` and `Signer.GenerateSignedURL`
- `signature.ParseParams`
- `imagerequest.ParseImagePath`
- `PeekAndValidate`, `IsSupportedMIMEType`, `ImageFormatToMIME` and
`FormatOriginal` of `internal/magic`
- `SupportedInputFormats` and `SupportedOutputFormats` of the image
processor
- `IsEmpty` and `Count` of the allowlist
- `ssrfSafeDialer`
- `CacheStale`, `ErrCacheMiss`, `ErrNegativeCache` and
`ImageResponse.LastModified`
- `ContentStorage.Load` and `VariantStorage.Load`
- 2026-10-26 cache hits, misses, upstream fetches, the bytes those fetches read
and transforms are counted as Prometheus counters in memory (closes #311).
They are `pixa_cache_hits_total`, `pixa_cache_misses_total`,
`pixa_upstream_fetches_total`, `pixa_upstream_fetch_bytes_total` and
`pixa_transforms_total`, fields of the cache. `NewCache` makes them and
registers them on the registry it is given (`internal/imgcache/counters.go`).
pixad gives it Prometheus' default registry, which `/metrics` serves; each
test gives it a new one. The `Increment` methods add to their cache's
counters. `cache_stats`, `Cache.Stats`, `Service.Stats`, `CacheStats` and the
pending counts are gone. The background goroutine writes only access times
now: `StartPendingAccessTimeWrites` starts it and
`StopPendingAccessTimeWrites` stops it, in `pendingaccesstimes.go`.
- 2026-10-21 the schema file declares only the tables and columns pixa writes
and reads (closes #310): `output_content` and `request_cache` are gone, and so
are the columns nothing read: the host, path, query, `content_type` and
`fetched_at` of `source_content`, the `content_type` and `created_at` of
`variant_content`, and the `status_code`, `error_message` and `fetched_at` of
`negative_cache`. `StoreSource` no longer takes the fetch result,
`LookupSource` returns only the source key, `StoreVariant` no longer takes a
content type, `StoreNegative` no longer takes a status code or message, and
reconciliation adopts a variant file without reading its first bytes.
- 2026-10-21 a cache hit makes no database call (closes #309): `Service.Get`
opens the variant file with `Cache.GetVariant` and nothing else, with no
existence check first. The content type comes from the file's first bytes; a
file whose first bytes are not those of an output format, SVG included, is a
miss, and the variant is made again. A hit hands its access time to a channel
that holds up to 10,000 and drops the rest, and its hit count to the pending
counts, without waiting; every count is now a pending count. A miss that finds
a cached source hands that source's access time to a second such channel. The
goroutine `StartPendingCountWrites` starts writes, once a second, the access
times in one transaction and the counts in one `UPDATE`;
`StopPendingCountWrites` writes what is left once at shutdown and logs
separately the access times and the counts it could not write.
`checkNegativeCache` no longer deletes an expired row; `CleanExpired` in the
eviction goroutine does. The `.meta` files, `VariantMeta`, `LoadWithMeta`,
`variantContentTypeFromSidecar`, `metaCache`, `Lookup` and `golang-lru` are
gone. `make bench` (`script/bench`) runs `BenchmarkHandleImage_CacheHits` with
Go on the host after `script/bootstrap --cgo`, outside `make check`.
- 2026-10-21 a source is stored the way a variant is (closes #308): one
`source_content` row and one file per source URL, the file named by the
source's `SourceKey`, the SHA-256 of its host, path and query made as the
variant key is, under `cache/sources/`. A store writes the file, then the row,
and removes the file when the row cannot be written; an eviction deletes the
row, then the file, with no lock and no transaction. `source_metadata`, the
JSON files under `cache/metadata/`, `MetadataStorage` and the content lock are
gone. Two source URLs that serve the same bytes now store them twice.
- 2026-10-21 one image request type instead of five (closes #307): the new
package `internal/imagerequest`, which imports no other package of pixa, holds
`imagerequest.Request` with its two parts, `Source` (host, path and query) and
`Transform` (size, format, quality and fit), the `Format` and `Fit` types, the
checks, the parsing of the `/v1/image/` path and the variant key.
`imgcache.ParsedURL`, `imgcache.ImageRequest`, `signature.Request`,
`imageprocessor.Request`, the second `Size`, format and fit types and the
conversions between them are gone. The image processor takes a `Transform`
only, and `encurl` keeps a private struct with the token's short CBOR field
names, so tokens made before still decode. Every URL of both image routes
answers as before.
- 2026-10-21 the Gitea workflow checks out the repo and runs `script/cibuild`,
nothing else (closes #203): its `script/docker-smoke` step is removed, as the
canonical `REPO_POLICIES.md` says, since `make check` covers what CI needs to
check. CI no longer starts the image and waits for its healthcheck.
`script/docker-smoke` and `make docker-smoke` stay, to run by hand.
- 2026-10-20 a WebP, AVIF or JPEG XL encode at the limits takes at most about 34
seconds (closes #272): with one libvips thread, 8192x8192 random pixels with
an alpha channel took about 61 seconds to encode as WebP, 81 as JPEG XL and,
at quality 85, 96 as AVIF, so with the default `downstream_timeout` of 60
seconds such an image was never delivered, yet its encode held a processing
slot to the end. WebP is now saved at effort 2 and JPEG XL at effort 4, which
take up to about 23 and 27 seconds at any quality. AVIF was already at the
lowest effort govips can set, so an AVIF output of more than 16,777,216 pixels
is refused with 400 before anything is decoded; 4096x4096, 8192x2048 or
2048x8192 takes up to about 34 seconds at any quality. Resizing with
`fit=cover` already decodes the input, so the output's dimensions are worked
out before resizing. `README.md` gives the limit and the new times.
- 2026-10-20 a source whose first bytes do not match its `Content-Type` answers
415 (closes #253): a body whose first bytes are not those of the format its
`Content-Type` names, such as a JPEG sent as `image/png`, text, or a body
under 12 bytes, answered 500 on both image routes and was logged as an error
of pixa's, and each request for it fetched the whole body again. Both routes
now answer 415, and the image service logs it at warning level as a refused
source. It also gets a negative cache entry, as an upstream error status does,
so the source URL is not fetched again for 5 minutes, and meanwhile an image
that would need that fetch answers 502.
- 2026-10-20 `AGENTS.md`: a bug fix may change an existing test without asking
when that test pins exactly the behavior the filed bug calls wrong, and the
change moves it to the fixed behavior and nothing else (closes #280).
- 2026-10-18 re-vendored from `sneak/prompts` commit `0653579` (closes #221):
each `Dockerfile` stage installs its system packages and downloads the Go
modules above `ARG CACHEBUST`, so check builds keep them, and the lint step
keeps golangci-lint's cache.
- 2026-10-19 a `trusted_proxies` entry in the IPv4-mapped form trusted nothing;
it is now read as the IPv4 range it covers by the same code that reads
`blocked_networks`, and such an entry shorter than `/96` aborts startup
(closes #303).
- 2026-10-18 the next reconciliation pass waits at least ten times as long as
the last one took, so passes on a large cache no longer run back to back
(closes #250).
- 2026-10-18 a `blocked_networks` entry in the IPv4-mapped form refused nothing;
`::ffff:10.0.0.0/104` is now read as `10.0.0.0/8`, and such an entry shorter
than `/96` aborts startup (closes #301).
- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal
addresses refuses the local-use NAT64 range (`64:ff9b:1::/48`, RFC 8215) as it
refuses `64:ff9b::/96`, and refuses an IPv6 address in the IPv4-compatible
form (`::/96`) when `blocked_networks` contains either the address as written
or the IPv4 address it holds (closes #299). A source host such as
`[64:ff9b:1::a00:1]` was fetched, which on a network that runs NAT64 on that
range reaches the IPv4 address it holds, here `10.0.0.1`; and with
`203.0.113.0/24` in `blocked_networks`, `[::ffff:203.0.113.5]` was refused but
`[::203.0.113.5]` was fetched. Both now answer 403. The mapped form
(`::ffff:a.b.c.d`) is still matched only as the IPv4 address it holds: Go
keeps a plain IPv4 address in that form, and an IPv6 entry such as `::/0` must
not refuse every IPv4 address.
- 2026-10-16 P1 security bug: the check that keeps pixa from fetching internal
addresses judges an IPv6 address in the IPv4-compatible form (`::/96`), such
as `::127.0.0.1` or `::10.0.0.1`, by the IPv4 address in its last four bytes
(closes #297). It read only the mapped form (`::ffff:a.b.c.d`) as IPv4, so a
source host such as `[::127.0.0.1]` was checked as a public IPv6 address and
fetched, which on a network that routes it reaches the internal IPv4 host; it
is now refused with 403. The NAT64 form (`64:ff9b::/96`) was already refused
as a whole, as one of the built-in blocked ranges, and still is.
- 2026-10-16 a source host that is an IPv6 address in brackets with no port,
such as `[2001:db8::a]`, is fetched like one with a port, through `/v1/image/`
and through an encrypted URL (closes #295). The fetcher's check of the source
URL kept the brackets when no port followed them, could not look the host up,
and the request answered 500. It now checks the address without brackets, so a
private one is refused with 403, as with a port.
- 2026-10-16 a `state_dir` whose name holds `%`, `?` or `#` keeps its database
at `state.sqlite3` in that directory (closes #290): the `db_url` derived from
it put the name into a `file:` URL as it was, so SQLite read `%41` as `A`, a
`?` or `#` ended the path early, and the journal mode parameter was lost. The
derived `db_url` is now built with Go's `net/url`, which writes those
characters as `%25`, `%3F` and `%23`. For a `state_dir` of `/` or one starting
with `//` it starts `file:////`, so SQLite no longer reads the first name in
the path as a host.
- 2026-10-16 a size with one side `0` keeps the side computed from the source's
proportions at 1 or more (closes #283): it was rounded down, so for a thin
source it came out at 0, and libvips then made the size square. An 8192x1
source at `100x0` came out at 100x100, and at `1300x0` with `fit=cover` made
libvips ask for about 39 GB at once; an 8192x2 source at `100x0` answered 500.
`fit=contain` and `fit=inside` work both sides out again from the smaller of
the two scales and `fit=outside` from the larger, and each keeps both sides at
1 or more too, so a 1x8192 source at `0x100` with `fit=contain`, and a 49x8192
source at `0x100` or a 49x49 source at `1x1` with `fit=outside`, no longer
answer 500. An 8192x1 source at `1300x0` now comes out at 1300x1, and the
process running it alone peaked at about 34 MiB resident. With `fit=outside`
the image is scaled to cover the size, so that source at `100x0` comes out at
8192x1. `README.md` now says this under Signature Specification, and says how
a `0` side is worked out where it said `0` meant the original.
- 2026-10-16 the generator page lower-cases the source host and writes an empty
source path as `/` before it makes the encrypted URL, as `/v1/image/` does
(closes #293). A host written in upper and in lower case, or a source URL with
no path and with the path `/`, made two cache keys and two upstream requests
for one image.
- 2026-10-16 the source URL a request fetches is built with `net/url` from the
source host, path and query, not joined as text (closes #288). The source
path, decoded, is fetched percent-encoded again, so `100%25.png` fetches
`/100%25.png` where it answered 401, and `my%20cat.png` and `my%2520cat.png`
fetch different paths where both fetched `/my%20cat.png` under two cache keys.
The source query stays as the source URL writes it; the characters a browser
percent-encodes in a query, such as `'` or a space, which was sent raw and
answered 502, are percent-encoded when the image URL is read, so the query is
signed and cached as it is sent. The generator page answers 400, showing the
URL pixa would fetch, for a source URL its image URL would fetch written
another way, such as with `%2F` in the path, whatever else the path holds, or
with a user name and password. It also answers 400 for a source path holding a
`..` segment, as `/v1/image/` refuses one.
- 2026-10-15 a thin source requested with `fit=cover` no longer makes libvips
build a huge image (closes #282): libvips' thumbnail scaled the whole source
to cover the requested size and copied that into memory before it cut out the
centre, so an 8192x64 source at 8192x8192 made it allocate about 24 GiB, and
an 8192x1 source answered 500. The image processor now first cuts the centre
of the source, widened to whole source pixels, and thumbnail scales and cuts
only that, by the same amount on both sides as before. What it copies into
memory is then larger than the output by less than one scaled source pixel on
each side. With the runtime image's libvips and JPEG output at 8192x8192,
resident memory peaked at about 242 MiB for the 8192x64 source and 431 MiB for
the 8192x1 one. The output can sit a fraction of an output pixel from where it
was.
- 2026-10-15 a `/v1/image/` URL whose source path or source query holds a `#`,
written as `%23`, is refused with 400 (closes #286): the source URL built from
it ended at the `#` when fetched, so each text after it fetched the same
upstream image, without a signature for an allowlisted host, and cached it
under keys of its own. `README.md` says neither may hold a `#`.
- 2026-10-15 the host of a `/v1/image/` URL must be a host name or an IPv4
address (letters, digits, hyphens, underscores and dots) or an IPv6 address in
brackets, either one optionally followed by `:` and a port, and any other host
is refused with 400 (closes #275): only `@` was refused, so a `?` written as
`%3F` gave the host `cdn.example.com?x=1`, which Go's URL parser reads as
`cdn.example.com` with a query. Such a URL passed the allowlist, and each text
after the `?` fetched the host's root path and cached the answer under keys of
its own. A `#`, `%`, `\` or space written percent-encoded is refused the same
way, and so is an IPv6 address without brackets, which needed a signature
before. Underscores are kept because `allowlist_hosts` accepts host names
holding them.
- 2026-10-15 the README says how a source URL with a query is written into an
image URL, and the image route accepts it (closes #255): the query follows the
source path after `%3F`. pixa now reads the image URL's path decoded once, so
the `?` before the query, any `?` in it and every `%` in the source path or
query must be percent-encoded, and any other character may be. The route used
to parse chi's wildcard, which holds the path as the client wrote it whenever
that is not Go's own encoding of it; a `/`, `;` or `,` in the query written as
`%2F`, `%3B` or `%2C`, as `GenerateSignedURL` wrote them, kept the `%3F` from
being read as `?`, and the request failed with 401. `GenerateSignedURL` now
writes Go's own encoding of the path. The README has an example with a query
and its signature. The path traversal check no longer reads the source query,
and keeps a `%` that does not start a percent-escape as it is instead of
refusing the URL, so a source path or query holding `%25` is accepted, while a
double-encoded `..` next to such a `%` is still refused.
- 2026-10-15 `max_concurrent_processing` defaults to half the CPUs pixa sees,
rounded down, and at least 1 (closes #242): it defaulted to one image per CPU,
and a container with no CPU limit sees every CPU of the host, so on a 48-CPU
host pixa processed 48 images at once, each of which can take about 4.2 GiB.
`config.DefaultMaxConcurrentProcessing` gives the default to the config and to
`imageprocessor.New`; `PIXA_MAX_CONCURRENT_PROCESSING` still overrides it.
"Running under upaas" in `README.md` gives the worst-case memory, measured,
and tells the operator to set `PIXA_MAX_CONCURRENT_PROCESSING` and
`PIXA_UPSTREAM_CONNECTIONS` to fit the container's memory. It also gives the
file libvips writes to `/tmp` for each large source, up to 1 GiB, measured,
which is outside `/var/lib/pixa` and `cache_max_bytes` and is memory where
`/tmp` is a tmpfs.
- 2026-10-15 the canonical files are re-vendored from `sneak/prompts` commit
`e3f79c4` (closes #221): `.dockerignore`, `.editorconfig`,
`.gitea/workflows/check.yml`, `.gitignore`, `.golangci.yml`,
`.prettierignore`, `.prettierrc`, `REPO_POLICIES.md` and the standard scripts
are its copies. pixa's own entries follow the standard part: `/bin`, `/data`
and the local config files in `.dockerignore`; Go's `*.log`, `*.out` and
`*.test`, `vendor/`, the binaries, `/data/`, `*.sqlite3` and the local config
files in `.gitignore`. The workflow keeps its `script/docker-smoke` step while
https://git.eeqj.de/sneak/pixa/issues/203 is open. The scripts pass a new
`CACHEBUST` build argument in place of `--no-cache`, and every `Dockerfile`
stage declares it first, so every `RUN` step still runs on every build while
Go's build cache, mounted on the steps that compile, is kept; both `go test`
runs keep `-count=1`. The lint and test builds write no image. The lint phase
runs golangci-lint v2.14.0, the test phase runs on the Debian Go image, and
the build stage marks `/src` safe for git. `script/bootstrap`, `script/fmt`
and `script/fmt-check` use the node on the `PATH` only when it has the pinned
major version. Of golangci-lint v2.14.0's two findings, an unused `nolint`
directive is removed, and gosec's new path check is marked as not applying to
the fixed config file locations. The agent guidance moves unchanged from
`CLAUDE.md` to a root `AGENTS.md`, where `REPO_POLICIES.md` keeps it.
- 2026-10-15 the request metrics record any method that is not a standard one as
`other` (closes #274): their `method` label was each request's method as sent,
and Go's HTTP server accepts any method name, so a client sending made-up
methods added series to the Prometheus registry that were never removed. The
label is now the request's method only for `GET`, `HEAD`, `POST`, `PUT`,
`PATCH`, `DELETE`, `OPTIONS`, `CONNECT` and `TRACE`. The metrics library reads
the method from the request it is handed, so the metrics middleware hands it a
copy of the request carrying the label; the handlers still see the method the
client sent.
- 2026-10-15 logging out ends every login session (closes #256): a login session
was a cookie with nothing kept on the server, so `/logout` removed it from one
browser and a copy of it kept working for its 30 days. Every login is made
with the one signing key, so logging out now ends them all: the session
manager keeps a random value, made with `crypto/rand` when the new `logout`
table has none and replaced by each logout, saves it there so that it outlasts
a restart, puts the value at login in each session, and refuses a session
carrying another value. The check for a logout compares no time, so a login
after a logout works whatever the host's clock does; a session's 30 days are
still checked against the clock. Losing the state directory ends every login
session, and no logout can make an ended one work again. Logging out is now
`POST /logout` from a form on the generator page, behind the CSRF check the
other forms have, so a link or image elsewhere on the same site cannot end the
sessions. A logout without a valid login session ends none and only removes
the cookie, as anyone can take a CSRF token from the login page. A logout that
cannot be saved answers 500. `session.NewManager`, which the tests use, keeps
the value in memory only; pixad uses `session.NewManagerWithDatabase`.
- 2026-10-15 a cached file left empty or partial is no longer served, or fetched
again on every request (closes #252). Source and transformed image files, and
each transformed image's `.meta` file, are synced to disk before they are
renamed into place, and their directory after; the `.meta` file is written
before the image, and a failed directory sync removes the renamed file. The
reconciliation pass, at startup and every five minutes, removes a cached
source or transformed image whose file is empty, or not the size the database
records for it, with its rows and a warning naming the file, so the next
request fetches the source or converts the image again; requests compare no
sizes. Storing a source over a file of another size replaces that file. A
source that fails to decode is not removed for that alone: a damaged upstream
image would then be fetched on every request.
- 2026-10-15 a transcode stops once its request's `downstream_timeout` has
ended, unless it has begun encoding (closes #243): the image processor looked
at its context only while it waited for a processing slot, so an image was
decoded and encoded, holding the slot, however long after its request's
deadline that took. `Process` now returns the context's error, and gives the
slot back, when the context has ended by the time it has the slot or before it
encodes. A request that waited for a shared transcode stopped at the first
request's deadline is answered 503 `server busy, try again later`, not 500,
though its own `downstream_timeout` may not have ended. govips gives pixa no
way to stop libvips once an encode has begun, so that encode keeps its slot
until it finishes; the limit of 8192 on each side of the source and of the
output bounds how long that takes. With one libvips thread, 8192x8192 random
pixels take up to about 1 second to encode as JPEG, 15 as PNG, 16 as GIF, 63
as WebP, 110 as AVIF and 96 as JPEG XL, the last three with an alpha channel,
plus up to about 11 seconds to decode the source; `README.md` gives these
times and no longer says the work stops when `downstream_timeout` ends. The
settings of the WebP, AVIF and JPEG XL encoders, which can hold a slot past
the default `downstream_timeout` of 60 seconds, are unchanged.
- 2026-10-15 "Running under upaas" in `README.md` tells the operator to set
`PIXA_CACHE_MAX_BYTES` (closes #251): it listed the variable as optional, but
under upaas the volume is a directory on the host's filesystem. The default,
75% of the sum of that filesystem's free space and what the cache holds, and
at least 500 MiB, let one pixa take three quarters or more of the space the
host and every other app had left, and all of it when that was 500 MiB or
less. The section now says so. The default is unchanged.
- 2026-10-15 the request metrics are labelled by route, not by path (closes
#254): their `handler` label was each request's path, so every new path, a 404
or an encrypted URL, added series to the Prometheus registry that were never
removed, and `/metrics` listed every encrypted URL's token. The label is now
the pattern of the chi route the request matches, such as `/v1/image/*`, or
`unmatched` for a request that matches none: one the router answers 404 or
405, or a CORS preflight. The metrics library takes the label before the
request is routed, so the metrics middleware asks the router which route
matches.
- 2026-10-15 expired negative cache rows are removed on a schedule (closes
#244): the row of a failed fetch, which makes pixa answer 502 for that source
URL for 5 minutes, was deleted only when the same URL was requested again
after that, so a client asking for a new missing path each time added rows
that stayed for good. The eviction goroutine now calls `CleanExpired` at
startup and on each periodic tick, every 5 minutes in pixad, so a row stays at
most about 10 minutes. `CleanExpired` deletes at most 1000 rows per statement,
so a request waits behind 1000 deletions at most, however many rows have
expired. The goroutine runs with the disk cache disabled
(`cache_max_bytes: 0`) too, where it does nothing else, as the negative cache
is used either way.
- 2026-10-15 a cached variant is served while its source URL is in the negative - 2026-10-15 a cached variant is served while its source URL is in the negative
cache (closes #245): `Get` checked the negative cache before the variant cache (closes #245): `Get` checked the negative cache before the variant
cache, so once a fetch of the source failed, as one does when another variant cache, so once a fetch of the source failed, as one does when another variant
misses after the source was evicted and the upstream is down, every variant of misses after the source was evicted and the upstream is down, every cached
that source answered 502 for 5 minutes, also one on disk or one that could be variant of that source answered 502 for 5 minutes. `Get` now serves a cached
made from the cached source. The negative cache is now checked only just variant first and checks the negative cache only on a miss, before processing.
before a fetch from the upstream, so a cached variant is served and a variant `README.md` says so under `/v1/image/`.
not in the cache is made from the cached source when there is one. `README.md`
says so under `/v1/image/`. A refusal from the negative cache now counts as a
miss, as any other request that misses the variant cache does.
- 2026-10-15 the host of a `/v1/image/` URL is lower-cased, and one with a port
needs a signature (closes #246): with `cdn.example.com` allowlisted,
`CDN.Example.com` and `cdn.example.com:8443` passed the allowlist too, and
each spelling was fetched and cached on its own and had connections of its own
under `upstream_connections_per_host`. The image URL parser now lower-cases
the host and refuses one holding `@`, which a URL reads as a user name and
password, with 400. A host with a port, or with a `:` and no port, needs a
signature even when it is allowlisted. That is checked in the image service,
not in `allowlist.IsAllowed`, which also matches `referer_blocklist`, where a
`Referer` is blocked whatever its port. A URL is signed over its host as it
writes it, in lower case, with any IPv6 brackets, `:` and port kept.
- 2026-10-15 a `:` in the source path or query no longer moves where the path
ends and the query begins (closes #247): the signed data and the cache key
joined host, path and query with `:` and no escaping, so a URL signed for
`https://h/a:b` verified for `https://h/a?b:`; the two also shared a cache
key, and whichever was fetched first was served for both. In both, the path
and the query now have each `%` written as `%25` and each `:` as `%3A`; a path
and query holding neither are signed and keyed as before. Signature
Specification in `README.md` says so.
- 2026-10-15 the example config allowlists no host (closes #249): it allowlisted - 2026-10-15 the example config allowlists no host (closes #249): it allowlisted
`github.com` and `user-images.githubusercontent.com`, where anyone can put an `github.com` and `user-images.githubusercontent.com`, where anyone can put an
image, so a deployment set up as Getting Started says served those images to image, so a deployment set up as Getting Started says served those images to
@@ -624,15 +45,6 @@ P2: security: per-origin rate limiting
commented entry under `example.com`. Getting Started and "Running under upaas" commented entry under `example.com`. Getting Started and "Running under upaas"
in `README.md` say that every image on an allowlisted host is served without a in `README.md` say that every image on an allowlisted host is served without a
signature, so an allowlisted host must be one the operator controls. signature, so an allowlisted host must be one the operator controls.
- 2026-10-15 a chunked upstream body of exactly `upstream_max_response_size`
bytes is accepted (closes #248): once the limit was used up, the fetcher's
next read refused the body without looking for its end. Go's HTTP client
returns the end of a body sent with a `Content-Length` together with its last
bytes, but the end of a chunked body can come in a later read, so such a body
was refused after it had been downloaded whole, and the image routes
answered 500. Once the limit is used up, the fetcher now reads one more byte:
if the body has ended it is accepted, and if there is more it is refused with
`ErrResponseTooLarge`, as before.
- 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side - 2026-10-13 `fit=outside` no longer makes an output above 8192 on a side
(closes #257): it scales the image to cover the requested size, and the side (closes #257): it scales the image to cover the requested size, and the side
scaled past that size was held to no maximum, so `1x8192?fit=outside` of a scaled past that size was held to no maximum, so `1x8192?fit=outside` of a
@@ -1079,11 +491,11 @@ P2: security: per-origin rate limiting
cancellation but keeps its deadline, and the others wait for its image or cancellation but keeps its deadline, and the others wait for its image or
error holding no upstream connection or processing slot, and stop waiting when error holding no upstream connection or processing slot, and stop waiting when
their own context ends; the request doing the processing waits for it even their own context ends; the request doing the processing waits for it even
then, until it ends, which can be after its deadline; a request whose context then, up to its deadline; a request whose context has already ended starts
has already ended starts nothing; each request counts one miss, and the nothing; each request counts one miss, and the processing counts its fetch and
processing counts its fetch and transcode once; a panic while processing is transcode once; a panic while processing is reported to Sentry when
reported to Sentry when `sentry_dsn` is set and becomes an error for every `sentry_dsn` is set and becomes an error for every waiting request instead of
waiting request instead of stopping pixad; documented in `README.md`. stopping pixad; documented in `README.md`.
- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS - 2026-09-29 only the image routes send CORS headers (closes #98): the CORS
middleware, with the `access_control_allow_origin` origin, moved from the middleware, with the `access_control_allow_origin` origin, moved from the
router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it router root onto a `/v1` subrouter holding `/v1/image/` and `/v1/e/`, where it
+4 -5
View File
@@ -108,11 +108,10 @@ upstream_connections_per_host: 20
# downstream_timeout has ended first. # downstream_timeout has ended first.
upstream_connections: 64 upstream_connections: 64
# Maximum number of images decoded and encoded at once (default: half the # Maximum number of images decoded and encoded at once (default: the
# number of CPUs pixa can use, at least 1; the number of CPUs follows a # number of CPUs pixa can use, which follows a container's CPU limit). A
# container's CPU limit). A request that finds none free waits up to 10 # request that finds none free waits up to 10 seconds for one, and if none
# seconds for one, and if none frees up it is answered 503, unless # frees up it is answered 503, unless downstream_timeout has ended first.
# downstream_timeout has ended first.
# max_concurrent_processing: 4 # max_concurrent_processing: 4
# How many requests to the image routes (/v1/image/ and /v1/e/) one client # How many requests to the image routes (/v1/image/ and /v1/e/) one client
+1
View File
@@ -14,6 +14,7 @@ require (
github.com/go-chi/httprate v0.16.0 github.com/go-chi/httprate v0.16.0
github.com/gorilla/csrf v1.7.3 github.com/gorilla/csrf v1.7.3
github.com/gorilla/securecookie v1.1.2 github.com/gorilla/securecookie v1.1.2
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_golang v1.23.2
github.com/slok/go-http-metrics v0.13.0 github.com/slok/go-http-metrics v0.13.0
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
+2
View File
@@ -228,6 +228,8 @@ github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I=
github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+10
View File
@@ -72,3 +72,13 @@ func (w *HostAllowList) IsAllowed(u *url.URL) bool {
return false return false
} }
// IsEmpty returns true if the allow list has no entries.
func (w *HostAllowList) IsEmpty() bool {
return len(w.exactHosts) == 0 && len(w.suffixHosts) == 0
}
// Count returns the total number of allow list entries.
func (w *HostAllowList) Count() int {
return len(w.exactHosts) + len(w.suffixHosts)
}
+84
View File
@@ -139,3 +139,87 @@ func TestHostAllowList_IsAllowed_SuffixMatch(t *testing.T) {
}, },
}) })
} }
func TestHostAllowList_IsEmpty(t *testing.T) {
t.Parallel()
tests := []struct {
name string
patterns []string
want bool
}{
{
name: "empty",
patterns: []string{},
want: true,
},
{
name: "nil",
patterns: nil,
want: true,
},
{
name: "whitespace only",
patterns: []string{" ", ""},
want: true,
},
{
name: "has entries",
patterns: []string{"example.com"},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
w := allowlist.New(tt.patterns)
if got := w.IsEmpty(); got != tt.want {
t.Errorf("IsEmpty() = %v, want %v", got, tt.want)
}
})
}
}
func TestHostAllowList_Count(t *testing.T) {
t.Parallel()
tests := []struct {
name string
patterns []string
want int
}{
{
name: "empty",
patterns: []string{},
want: 0,
},
{
name: "exact hosts only",
patterns: []string{"a.com", "b.com", "c.com"},
want: 3,
},
{
name: "suffix hosts only",
patterns: []string{".a.com", ".b.com"},
want: 2,
},
{
name: "mixed",
patterns: []string{"exact.com", ".suffix.com"},
want: 2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
w := allowlist.New(tt.patterns)
if got := w.Count(); got != tt.want {
t.Errorf("Count() = %v, want %v", got, tt.want)
}
})
}
}
-5
View File
@@ -7,7 +7,6 @@ import (
"os/signal" "os/signal"
"syscall" "syscall"
"github.com/prometheus/client_golang/prometheus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/config"
@@ -17,7 +16,6 @@ import (
"sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/logger"
"sneak.berlin/go/pixa/internal/middleware" "sneak.berlin/go/pixa/internal/middleware"
"sneak.berlin/go/pixa/internal/providers"
"sneak.berlin/go/pixa/internal/server" "sneak.berlin/go/pixa/internal/server"
) )
@@ -63,10 +61,7 @@ func run(_ *cobra.Command, _ []string) {
server.New, server.New,
middleware.New, middleware.New,
healthcheck.New, healthcheck.New,
// The registry /metrics serves
func() prometheus.Registerer { return prometheus.DefaultRegisterer },
), ),
fx.Provide(providers.Constructors()...),
fx.Invoke( fx.Invoke(
func(log *logger.Logger) { log.Identify() }, func(log *logger.Logger) { log.Identify() },
func(*server.Server) {}, func(*server.Server) {},
@@ -88,56 +88,3 @@ func TestBlockedNetworksInvalidAbortsStartup(t *testing.T) {
}, },
}) })
} }
// TestBlockedNetworksMappedEntryIsIPv4Range checks that an entry written in
// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
// since the fetcher matches an IPv4-mapped address as the IPv4 address it
// holds. /96, the shortest such entry, covers every IPv4 address.
func TestBlockedNetworksMappedEntryIsIPv4Range(t *testing.T) {
t.Parallel()
yamlContent := signingKeyLine + `blocked_networks:
- "::ffff:10.0.0.0/104"
- "::ffff:0:0/96"
`
c, err := configFromYAML(t, yamlContent)
if err != nil {
t.Fatalf("IPv4-mapped blocked_networks entries should load, got error: %v", err)
}
want := []string{"10.0.0.0/8", "0.0.0.0/0"}
if len(c.BlockedNetworks) != len(want) {
t.Fatalf("BlockedNetworks = %v, want %d entries", c.BlockedNetworks, len(want))
}
for i, w := range want {
if got := c.BlockedNetworks[i].String(); got != w {
t.Errorf("BlockedNetworks[%d] = %q, want %q", i, got, w)
}
}
}
// TestBlockedNetworksMappedEntryShorterThan96AbortsStartup checks that an
// entry in the IPv4-mapped form shorter than /96, which covers more than the
// IPv4 addresses, aborts startup with an error naming the key and the value.
func TestBlockedNetworksMappedEntryShorterThan96AbortsStartup(t *testing.T) {
t.Parallel()
runAbortCases(t, []abortCase{
{
name: "mapped-shorter-than-96",
yaml: signingKeyLine + `blocked_networks:
- "::ffff:0.0.0.0/80"
`,
wantErrSubstrings: []string{keyBlockedNetworks, "::ffff:0.0.0.0/80"},
},
{
name: "mapped-95",
yaml: signingKeyLine + `blocked_networks:
- "::ffff:0:0/95"
`,
wantErrSubstrings: []string{keyBlockedNetworks},
},
})
}
@@ -135,38 +135,10 @@ func TestCacheMaxBytesInvalidValuesAbortStartup(t *testing.T) {
} }
} }
// TestDefaultCacheMaxBytesAppliesFloor verifies that when 75% of the
// free space plus what the cache holds is below 500 MiB, the default is
// floored at DefaultCacheMaxBytesFloor.
func TestDefaultCacheMaxBytesAppliesFloor(t *testing.T) {
t.Parallel()
cases := []struct {
name string
freeBytes uint64
}{
{name: "100 MiB free", freeBytes: 104857600},
{name: "zero free", freeBytes: 0},
{name: "just below floor threshold", freeBytes: 699050665},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := DefaultCacheMaxBytes(tc.freeBytes, 0)
if got != DefaultCacheMaxBytesFloor {
t.Errorf("DefaultCacheMaxBytes = %d, want floor %d",
got, DefaultCacheMaxBytesFloor)
}
})
}
}
// TestCacheMaxBytesExplicitIsRecorded verifies that an omitted // TestCacheMaxBytesExplicitIsRecorded verifies that an omitted
// cache_max_bytes is recorded as not explicit, so pixad works out the // cache_max_bytes is recorded as not explicit, so the cache works out
// default when it builds the cache, and that an explicit zero is recorded // the default when it opens, and that an explicit zero is recorded as
// as explicit, so it disables the disk cache instead. // explicit, so it disables the disk cache instead.
func TestCacheMaxBytesExplicitIsRecorded(t *testing.T) { func TestCacheMaxBytesExplicitIsRecorded(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1,7 +1,6 @@
package config package config
import ( import (
"fmt"
"runtime" "runtime"
"testing" "testing"
) )
@@ -13,8 +12,8 @@ const (
) )
// TestOmittedConcurrencyLimitsUseDefaults checks that an omitted // TestOmittedConcurrencyLimitsUseDefaults checks that an omitted
// max_concurrent_processing is half the CPUs Go uses, and at least 1, and // max_concurrent_processing is the number of CPUs Go uses and an omitted
// an omitted upstream_connections is 64. // upstream_connections is 64.
func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) { func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
t.Parallel() t.Parallel()
@@ -23,9 +22,9 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
t.Fatalf("minimal config should be valid, got error: %v", err) t.Fatalf("minimal config should be valid, got error: %v", err)
} }
if c.MaxConcurrentProcessing != max(runtime.GOMAXPROCS(0)/2, 1) { if c.MaxConcurrentProcessing != runtime.GOMAXPROCS(0) {
t.Errorf("MaxConcurrentProcessing = %d, want %d, half the CPUs", t.Errorf("MaxConcurrentProcessing = %d, want %d, one per CPU",
c.MaxConcurrentProcessing, max(runtime.GOMAXPROCS(0)/2, 1)) c.MaxConcurrentProcessing, runtime.GOMAXPROCS(0))
} }
if c.UpstreamConnections != 64 { if c.UpstreamConnections != 64 {
@@ -33,36 +32,6 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
} }
} }
// TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne checks
// that an omitted max_concurrent_processing is half the CPUs Go uses,
// rounded down, and 1 when Go uses one CPU.
//
//nolint:paralleltest // it sets GOMAXPROCS, which other tests read
func TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne(t *testing.T) {
cpusBefore := runtime.GOMAXPROCS(0)
t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) })
for _, tc := range []struct{ cpus, want int }{
{1, 1}, {2, 1}, {3, 1}, {8, 4},
} {
runtime.GOMAXPROCS(tc.cpus)
c, err := configFromYAML(t, signingKeyLine)
if err != nil {
t.Fatalf("minimal config should be valid, got error: %v", err)
}
t.Logf("GOMAXPROCS %d: MaxConcurrentProcessing %d",
tc.cpus, c.MaxConcurrentProcessing)
if c.MaxConcurrentProcessing != tc.want {
t.Errorf("GOMAXPROCS %d: MaxConcurrentProcessing = %d, want %d",
tc.cpus, c.MaxConcurrentProcessing, tc.want)
}
}
}
// TestExplicitConcurrencyLimitsAreUsed checks that valid values for the // TestExplicitConcurrencyLimitsAreUsed checks that valid values for the
// two limits are used as given. // two limits are used as given.
func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) { func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) {
@@ -191,19 +160,3 @@ func TestConcurrencyLimitsFromEnvironment(t *testing.T) {
}) })
} }
} }
// TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup checks
// that a PIXA_MAX_CONCURRENT_PROCESSING that is not a whole number of at
// least 1, an empty one included, aborts startup naming the variable and
// the key instead of taking the default.
func TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup(t *testing.T) {
for _, value := range []string{"", " ", "0.5", "half", "-1", "4 images"} {
t.Run(fmt.Sprintf("value %q", value), func(t *testing.T) {
t.Setenv(testMaxConcurrentProcessingVar, value)
_, err := configFromYAML(t, signingKeyLine)
wantStartupError(t, err,
testMaxConcurrentProcessingVar, keyMaxConcurrentProcessing)
})
}
}
+591 -429
View File
File diff suppressed because it is too large Load Diff
+85
View File
@@ -1,12 +1,97 @@
package config package config
import ( import (
"os"
"path/filepath" "path/filepath"
"testing" "testing"
"git.eeqj.de/sneak/smartconfig" "git.eeqj.de/sneak/smartconfig"
) )
// writeTestConfig writes yamlContent to a temp config file and returns
// the file path.
func writeTestConfig(t *testing.T, yamlContent string) string {
t.Helper()
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "config.yml")
err := os.WriteFile(configPath, []byte(yamlContent), 0o600)
if err != nil {
t.Fatalf("failed to write test config: %v", err)
}
return configPath
}
// checkAllowlistHosts loads the config at configPath and asserts that
// getStringSlice returns the three expected hosts.
func checkAllowlistHosts(t *testing.T, configPath string) {
t.Helper()
sc, err := loadTestConfig(configPath)
if err != nil {
t.Fatalf("failed to load config: %v", err)
}
hosts := getStringSlice(sc)
if len(hosts) != 3 {
t.Errorf("expected 3 hosts, got %d: %v", len(hosts), hosts)
}
expected := []string{"static.sneak.cloud", "sneak.berlin", testHostS3}
for i, want := range expected {
if i >= len(hosts) {
t.Errorf("missing host at index %d: want %q", i, want)
continue
}
if hosts[i] != want {
t.Errorf("host[%d] = %q, want %q", i, hosts[i], want)
}
}
}
func TestGetStringSlice_YAMLList(t *testing.T) {
t.Parallel()
yamlContent := `
allowlist_hosts:
- static.sneak.cloud
- sneak.berlin
- s3.sneak.cloud
`
checkAllowlistHosts(t, writeTestConfig(t, yamlContent))
}
func TestGetStringSlice_CommaSeparated(t *testing.T) {
t.Parallel()
// Backwards compatibility with comma-separated string values.
yamlContent := `allowlist_hosts: "static.sneak.cloud, sneak.berlin, s3.sneak.cloud"`
checkAllowlistHosts(t, writeTestConfig(t, yamlContent))
}
func TestGetStringSlice_Empty(t *testing.T) {
t.Parallel()
configPath := writeTestConfig(t, `port: 8080`)
sc, err := loadTestConfig(configPath)
if err != nil {
t.Fatalf("failed to load config: %v", err)
}
hosts := getStringSlice(sc)
if len(hosts) != 0 {
t.Errorf("expected nil or empty slice, got %v", hosts)
}
}
// TestExampleConfigAllowlistsNoHost loads configs/config.example.yml // TestExampleConfigAllowlistsNoHost loads configs/config.example.yml
// with a real signing key, as Getting Started in README.md has it copied // with a real signing key, as Getting Started in README.md has it copied
// and edited, and checks that it allowlists no host: every image on an // and edited, and checks that it allowlists no host: every image on an
@@ -15,36 +15,8 @@ import (
_ "modernc.org/sqlite" // SQLite driver registration _ "modernc.org/sqlite" // SQLite driver registration
) )
// The config keys the tests name in the errors they expect.
const (
keyDebug = "debug"
keyMaintenanceMode = "maintenance_mode"
keyPort = "port"
keyStateDir = "state_dir"
keySentryDSN = "sentry_dsn"
keyDBURL = "db_url"
keyMetrics = "metrics"
keyMetricsUsername = "metrics.username"
keyMetricsPassword = "metrics.password"
keySigningKey = "signing_key"
keyAllowlistHosts = "allowlist_hosts"
keyRefererBlocklist = "referer_blocklist"
keyAllowHTTP = "allow_http"
keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
keyUpstreamConnections = "upstream_connections"
keyMaxConcurrentProcessing = "max_concurrent_processing"
keyCacheMaxBytes = "cache_max_bytes"
keyBlockedNetworks = "blocked_networks"
keyTrustedProxies = "trusted_proxies"
keyAccessControlAllowOrigin = "access_control_allow_origin"
keyUpstreamFetchTimeout = "upstream_fetch_timeout"
keyUpstreamMaxResponseSize = "upstream_max_response_size"
keyDownstreamTimeout = "downstream_timeout"
keyImageRequestsPerMinute = "image_requests_per_minute"
)
// validTestSigningKey is a 32-character signing key that satisfies the // validTestSigningKey is a 32-character signing key that satisfies the
// minimum length requirement in validateSigningKey. // minimum length requirement in validate().
const validTestSigningKey = "0123456789abcdef0123456789abcdef" const validTestSigningKey = "0123456789abcdef0123456789abcdef"
// signingKeyLine is a valid signing_key config line used as the base of // signingKeyLine is a valid signing_key config line used as the base of
@@ -163,104 +135,6 @@ func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
} }
} }
// TestDerivedDBURLOpensStateDirHoldingURLCharacters opens the db_url
// derived from a state_dir whose name holds %41, ? and #, and checks that
// the database is state.sqlite3 in that directory, in WAL mode. Put into
// a file: URL as it is, the name would lose its meaning: %41 is read as
// A, and the path ends at the ?, which also loses the journal mode.
func TestDerivedDBURLOpensStateDirHoldingURLCharacters(t *testing.T) {
t.Parallel()
stateDir := filepath.Join(t.TempDir(), "a%41b?c#d")
err := os.Mkdir(stateDir, 0o750)
if err != nil {
t.Fatalf("failed to create %q: %v", stateDir, err)
}
// Quoted so YAML takes the whole name as it is.
c, err := configFromYAML(t, signingKeyLine+"state_dir: '"+stateDir+"'\n")
if err != nil {
t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
}
t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(stateDir, "state.sqlite3"))
}
// TestDerivedDBURLOpensStateDirStartingWithTwoSlashes opens the db_url
// derived from a state_dir that starts with //, and checks that the
// database is state.sqlite3 in that directory, in WAL mode. Written right
// after file:, such a name gives a URL starting file://, and SQLite reads
// the first directory name as a host and refuses to open it.
func TestDerivedDBURLOpensStateDirStartingWithTwoSlashes(t *testing.T) {
t.Parallel()
dir := t.TempDir()
stateDir := "/" + dir
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+stateDir+"\n")
if err != nil {
t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
}
t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(dir, "state.sqlite3"))
}
// TestDerivedDBURLForRootStateDir checks the db_url derived from a
// state_dir of /. The path written into the URL is then
// //state.sqlite3, and written right after file: it gives a URL starting
// file://, where SQLite reads state.sqlite3 as a host and refuses to open
// it. The test only checks the URL, since it cannot write to /.
func TestDerivedDBURLForRootStateDir(t *testing.T) {
t.Parallel()
c, err := configFromYAML(t, signingKeyLine+"state_dir: /\n")
if err != nil {
t.Fatalf("config with state_dir / should be valid, got: %v", err)
}
t.Logf("state_dir / gives db_url %q", c.DBURL)
wantDBURL := "file:////state.sqlite3?_pragma=journal_mode(WAL)"
if c.DBURL != wantDBURL {
t.Errorf("DBURL = %q, want %q", c.DBURL, wantDBURL)
}
}
// checkDBURLOpensFileInWALMode opens dbURL with the SQLite driver pixad
// uses, and checks that the database is in WAL mode and is the file
// dbFile.
func checkDBURLOpensFileInWALMode(t *testing.T, dbURL, dbFile string) {
t.Helper()
db, err := sql.Open("sqlite", dbURL)
if err != nil {
t.Fatalf("failed to open %q: %v", dbURL, err)
}
t.Cleanup(func() { _ = db.Close() })
var journalMode string
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
if err != nil {
t.Fatalf("failed to read the journal mode of %q: %v", dbURL, err)
}
if journalMode != "wal" {
t.Errorf("journal mode of %q = %q, want wal", dbURL, journalMode)
}
_, err = os.Stat(dbFile)
if err != nil {
t.Errorf("opening %q did not create %q: %v", dbURL, dbFile, err)
}
}
func TestExplicitValidValuesAreUsed(t *testing.T) { func TestExplicitValidValuesAreUsed(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1,127 +0,0 @@
package config
import (
"os"
"path/filepath"
"regexp"
"strings"
"testing"
)
// TestReadmeNamesEverySetting checks that the list of settings in
// README.md names exactly the keys of the settings table, one item each.
// The list is the first one in the Configuration section, and it ends at
// the first blank line. Each item starts with its key in backticks.
func TestReadmeNamesEverySetting(t *testing.T) {
t.Parallel()
readme, err := os.ReadFile(filepath.Join("..", "..", "README.md"))
if err != nil {
t.Fatalf("failed to read README.md: %v", err)
}
_, section, found := strings.Cut(string(readme), "\n### Configuration\n")
if !found {
t.Fatal("README.md has no Configuration section")
}
// The section ends at the next heading of its level or above.
section, _, _ = strings.Cut(section, "\n## ")
section, _, _ = strings.Cut(section, "\n### ")
_, list, found := strings.Cut(section, "\n- ")
if !found {
t.Fatal("README.md's Configuration section has no list")
}
list, _, _ = strings.Cut("- "+list, "\n\n")
itemKey := regexp.MustCompile("^- `([^`]+)`")
named := map[string]bool{}
for line := range strings.SplitSeq(list, "\n") {
// A line that does not start with "- " goes on with the item above.
if !strings.HasPrefix(line, "- ") {
continue
}
match := itemKey.FindStringSubmatch(line)
if match == nil {
t.Errorf("README.md's list of settings has an item "+
"that does not start with a key in backticks: %s", line)
continue
}
if named[match[1]] {
t.Errorf("README.md's list of settings names %s twice", match[1])
}
named[match[1]] = true
}
for _, s := range new(Config).settings() {
if !named[s.key] {
t.Errorf("README.md's list of settings does not name %s", s.key)
}
delete(named, s.key)
}
for key := range named {
t.Errorf("README.md's list of settings names %s, "+
"which is not a setting", key)
}
}
// TestExampleConfigNamesEverySetting checks that
// configs/config.example.yml names exactly the keys of the settings
// table, each one set or commented out with "# ". A key line holds the
// key, a colon and at most a one-word value, which tells it from a
// comment line that starts with a word and a colon. An indented key is
// inside the map of the key above it, as username is inside metrics.
func TestExampleConfigNamesEverySetting(t *testing.T) {
t.Parallel()
const examplePath = "configs/config.example.yml"
example, err := os.ReadFile(filepath.Join("..", "..", examplePath))
if err != nil {
t.Fatalf("failed to read %s: %v", examplePath, err)
}
keyLine := regexp.MustCompile(`(?m)^(?:# )?( *)([a-z_]+):(?: \S+)?$`)
named := map[string]bool{}
var mapKey string
for _, match := range keyLine.FindAllStringSubmatch(string(example), -1) {
indent, key := match[1], match[2]
if indent == "" {
mapKey = key
named[key] = true
continue
}
// The key holding the map is not itself a setting.
delete(named, mapKey)
named[mapKey+"."+key] = true
}
// The env section sets environment variables; it is not a setting.
delete(named, "env")
for _, s := range new(Config).settings() {
if !named[s.key] {
t.Errorf("%s does not name %s", examplePath, s.key)
}
delete(named, s.key)
}
for key := range named {
t.Errorf("%s names %s, which is not a setting", examplePath, key)
}
}
+3 -4
View File
@@ -138,8 +138,7 @@ func TestPixaPortAbortsStartupPointingToPort(t *testing.T) {
// TestSettingVariablesAndConfigPathAreAccepted checks that every // TestSettingVariablesAndConfigPathAreAccepted checks that every
// setting's variable and PIXA_CONFIG_PATH pass the check for unknown // setting's variable and PIXA_CONFIG_PATH pass the check for unknown
// PIXA_ variables. TestEnvironmentSetsEveryKey pins the names in the // PIXA_ variables. TestEnvironmentSetsEveryKey pins the names in the list.
// settings table.
func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) { func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) {
// A config file's env section loaded by another test can leave a // A config file's env section loaded by another test can leave a
// PIXA_ variable set for the whole process, so every one is unset // PIXA_ variable set for the whole process, so every one is unset
@@ -160,8 +159,8 @@ func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) {
t.Setenv("PIXA_CONFIG_PATH", "/etc/pixa/config.yml") t.Setenv("PIXA_CONFIG_PATH", "/etc/pixa/config.yml")
for _, s := range new(Config).settings() { for _, name := range envVarNames() {
t.Setenv(s.envVar, "") t.Setenv(name, "")
} }
err := validateKnownEnvVars() err := validateKnownEnvVars()
@@ -83,41 +83,3 @@ func TestTrustedProxiesInvalidAbortsStartup(t *testing.T) {
}, },
}) })
} }
// TestTrustedProxiesMappedEntryIsIPv4Range checks that an entry written in
// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
// since the client address is matched as the IPv4 address it holds. /96, the
// shortest such entry, covers every IPv4 address.
func TestTrustedProxiesMappedEntryIsIPv4Range(t *testing.T) {
t.Parallel()
c, err := configFromYAML(t, signingKeyLine+
`trusted_proxies: ["::ffff:10.0.0.0/104", "::ffff:0:0/96"]`+"\n")
if err != nil {
t.Fatalf("IPv4-mapped trusted_proxies entries should load: %v", err)
}
got := make([]string, len(c.TrustedProxies))
for i, p := range c.TrustedProxies {
got[i] = p.String()
}
if joined := strings.Join(got, ","); joined != "10.0.0.0/8,0.0.0.0/0" {
t.Errorf("TrustedProxies = %v, want 10.0.0.0/8 and 0.0.0.0/0", got)
}
}
// TestTrustedProxiesMappedEntryShorterThan96AbortsStartup checks that an
// entry in the IPv4-mapped form shorter than /96, which covers more than the
// IPv4 addresses, aborts startup with an error naming the key and the value.
func TestTrustedProxiesMappedEntryShorterThan96AbortsStartup(t *testing.T) {
t.Parallel()
runAbortCases(t, []abortCase{
{
name: "mapped-95",
yaml: signingKeyLine + `trusted_proxies: ["::ffff:0:0/95"]` + "\n",
wantErrSubstrings: []string{keyTrustedProxies, "::ffff:0:0/95"},
},
})
}
@@ -104,31 +104,40 @@ func writeConcurrently(t *testing.T, db *sql.DB) {
wg.Wait() wg.Wait()
var sources int var hits, sources int
err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM source_content`). err := db.QueryRowContext(ctx, `
Scan(&sources) SELECT hit_count, (SELECT COUNT(*) FROM source_content)
FROM cache_stats WHERE id = 1
`).Scan(&hits, &sources)
if err != nil { if err != nil {
t.Fatalf("failed to count the writes: %v", err) t.Fatalf("failed to count the writes: %v", err)
} }
if sources != totalRequests { if hits != totalRequests || sources != totalRequests {
t.Errorf("%d source_content rows, want %d", sources, totalRequests) t.Errorf("hit_count = %d and %d source_content rows, want %d of each",
hits, sources, totalRequests)
} }
} }
// writeLikeOneRequest makes the writes one request and the eviction pass // writeLikeOneRequest makes the writes one request and the eviction pass
// make: it stores a source, records a transformed image and deletes that // make: it counts a cache hit, stores a source, records a transformed image
// record again. // and deletes that record again.
func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error { func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error {
_, err := db.ExecContext(ctx, `INSERT INTO source_content _, err := db.ExecContext(ctx,
(source_key, size_bytes) VALUES (?, 1)`, key) `UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`)
if err != nil {
return fmt.Errorf("counting a cache hit: %w", err)
}
_, err = db.ExecContext(ctx, `INSERT INTO source_content
(content_hash, content_type, size_bytes) VALUES (?, 'image/png', 1)`, key)
if err != nil { if err != nil {
return fmt.Errorf("storing a source: %w", err) return fmt.Errorf("storing a source: %w", err)
} }
_, err = db.ExecContext(ctx, `INSERT INTO variant_content _, err = db.ExecContext(ctx, `INSERT INTO variant_content
(cache_key, size_bytes) VALUES (?, 1)`, key) (cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key)
if err != nil { if err != nil {
return fmt.Errorf("recording a transformed image: %w", err) return fmt.Errorf("recording a transformed image: %w", err)
} }
+10 -10
View File
@@ -82,11 +82,7 @@ func ParseMigrationVersion(filename string) (int, error) {
return version, nil return version, nil
} }
// New opens the database and applies the migrations, so that the parts // New creates a new Database instance.
// fx builds after it can read the database while they are built. Its
// stop hook closes the database. fx runs stop hooks in the reverse of the
// order it built their parts in, so the stop hooks of those parts run
// before this one.
func New(lc fx.Lifecycle, params Params) (*Database, error) { func New(lc fx.Lifecycle, params Params) (*Database, error) {
s := &Database{ s := &Database{
log: params.Logger.Get(), log: params.Logger.Get(),
@@ -95,16 +91,20 @@ func New(lc fx.Lifecycle, params Params) (*Database, error) {
s.log.Info("Database instantiated") s.log.Info("Database instantiated")
err := s.connect(context.Background())
if err != nil {
return nil, err
}
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
s.log.Info("Database OnStart Hook")
return s.connect(ctx)
},
OnStop: func(_ context.Context) error { OnStop: func(_ context.Context) error {
s.log.Info("Database OnStop Hook") s.log.Info("Database OnStop Hook")
if s.db != nil {
return s.db.Close() return s.db.Close()
}
return nil
}, },
}) })
+4 -1
View File
@@ -159,7 +159,10 @@ func TestApplyMigrations_CreatesSchemaAndTables(t *testing.T) {
} }
// Verify that the application tables created by 001.sql exist. // Verify that the application tables created by 001.sql exist.
tables := []string{"source_content", "negative_cache"} tables := []string{
"source_content", "source_metadata", "output_content",
"request_cache", "negative_cache", "cache_stats",
}
for _, table := range tables { for _, table := range tables {
var count int var count int
+79 -18
View File
@@ -1,27 +1,52 @@
-- Migration 001: Initial schema -- Migration 001: Initial schema
-- Creates all tables for the pixa caching image proxy -- Creates all tables for the pixa caching image proxy
-- Source images, one row per source URL -- Source content blobs
-- Files stored at: cache/sources/<ab>/<cd>/<source_key>, where source_key -- Files stored at: cache/sources/<ab>/<cd>/<sha256>
-- is the SHA-256 of the source's host, path and query
CREATE TABLE IF NOT EXISTS source_content ( CREATE TABLE IF NOT EXISTS source_content (
source_key TEXT PRIMARY KEY, content_hash TEXT PRIMARY KEY,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL, size_bytes INTEGER NOT NULL,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
); );
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
ON source_content(last_accessed_at); ON source_content(last_accessed_at);
-- Source URL metadata - maps URLs to content hashes
-- JSON stored at: cache/metadata/<hostname>/<path_hash>.json
CREATE TABLE IF NOT EXISTS source_metadata (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_host TEXT NOT NULL,
source_path TEXT NOT NULL,
source_query TEXT NOT NULL DEFAULT '',
path_hash TEXT NOT NULL,
content_hash TEXT,
status_code INTEGER NOT NULL,
content_type TEXT,
response_headers TEXT,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
expires_at DATETIME,
etag TEXT,
last_modified TEXT,
UNIQUE(source_host, source_path, source_query),
FOREIGN KEY (content_hash) REFERENCES source_content(content_hash)
);
CREATE INDEX IF NOT EXISTS idx_source_meta_host ON source_metadata(source_host);
CREATE INDEX IF NOT EXISTS idx_source_meta_path_hash ON source_metadata(path_hash);
CREATE INDEX IF NOT EXISTS idx_source_meta_expires ON source_metadata(expires_at);
CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(content_hash);
-- Processed variant blobs -- Processed variant blobs
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key>. Tracked here -- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
-- (like source content blobs above) so total cache usage is known -- sidecar with the content type). Tracked here (like source content
-- without a directory scan, and so LRU eviction has a timestamp to order -- blobs above) so total cache usage is known without a directory scan,
-- on. A cache hit does not read this table: it takes the content type -- and so LRU eviction has a timestamp to order on.
-- from the file's first bytes, and its last_accessed_at is written
-- later, in a batch.
CREATE TABLE IF NOT EXISTS variant_content ( CREATE TABLE IF NOT EXISTS variant_content (
cache_key TEXT PRIMARY KEY, cache_key TEXT PRIMARY KEY,
size_bytes INTEGER NOT NULL, size_bytes INTEGER NOT NULL,
content_type TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
); );
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
@@ -95,23 +120,59 @@ BEGIN
WHERE id = 1; WHERE id = 1;
END; END;
-- Negative cache: source URLs whose fetch failed with an upstream error -- Output/transformed content blobs
-- status, or whose first bytes did not match their Content-Type. pixa does -- Not written: transformed images are stored in cache/variants and
-- not fetch such a URL again until its row expires. -- tracked in variant_content above.
CREATE TABLE IF NOT EXISTS output_content (
content_hash TEXT PRIMARY KEY,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Request cache - maps full request params to output content
CREATE TABLE IF NOT EXISTS request_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT,
cache_key TEXT NOT NULL UNIQUE,
source_metadata_id INTEGER NOT NULL,
output_hash TEXT NOT NULL,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
format TEXT NOT NULL,
quality INTEGER NOT NULL DEFAULT 85,
fit_mode TEXT NOT NULL DEFAULT 'cover',
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
access_count INTEGER NOT NULL DEFAULT 1,
FOREIGN KEY (source_metadata_id) REFERENCES source_metadata(id),
FOREIGN KEY (output_hash) REFERENCES output_content(content_hash)
);
CREATE INDEX IF NOT EXISTS idx_request_cache_key ON request_cache(cache_key);
CREATE INDEX IF NOT EXISTS idx_request_cache_source ON request_cache(source_metadata_id);
CREATE INDEX IF NOT EXISTS idx_request_cache_output ON request_cache(output_hash);
CREATE INDEX IF NOT EXISTS idx_request_cache_fetched ON request_cache(fetched_at);
-- Negative cache for failed fetches (404s, timeouts, etc.)
CREATE TABLE IF NOT EXISTS negative_cache ( CREATE TABLE IF NOT EXISTS negative_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
source_host TEXT NOT NULL, source_host TEXT NOT NULL,
source_path TEXT NOT NULL, source_path TEXT NOT NULL,
source_query TEXT NOT NULL DEFAULT '', source_query TEXT NOT NULL DEFAULT '',
status_code INTEGER NOT NULL,
error_message TEXT,
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
expires_at DATETIME NOT NULL, expires_at DATETIME NOT NULL,
UNIQUE(source_host, source_path, source_query) UNIQUE(source_host, source_path, source_query)
); );
CREATE INDEX IF NOT EXISTS idx_negative_cache_expires ON negative_cache(expires_at); CREATE INDEX IF NOT EXISTS idx_negative_cache_expires ON negative_cache(expires_at);
-- A random value that a login session carries from its login. pixa makes it -- Cache statistics for monitoring
-- when the table has no row, and each logout replaces it; a login session CREATE TABLE IF NOT EXISTS cache_stats (
-- carrying another value is refused, so logging out ends every login session.
CREATE TABLE IF NOT EXISTS logout (
id INTEGER PRIMARY KEY CHECK (id = 1), id INTEGER PRIMARY KEY CHECK (id = 1),
logout_value TEXT NOT NULL hit_count INTEGER NOT NULL DEFAULT 0,
miss_count INTEGER NOT NULL DEFAULT 0,
upstream_fetch_count INTEGER NOT NULL DEFAULT 0,
upstream_fetch_bytes INTEGER NOT NULL DEFAULT 0,
transform_count INTEGER NOT NULL DEFAULT 0,
last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
); );
INSERT OR IGNORE INTO cache_stats (id) VALUES (1);
+70 -50
View File
@@ -7,15 +7,18 @@ import (
"github.com/fxamacker/cbor/v2" "github.com/fxamacker/cbor/v2"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/seal"
) )
// Default values for optional fields. // Default values for optional fields.
const ( const (
DefaultQuality = 85 DefaultQuality = 85
DefaultFormat = imagerequest.FormatJXL DefaultFormat = imgcache.FormatJXL
DefaultFitMode = imagerequest.FitCover DefaultFitMode = imgcache.FitCover
// HKDF salt for URL encryption key derivation
urlKeySalt = "pixa-urlenc-v1"
) )
// Errors returned by encurl operations. // Errors returned by encurl operations.
@@ -25,18 +28,18 @@ var (
ErrDecryptFailed = errors.New("failed to decrypt URL") ErrDecryptFailed = errors.New("failed to decrypt URL")
) )
// payload contains all image request parameters in a compact format. // Payload contains all image request parameters in a compact format.
// Field names are short to minimize encoded size. // Field names are short to minimize encoded size.
// Fields with omitempty use sane defaults when absent. // Fields with omitempty use sane defaults when absent.
type payload struct { type Payload struct {
SourceHost string `cbor:"h"` // required SourceHost string `cbor:"h"` // required
SourcePath string `cbor:"p"` // required SourcePath string `cbor:"p"` // required
SourceQuery string `cbor:"q,omitempty"` // optional SourceQuery string `cbor:"q,omitempty"` // optional
Width int `cbor:"w,omitempty"` // 0 = original Width int `cbor:"w,omitempty"` // 0 = original
Height int `cbor:"ht,omitempty"` // 0 = original Height int `cbor:"ht,omitempty"` // 0 = original
Format imagerequest.Format `cbor:"f,omitempty"` // default: jxl Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: jxl
Quality int `cbor:"ql,omitempty"` // default: 85 Quality int `cbor:"ql,omitempty"` // default: 85
FitMode imagerequest.Fit `cbor:"fm,omitempty"` // default: cover FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover
ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
} }
@@ -45,35 +48,20 @@ type Generator struct {
key [seal.KeySize]byte key [seal.KeySize]byte
} }
// NewGenerator creates an encrypted URL generator that encrypts its tokens // NewGenerator creates an encrypted URL generator with a key derived
// with key. // from the signing key.
func NewGenerator(key [seal.KeySize]byte) *Generator { func NewGenerator(signingKey string) (*Generator, error) {
return &Generator{key: key} key, err := seal.DeriveKey([]byte(signingKey), urlKeySalt)
if err != nil {
return nil, err
} }
// Generate creates an encrypted URL token from the request's source and return &Generator{key: key}, nil
// transform, each as the request holds it, and from expires; an empty format,
// a quality of 0 or an empty fit mode is left out of the token, and a zero
// expires makes a token that never expires. The token is CBOR-encoded,
// encrypted with NaCl secretbox, and base64url-encoded.
func (g *Generator) Generate(
req *imagerequest.Request, expires time.Time,
) (string, error) {
p := &payload{
SourceHost: req.Source.Host,
SourcePath: req.Source.Path,
SourceQuery: req.Source.Query,
Width: req.Transform.Size.Width,
Height: req.Transform.Size.Height,
Format: req.Transform.Format,
Quality: req.Transform.Quality,
FitMode: req.Transform.FitMode,
}
if !expires.IsZero() {
p.ExpiresAt = expires.Unix()
} }
// Generate creates an encrypted URL token from the payload.
// The token is CBOR-encoded, encrypted with NaCl secretbox, and base64url-encoded.
func (g *Generator) Generate(p *Payload) (string, error) {
// CBOR encode the payload // CBOR encode the payload
data, err := cbor.Marshal(p) data, err := cbor.Marshal(p)
if err != nil { if err != nil {
@@ -84,35 +72,40 @@ func (g *Generator) Generate(
return seal.Encrypt(g.key, data) return seal.Encrypt(g.key, data)
} }
// Parse decrypts and validates an encrypted URL token and returns the // Parse decrypts and validates an encrypted URL token.
// request it holds, with default values for omitted optional fields, and its
// expiry. A token that never expires gives the zero time.
// Returns ErrExpired if the token has expired, or other errors for invalid tokens. // Returns ErrExpired if the token has expired, or other errors for invalid tokens.
func (g *Generator) Parse(token string) (*imagerequest.Request, time.Time, error) { func (g *Generator) Parse(token string) (*Payload, error) {
// Decrypt // Decrypt
data, err := seal.Decrypt(g.key, token) data, err := seal.Decrypt(g.key, token)
if err != nil { if err != nil {
if errors.Is(err, seal.ErrDecryptionFailed) || if errors.Is(err, seal.ErrDecryptionFailed) ||
errors.Is(err, seal.ErrInvalidPayload) { errors.Is(err, seal.ErrInvalidPayload) {
return nil, time.Time{}, ErrDecryptFailed return nil, ErrDecryptFailed
} }
return nil, time.Time{}, err return nil, err
} }
// CBOR decode // CBOR decode
var p payload var p Payload
err = cbor.Unmarshal(data, &p) err = cbor.Unmarshal(data, &p)
if err != nil { if err != nil {
return nil, time.Time{}, ErrInvalidFormat return nil, ErrInvalidFormat
} }
// Check expiration (0 = never expires) // Check expiration (0 = never expires)
if p.ExpiresAt != 0 && time.Now().Unix() > p.ExpiresAt { if p.ExpiresAt != 0 && time.Now().Unix() > p.ExpiresAt {
return nil, time.Time{}, ErrExpired return nil, ErrExpired
} }
return &p, nil
}
// ToImageRequest converts the payload to an ImageRequest.
// Applies default values for omitted optional fields. An ExpiresAt of 0, a URL
// that never expires, gives the zero Expires.
func (p *Payload) ToImageRequest() *imgcache.ImageRequest {
format := p.Format format := p.Format
if format == "" { if format == "" {
format = DefaultFormat format = DefaultFormat
@@ -133,17 +126,44 @@ func (g *Generator) Parse(token string) (*imagerequest.Request, time.Time, error
expires = time.Unix(p.ExpiresAt, 0) expires = time.Unix(p.ExpiresAt, 0)
} }
return &imagerequest.Request{ return &imgcache.ImageRequest{
Source: imagerequest.Source{ SourceHost: p.SourceHost,
Host: p.SourceHost, SourcePath: p.SourcePath,
Path: p.SourcePath, SourceQuery: p.SourceQuery,
Query: p.SourceQuery, Size: imgcache.Size{
Width: p.Width,
Height: p.Height,
}, },
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: p.Width, Height: p.Height},
Format: format, Format: format,
Quality: quality, Quality: quality,
FitMode: fitMode, FitMode: fitMode,
}, Expires: expires,
}, expires, nil }
}
// FromImageRequest creates a Payload from an ImageRequest with the given expiration.
func FromImageRequest(req *imgcache.ImageRequest, expiresAt time.Time) *Payload {
p := &Payload{
SourceHost: req.SourceHost,
SourcePath: req.SourcePath,
SourceQuery: req.SourceQuery,
Width: req.Size.Width,
Height: req.Size.Height,
ExpiresAt: expiresAt.Unix(),
}
// Only set non-default values to benefit from omitempty
if req.Format != "" && req.Format != DefaultFormat {
p.Format = req.Format
}
if req.Quality != 0 && req.Quality != DefaultQuality {
p.Quality = req.Quality
}
if req.FitMode != "" && req.FitMode != DefaultFitMode {
p.FitMode = req.FitMode
}
return p
} }
+186 -108
View File
@@ -6,8 +6,7 @@ import (
"time" "time"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
) )
// Shared test fixture strings. // Shared test fixture strings.
@@ -20,29 +19,24 @@ const (
func TestGenerator_GenerateAndParse(t *testing.T) { func TestGenerator_GenerateAndParse(t *testing.T) {
t.Parallel() t.Parallel()
keys, err := seal.DeriveKeys("test-signing-key-12345") gen, err := encurl.NewGenerator("test-signing-key-12345")
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("NewGenerator() error = %v", err)
} }
gen := encurl.NewGenerator(keys.EncryptedURLKey) payload := &encurl.Payload{
SourceHost: testSourceHost,
req := &imagerequest.Request{ SourcePath: testSourcePath,
Source: imagerequest.Source{ SourceQuery: testSourceQuery,
Host: testSourceHost, Width: 800,
Path: testSourcePath, Height: 600,
Query: testSourceQuery, Format: imgcache.FormatWebP,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 800, Height: 600},
Format: imagerequest.FormatWebP,
Quality: 90, Quality: 90,
FitMode: imagerequest.FitContain, FitMode: imgcache.FitContain,
}, ExpiresAt: time.Now().Add(time.Hour).Unix(),
} }
expires := time.Now().Add(time.Hour)
token, err := gen.Generate(req, expires) token, err := gen.Generate(payload)
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
@@ -51,73 +45,66 @@ func TestGenerator_GenerateAndParse(t *testing.T) {
t.Fatal("Generate() returned empty token") t.Fatal("Generate() returned empty token")
} }
parsed, parsedExpires, err := gen.Parse(token) parsed, err := gen.Parse(token)
if err != nil { if err != nil {
t.Fatalf("Parse() error = %v", err) t.Fatalf("Parse() error = %v", err)
} }
// Verify all fields // Verify all fields
if parsed.Source.Host != req.Source.Host { if parsed.SourceHost != payload.SourceHost {
t.Errorf("SourceHost = %q, want %q", parsed.Source.Host, req.Source.Host) t.Errorf("SourceHost = %q, want %q", parsed.SourceHost, payload.SourceHost)
} }
if parsed.Source.Path != req.Source.Path { if parsed.SourcePath != payload.SourcePath {
t.Errorf("SourcePath = %q, want %q", parsed.Source.Path, req.Source.Path) t.Errorf("SourcePath = %q, want %q", parsed.SourcePath, payload.SourcePath)
} }
if parsed.Source.Query != req.Source.Query { if parsed.SourceQuery != payload.SourceQuery {
t.Errorf("SourceQuery = %q, want %q", parsed.Source.Query, req.Source.Query) t.Errorf("SourceQuery = %q, want %q", parsed.SourceQuery, payload.SourceQuery)
} }
if parsed.Transform.Size.Width != req.Transform.Size.Width { if parsed.Width != payload.Width {
t.Errorf("Width = %d, want %d", t.Errorf("Width = %d, want %d", parsed.Width, payload.Width)
parsed.Transform.Size.Width, req.Transform.Size.Width)
} }
if parsed.Transform.Size.Height != req.Transform.Size.Height { if parsed.Height != payload.Height {
t.Errorf("Height = %d, want %d", t.Errorf("Height = %d, want %d", parsed.Height, payload.Height)
parsed.Transform.Size.Height, req.Transform.Size.Height)
} }
if parsed.Transform.Format != req.Transform.Format { if parsed.Format != payload.Format {
t.Errorf("Format = %q, want %q", t.Errorf("Format = %q, want %q", parsed.Format, payload.Format)
parsed.Transform.Format, req.Transform.Format)
} }
if parsed.Transform.Quality != req.Transform.Quality { if parsed.Quality != payload.Quality {
t.Errorf("Quality = %d, want %d", t.Errorf("Quality = %d, want %d", parsed.Quality, payload.Quality)
parsed.Transform.Quality, req.Transform.Quality)
} }
if parsed.Transform.FitMode != req.Transform.FitMode { if parsed.FitMode != payload.FitMode {
t.Errorf("FitMode = %q, want %q", t.Errorf("FitMode = %q, want %q", parsed.FitMode, payload.FitMode)
parsed.Transform.FitMode, req.Transform.FitMode)
} }
if parsedExpires.Unix() != expires.Unix() { if parsed.ExpiresAt != payload.ExpiresAt {
t.Errorf("ExpiresAt = %d, want %d", parsedExpires.Unix(), expires.Unix()) t.Errorf("ExpiresAt = %d, want %d", parsed.ExpiresAt, payload.ExpiresAt)
} }
} }
func TestGenerator_Parse_Expired(t *testing.T) { func TestGenerator_Parse_Expired(t *testing.T) {
t.Parallel() t.Parallel()
keys, _ := seal.DeriveKeys("test-signing-key-12345") gen, _ := encurl.NewGenerator("test-signing-key-12345")
gen := encurl.NewGenerator(keys.EncryptedURLKey)
req := &imagerequest.Request{ payload := &encurl.Payload{
Source: imagerequest.Source{ SourceHost: testSourceHost,
Host: testSourceHost, SourcePath: testSourcePath,
Path: testSourcePath, ExpiresAt: time.Now().Add(-time.Hour).Unix(), // Already expired
},
} }
token, err := gen.Generate(req, time.Now().Add(-time.Hour)) // Already expired token, err := gen.Generate(payload)
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
_, _, err = gen.Parse(token) _, err = gen.Parse(token)
if err == nil { if err == nil {
t.Error("Parse() should fail for expired token") t.Error("Parse() should fail for expired token")
} }
@@ -130,10 +117,9 @@ func TestGenerator_Parse_Expired(t *testing.T) {
func TestGenerator_Parse_InvalidToken(t *testing.T) { func TestGenerator_Parse_InvalidToken(t *testing.T) {
t.Parallel() t.Parallel()
keys, _ := seal.DeriveKeys("test-signing-key-12345") gen, _ := encurl.NewGenerator("test-signing-key-12345")
gen := encurl.NewGenerator(keys.EncryptedURLKey)
_, _, err := gen.Parse("not-a-valid-token") _, err := gen.Parse("not-a-valid-token")
if err == nil { if err == nil {
t.Error("Parse() should fail for invalid token") t.Error("Parse() should fail for invalid token")
} }
@@ -142,17 +128,15 @@ func TestGenerator_Parse_InvalidToken(t *testing.T) {
func TestGenerator_Parse_TamperedToken(t *testing.T) { func TestGenerator_Parse_TamperedToken(t *testing.T) {
t.Parallel() t.Parallel()
keys, _ := seal.DeriveKeys("test-signing-key-12345") gen, _ := encurl.NewGenerator("test-signing-key-12345")
gen := encurl.NewGenerator(keys.EncryptedURLKey)
req := &imagerequest.Request{ payload := &encurl.Payload{
Source: imagerequest.Source{ SourceHost: testSourceHost,
Host: testSourceHost, SourcePath: testSourcePath,
Path: testSourcePath, ExpiresAt: time.Now().Add(time.Hour).Unix(),
},
} }
token, _ := gen.Generate(req, time.Now().Add(time.Hour)) token, _ := gen.Generate(payload)
// Tamper with the token // Tamper with the token
tampered := []byte(token) tampered := []byte(token)
@@ -160,7 +144,7 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) {
tampered[10] ^= 0x01 tampered[10] ^= 0x01
} }
_, _, err := gen.Parse(string(tampered)) _, err := gen.Parse(string(tampered))
if err == nil { if err == nil {
t.Error("Parse() should fail for tampered token") t.Error("Parse() should fail for tampered token")
} }
@@ -169,78 +153,172 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) {
func TestGenerator_Parse_WrongKey(t *testing.T) { func TestGenerator_Parse_WrongKey(t *testing.T) {
t.Parallel() t.Parallel()
keys1, _ := seal.DeriveKeys("signing-key-1") gen1, _ := encurl.NewGenerator("signing-key-1")
keys2, _ := seal.DeriveKeys("signing-key-2") gen2, _ := encurl.NewGenerator("signing-key-2")
gen1 := encurl.NewGenerator(keys1.EncryptedURLKey)
gen2 := encurl.NewGenerator(keys2.EncryptedURLKey)
req := &imagerequest.Request{ payload := &encurl.Payload{
Source: imagerequest.Source{ SourceHost: testSourceHost,
Host: testSourceHost, SourcePath: testSourcePath,
Path: testSourcePath, ExpiresAt: time.Now().Add(time.Hour).Unix(),
},
} }
token, _ := gen1.Generate(req, time.Now().Add(time.Hour)) token, _ := gen1.Generate(payload)
_, _, err := gen2.Parse(token) _, err := gen2.Parse(token)
if err == nil { if err == nil {
t.Error("Parse() should fail with different signing key") t.Error("Parse() should fail with different signing key")
} }
} }
func TestGenerator_Parse_Defaults(t *testing.T) { func TestPayload_ToImageRequest(t *testing.T) {
t.Parallel() t.Parallel()
keys, _ := seal.DeriveKeys("test-signing-key-12345") payload := &encurl.Payload{
gen := encurl.NewGenerator(keys.EncryptedURLKey) SourceHost: testSourceHost,
SourcePath: testSourcePath,
// Request with only required fields - should get defaults SourceQuery: testSourceQuery,
token, err := gen.Generate(&imagerequest.Request{ Width: 800,
Source: imagerequest.Source{ Height: 600,
Host: testSourceHost, Format: imgcache.FormatWebP,
Path: testSourcePath, Quality: 90,
}, FitMode: imgcache.FitContain,
}, time.Now().Add(time.Hour)) ExpiresAt: time.Now().Add(time.Hour).Unix(),
if err != nil {
t.Fatalf("Generate() error = %v", err)
} }
req, _, err := gen.Parse(token) req := payload.ToImageRequest()
if err != nil {
t.Fatalf("Parse() error = %v", err) if req.SourceHost != payload.SourceHost {
t.Errorf("SourceHost = %q, want %q", req.SourceHost, payload.SourceHost)
} }
if req.Transform.Format != encurl.DefaultFormat { if req.SourcePath != payload.SourcePath {
t.Errorf("Format = %q, want default %q", t.Errorf("SourcePath = %q, want %q", req.SourcePath, payload.SourcePath)
req.Transform.Format, encurl.DefaultFormat)
} }
if req.Transform.Quality != encurl.DefaultQuality { if req.SourceQuery != payload.SourceQuery {
t.Errorf("Quality = %d, want default %d", t.Errorf("SourceQuery = %q, want %q", req.SourceQuery, payload.SourceQuery)
req.Transform.Quality, encurl.DefaultQuality)
} }
if req.Transform.FitMode != encurl.DefaultFitMode { if req.Size.Width != payload.Width {
t.Errorf("FitMode = %q, want default %q", t.Errorf("Width = %d, want %d", req.Size.Width, payload.Width)
req.Transform.FitMode, encurl.DefaultFitMode) }
if req.Size.Height != payload.Height {
t.Errorf("Height = %d, want %d", req.Size.Height, payload.Height)
}
if req.Format != payload.Format {
t.Errorf("Format = %q, want %q", req.Format, payload.Format)
}
if req.Quality != payload.Quality {
t.Errorf("Quality = %d, want %d", req.Quality, payload.Quality)
}
if req.FitMode != payload.FitMode {
t.Errorf("FitMode = %q, want %q", req.FitMode, payload.FitMode)
}
}
func TestPayload_ToImageRequest_Defaults(t *testing.T) {
t.Parallel()
// Payload with only required fields - should get defaults
payload := &encurl.Payload{
SourceHost: testSourceHost,
SourcePath: testSourcePath,
ExpiresAt: time.Now().Add(time.Hour).Unix(),
}
req := payload.ToImageRequest()
if req.Format != encurl.DefaultFormat {
t.Errorf("Format = %q, want default %q", req.Format, encurl.DefaultFormat)
}
if req.Quality != encurl.DefaultQuality {
t.Errorf("Quality = %d, want default %d", req.Quality, encurl.DefaultQuality)
}
if req.FitMode != encurl.DefaultFitMode {
t.Errorf("FitMode = %q, want default %q", req.FitMode, encurl.DefaultFitMode)
}
}
func TestFromImageRequest(t *testing.T) {
t.Parallel()
req := &imgcache.ImageRequest{
SourceHost: testSourceHost,
SourcePath: testSourcePath,
SourceQuery: testSourceQuery,
Size: imgcache.Size{Width: 800, Height: 600},
Format: imgcache.FormatWebP,
Quality: 90,
FitMode: imgcache.FitContain,
}
expiresAt := time.Now().Add(time.Hour)
payload := encurl.FromImageRequest(req, expiresAt)
if payload.SourceHost != req.SourceHost {
t.Errorf("SourceHost = %q, want %q", payload.SourceHost, req.SourceHost)
}
if payload.SourcePath != req.SourcePath {
t.Errorf("SourcePath = %q, want %q", payload.SourcePath, req.SourcePath)
}
if payload.Width != req.Size.Width {
t.Errorf("Width = %d, want %d", payload.Width, req.Size.Width)
}
if payload.ExpiresAt != expiresAt.Unix() {
t.Errorf("ExpiresAt = %d, want %d", payload.ExpiresAt, expiresAt.Unix())
}
}
func TestFromImageRequest_OmitsDefaults(t *testing.T) {
t.Parallel()
// Request with default values - payload should omit them for
// smaller encoding
req := &imgcache.ImageRequest{
SourceHost: testSourceHost,
SourcePath: testSourcePath,
Format: encurl.DefaultFormat,
Quality: encurl.DefaultQuality,
FitMode: encurl.DefaultFitMode,
}
payload := encurl.FromImageRequest(req, time.Now().Add(time.Hour))
// These should be zero/empty because they match defaults
if payload.Format != "" {
t.Errorf("Format should be empty for default, got %q", payload.Format)
}
if payload.Quality != 0 {
t.Errorf("Quality should be 0 for default, got %d", payload.Quality)
}
if payload.FitMode != "" {
t.Errorf("FitMode should be empty for default, got %q", payload.FitMode)
} }
} }
func TestGenerator_TokenIsURLSafe(t *testing.T) { func TestGenerator_TokenIsURLSafe(t *testing.T) {
t.Parallel() t.Parallel()
keys, _ := seal.DeriveKeys("test-signing-key-12345") gen, _ := encurl.NewGenerator("test-signing-key-12345")
gen := encurl.NewGenerator(keys.EncryptedURLKey)
req := &imagerequest.Request{ payload := &encurl.Payload{
Source: imagerequest.Source{ SourceHost: testSourceHost,
Host: testSourceHost, SourcePath: testSourcePath,
Path: testSourcePath, ExpiresAt: time.Now().Add(time.Hour).Unix(),
},
} }
token, _ := gen.Generate(req, time.Now().Add(time.Hour)) token, _ := gen.Generate(payload)
// Check that token only contains URL-safe characters // Check that token only contains URL-safe characters
for _, c := range token { for _, c := range token {
+33 -103
View File
@@ -9,14 +9,12 @@ import (
"math" "math"
"net/http" "net/http"
"net/url" "net/url"
"slices"
"strconv" "strconv"
"strings"
"time" "time"
"sneak.berlin/go/pixa/internal/clientip" "sneak.berlin/go/pixa/internal/clientip"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/templates" "sneak.berlin/go/pixa/internal/templates"
) )
@@ -91,18 +89,10 @@ func (s *Handlers) handleLoginPost(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/", http.StatusSeeOther) http.Redirect(w, r, "/", http.StatusSeeOther)
} }
// HandleLogout logs out, which ends every login session (see // HandleLogout clears the session and redirects to login.
// session.Manager.Logout), and redirects to login.
func (s *Handlers) HandleLogout() http.HandlerFunc { func (s *Handlers) HandleLogout() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
err := s.sessMgr.Logout(w, r) s.sessMgr.ClearSession(w)
if err != nil {
s.log.Error("failed to log out", "error", err)
http.Error(w, "Failed to log out", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/", http.StatusSeeOther) http.Redirect(w, r, "/", http.StatusSeeOther)
} }
} }
@@ -127,7 +117,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
sourceURL := r.FormValue("url") sourceURL := r.FormValue("url")
// Validate source URL // Validate source URL
parsed, err := parseSourceURL(sourceURL) parsed, err := url.Parse(sourceURL)
if err != nil || parsed.Host == "" { if err != nil || parsed.Host == "" {
s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form, s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form,
http.StatusBadRequest) http.StatusBadRequest)
@@ -135,17 +125,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
return return
} }
// /v1/image/ refuses a ".." segment in the source path, and so does payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form)
// the generator. The path is decoded, so %2e%2e counts too
if slices.Contains(strings.Split(parsed.Path, "/"), "..") {
s.renderGeneratorWithForm(w, r,
"Source URL path cannot hold a .. segment", r.Form,
http.StatusBadRequest)
return
}
req, expiresAt, ttl, err := buildGenerateRequest(parsed, r.Form)
if err != nil { if err != nil {
s.renderGeneratorWithForm(w, r, err.Error(), r.Form, s.renderGeneratorWithForm(w, r, err.Error(), r.Form,
http.StatusBadRequest) http.StatusBadRequest)
@@ -153,20 +133,8 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
return return
} }
// The image URL must fetch the source URL as it was written. The
// fetcher fetches over http only when allow_http is set.
fetchURL := req.Source.URL(s.config.AllowHTTP)
if !fetchedAsWritten(parsed, fetchURL) {
s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+
"written: pixa would fetch "+fetchURL.String(),
r.Form, http.StatusBadRequest)
return
}
// Generate encrypted token // Generate encrypted token
token, err := s.encGen.Generate(req, expiresAt) token, err := s.encGen.Generate(payload)
if err != nil { if err != nil {
s.log.Error("failed to generate encrypted URL", "error", err) s.log.Error("failed to generate encrypted URL", "error", err)
s.renderGeneratorWithForm(w, r, "Failed to generate URL", r.Form, s.renderGeneratorWithForm(w, r, "Failed to generate URL", r.Form,
@@ -197,55 +165,14 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
} }
} }
// parseSourceURL parses the source URL entered on the generator page with its // buildGeneratePayload parses the numeric form fields and assembles the
// host lower-cased and an empty path written as "/", as /v1/image/ reads a // encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A
// source URL, so that each spelling of one source URL makes one cache key and // non-numeric or out-of-range width, height, quality or ttl, or an
// one upstream request. // unrecognized fit mode, is a client error naming the offending field. The
func parseSourceURL(sourceURL string) (*url.URL, error) { // format field is passed through unchecked.
parsed, err := url.Parse(sourceURL) func buildGeneratePayload(
if err != nil {
return nil, err
}
parsed.Host = strings.ToLower(parsed.Host)
if parsed.Path == "" {
parsed.Path = "/"
}
return parsed, nil
}
// fetchedAsWritten reports whether fetchURL, the URL pixa fetches for an
// image URL made from the source URL source, is source as it was written,
// apart from the scheme, which the token does not keep. It is not when source
// holds a user name and password, which the token does not keep either, or a
// "?" with no query after it. The token keeps the source path decoded and
// pixa fetches it percent-encoded again, so a path written another way, such
// as with %2F, is fetched as another path. url.URL keeps the path as written
// in RawPath when it is not the decoded path percent-encoded again.
// EscapedPath ignores RawPath when it also holds a character net/url encodes,
// such as a space, so RawPath is compared itself.
func fetchedAsWritten(source, fetchURL *url.URL) bool {
writtenPath := source.RawPath
if writtenPath == "" {
writtenPath = source.EscapedPath()
}
return source.User == nil && !source.ForceQuery &&
fetchURL.EscapedPath() == writtenPath &&
fetchURL.RawQuery == source.RawQuery
}
// buildGenerateRequest parses the numeric form fields and assembles the
// image request the encrypted URL's token is made from, and its expiry.
// ttl=0 means never expires (the expiry is the zero time). A non-numeric or
// out-of-range width, height, quality or ttl, or an unrecognized fit mode, is
// a client error naming the offending field. The format field is passed
// through unchecked.
func buildGenerateRequest(
parsed *url.URL, form url.Values, parsed *url.URL, form url.Values,
) (*imagerequest.Request, time.Time, int, error) { ) (*encurl.Payload, time.Time, int, error) {
width, err := parseFormDimension(form, "width") width, err := parseFormDimension(form, "width")
if err != nil { if err != nil {
return nil, time.Time{}, 0, err return nil, time.Time{}, 0, err
@@ -267,39 +194,42 @@ func buildGenerateRequest(
return nil, time.Time{}, 0, err return nil, time.Time{}, 0, err
} }
fitMode := imagerequest.Fit(form.Get("fit")) fitMode := imgcache.FitMode(form.Get("fit"))
err = imagerequest.ValidateFitMode(fitMode) err = imgcache.ValidateFitMode(fitMode)
if err != nil { if err != nil {
return nil, time.Time{}, 0, return nil, time.Time{}, 0,
fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, form.Get("fit")) fmt.Errorf("%w: %s", imgcache.ErrInvalidFitMode, form.Get("fit"))
} }
var expiresAt time.Time var (
expiresAt time.Time
expiresAtUnix int64
)
if ttl > 0 { if ttl > 0 {
expiresAt = time.Now().Add(time.Duration(ttl) * time.Second) expiresAt = time.Now().Add(time.Duration(ttl) * time.Second)
expiresAtUnix = expiresAt.Unix()
} }
req := &imagerequest.Request{ payload := &encurl.Payload{
Source: imagerequest.Source{ SourceHost: parsed.Host,
Host: parsed.Host, SourcePath: parsed.Path,
Path: parsed.Path, SourceQuery: parsed.RawQuery,
Query: parsed.RawQuery, Width: width,
}, Height: height,
Transform: imagerequest.Transform{ Format: imgcache.ImageFormat(form.Get("format")),
Size: imagerequest.Size{Width: width, Height: height},
Format: imagerequest.Format(form.Get("format")),
Quality: quality, Quality: quality,
FitMode: fitMode, FitMode: fitMode,
}, ExpiresAt: expiresAtUnix,
} }
return req, expiresAt, ttl, nil return payload, expiresAt, ttl, nil
} }
// parseFormDimension reads an optional width or height form field. An empty // parseFormDimension reads an optional width or height form field. An empty
// value means "original size" (0). A non-numeric value, or one // value means "original size" (0). A non-numeric value, or one
// imagerequest.ValidateDimension rejects, is an error naming the field. // imgcache.ValidateDimension rejects, is an error naming the field.
func parseFormDimension(form url.Values, field string) (int, error) { func parseFormDimension(form url.Values, field string) (int, error) {
raw := form.Get(field) raw := form.Get(field)
if raw == "" { if raw == "" {
@@ -311,7 +241,7 @@ func parseFormDimension(form url.Values, field string) (int, error) {
return 0, fmt.Errorf("%w %s: not a number", errInvalidFormField, field) return 0, fmt.Errorf("%w %s: not a number", errInvalidFormField, field)
} }
err = imagerequest.ValidateDimension(field, value) err = imgcache.ValidateDimension(field, value)
if err != nil { if err != nil {
return 0, err return 0, err
} }
+15 -6
View File
@@ -14,7 +14,6 @@ import (
"sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/session"
) )
@@ -45,17 +44,27 @@ func newCSRFTestRouter(t *testing.T) (*Handlers, http.Handler) {
cfg := &config.Config{SigningKey: testSigningKey, Debug: true} cfg := &config.Config{SigningKey: testSigningKey, Debug: true}
keys, err := seal.DeriveKeys(testSigningKey) sessMgr, err := session.NewManager(testSigningKey)
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("session.NewManager() error = %v", err)
}
encGen, err := encurl.NewGenerator(testSigningKey)
if err != nil {
t.Fatalf("encurl.NewGenerator() error = %v", err)
}
protect, err := newCSRFProtect(testSigningKey, cfg.Debug)
if err != nil {
t.Fatalf("newCSRFProtect() error = %v", err)
} }
h := &Handlers{ h := &Handlers{
log: slog.New(slog.DiscardHandler), log: slog.New(slog.DiscardHandler),
config: cfg, config: cfg,
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), sessMgr: sessMgr,
encGen: encurl.NewGenerator(keys.EncryptedURLKey), encGen: encGen,
csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug), csrfProtect: protect,
} }
r := chi.NewRouter() r := chi.NewRouter()
@@ -11,7 +11,7 @@ import (
"time" "time"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
) )
// Generator form field names, and a value that is not a number. // Generator form field names, and a value that is not a number.
@@ -125,10 +125,10 @@ func TestGeneratePostRejectsBadField(t *testing.T) {
} }
} }
// TestBuildGenerateRequestDefaultAndLimits verifies that an empty quality // TestBuildGeneratePayloadDefaultAndLimits verifies that an empty quality
// takes the default, and that the largest accepted width, height, quality and // takes the default, and that the largest accepted width, height, quality and
// ttl are accepted with an expiry still in the future. // ttl are accepted with an expiry still in the future.
func TestBuildGenerateRequestDefaultAndLimits(t *testing.T) { func TestBuildGeneratePayloadDefaultAndLimits(t *testing.T) {
t.Parallel() t.Parallel()
parsed, err := url.Parse(testSourceURL) parsed, err := url.Parse(testSourceURL)
@@ -136,19 +136,19 @@ func TestBuildGenerateRequestDefaultAndLimits(t *testing.T) {
t.Fatalf("url.Parse() error = %v", err) t.Fatalf("url.Parse() error = %v", err)
} }
req, _, _, err := buildGenerateRequest(parsed, url.Values{}) payload, _, _, err := buildGeneratePayload(parsed, url.Values{})
if err != nil { if err != nil {
t.Fatalf("empty form: error = %v", err) t.Fatalf("empty form: error = %v", err)
} }
if req.Transform.Quality != encurl.DefaultQuality { if payload.Quality != encurl.DefaultQuality {
t.Errorf("empty quality gave %d, want %d", t.Errorf("empty quality gave %d, want %d",
req.Transform.Quality, encurl.DefaultQuality) payload.Quality, encurl.DefaultQuality)
} }
_, expiresAt, _, err := buildGenerateRequest(parsed, url.Values{ _, expiresAt, _, err := buildGeneratePayload(parsed, url.Values{
widthField: {strconv.Itoa(imagerequest.MaxDimension)}, widthField: {strconv.Itoa(imgcache.MaxDimension)},
heightField: {strconv.Itoa(imagerequest.MaxDimension)}, heightField: {strconv.Itoa(imgcache.MaxDimension)},
qualityField: {strconv.Itoa(maxQuality)}, qualityField: {strconv.Itoa(maxQuality)},
ttlField: {strconv.Itoa(maxTTL)}, ttlField: {strconv.Itoa(maxTTL)},
}) })
@@ -10,7 +10,6 @@ import (
"testing" "testing"
"sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/session"
) )
@@ -23,15 +22,15 @@ func TestLoginLogLeavesOutSubmittedKey(t *testing.T) {
var buf bytes.Buffer var buf bytes.Buffer
keys, err := seal.DeriveKeys(testSigningKey) sessMgr, err := session.NewManager(testSigningKey)
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("session.NewManager() error = %v", err)
} }
h := &Handlers{ h := &Handlers{
log: slog.New(slog.NewJSONHandler(&buf, nil)), log: slog.New(slog.NewJSONHandler(&buf, nil)),
config: &config.Config{SigningKey: testSigningKey}, config: &config.Config{SigningKey: testSigningKey},
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), sessMgr: sessMgr,
} }
submittedKeys := []string{wrongKey, testSigningKey} submittedKeys := []string{wrongKey, testSigningKey}
+3 -101
View File
@@ -10,8 +10,7 @@ import (
"testing" "testing"
"time" "time"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/session"
) )
@@ -186,103 +185,6 @@ func TestHandleLogout_ClearsSessionCookie(t *testing.T) {
} }
} }
// TestHandleLogout_EndsCopiedSession verifies that after POST /logout a copy
// of the session cookie taken before it shows the login page, not the
// generator page.
func TestHandleLogout_EndsCopiedSession(t *testing.T) {
t.Parallel()
h, srv := newCSRFTestRouter(t)
copied := newSessionCookie(t, h)
req := httptest.NewRequestWithContext(
t.Context(), http.MethodPost, "/logout", nil)
req.AddCookie(copied)
rec := httptest.NewRecorder()
h.HandleLogout().ServeHTTP(rec, req)
if rec.Code != http.StatusSeeOther {
t.Fatalf("POST /logout status = %d, want %d",
rec.Code, http.StatusSeeOther)
}
req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
req.AddCookie(copied)
rec = httptest.NewRecorder()
srv.ServeHTTP(rec, req)
body := rec.Body.String()
t.Logf("GET / with the copied cookie: status %d, login page %v",
rec.Code, strings.Contains(body, loginForm))
if !strings.Contains(body, loginForm) || strings.Contains(body, generatorForm) {
t.Error("a cookie copied before /logout still opens the generator page")
}
}
// TestHandleLogout_SaveFails_Answers500 verifies that when the logout cannot
// be saved in the database, POST /logout answers 500, leaves the session
// cookie in place and ends no session.
func TestHandleLogout_SaveFails_Answers500(t *testing.T) {
t.Parallel()
h, srv := newCSRFTestRouter(t)
db := setupTestDB(t)
keys, err := seal.DeriveKeys(testSigningKey)
if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err)
}
sessMgr, err := session.NewManagerWithDatabase(
t.Context(), keys.SessionHashKey, keys.SessionBlockKey, db)
if err != nil {
t.Fatalf("session.NewManagerWithDatabase() error = %v", err)
}
h.sessMgr = sessMgr
cookie := newSessionCookie(t, h)
// Saving the logout fails on a closed database.
err = db.Close()
if err != nil {
t.Fatalf("closing the database: %v", err)
}
req := httptest.NewRequestWithContext(
t.Context(), http.MethodPost, "/logout", nil)
req.AddCookie(cookie)
rec := httptest.NewRecorder()
h.HandleLogout().ServeHTTP(rec, req)
t.Logf("POST /logout: status %d, Set-Cookie %q",
rec.Code, rec.Header().Get("Set-Cookie"))
if rec.Code != http.StatusInternalServerError {
t.Errorf("POST /logout status = %d, want %d",
rec.Code, http.StatusInternalServerError)
}
if c := findSessionCookie(rec); c != nil {
t.Errorf("a logout that failed set the session cookie: %s", c)
}
req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
req.AddCookie(cookie)
rec = httptest.NewRecorder()
srv.ServeHTTP(rec, req)
if !strings.Contains(rec.Body.String(), generatorForm) {
t.Error("a logout that failed ended the session")
}
}
// TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate // TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate
// with a valid CSRF token but no login session answers 303 to / and makes no // with a valid CSRF token but no login session answers 303 to / and makes no
// URL. // URL.
@@ -320,7 +222,7 @@ func TestGeneratePost_URLServesImage(t *testing.T) {
sourceURLField: {"https://" + signedHost + photoPath}, sourceURLField: {"https://" + signedHost + photoPath},
widthField: {"50"}, widthField: {"50"},
heightField: {"50"}, heightField: {"50"},
formatField: {string(imagerequest.FormatJPEG)}, formatField: {string(imgcache.FormatJPEG)},
}) })
if rec.Code != http.StatusOK { if rec.Code != http.StatusOK {
@@ -353,7 +255,7 @@ func TestGeneratePost_URLWithTTLExpires(t *testing.T) {
sourceURLField: {"https://" + signedHost + photoPath}, sourceURLField: {"https://" + signedHost + photoPath},
widthField: {"50"}, widthField: {"50"},
heightField: {"50"}, heightField: {"50"},
formatField: {string(imagerequest.FormatJPEG)}, formatField: {string(imgcache.FormatJPEG)},
ttlField: {"1"}, ttlField: {"1"},
}) })
@@ -1,39 +0,0 @@
package handlers
import (
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// TestAVIFOutputTooLargeAnswers400 checks that respondImageError, which
// answers the errors of both image routes, answers 400 naming the AVIF pixel
// limit when the image processor refuses an AVIF output above it, wrapped as
// the service wraps it. The request's source only goes into a log line, so it
// is left empty.
func TestAVIFOutputTooLargeAnswers400(t *testing.T) {
t.Parallel()
h := &Handlers{log: slog.New(slog.DiscardHandler)}
err := fmt.Errorf("image processing failed: %w: 4097x4096",
imageprocessor.ErrAVIFOutputTooLarge)
wantBody := fmt.Sprintf("above %d pixels", imageprocessor.MaxAVIFOutputPixels)
rec := httptest.NewRecorder()
h.respondImageError(rec, &imagerequest.Request{}, err)
t.Logf("status = %d, body = %q", rec.Code, rec.Body.String())
if rec.Code != http.StatusBadRequest {
t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
if !strings.Contains(rec.Body.String(), wantBody) {
t.Errorf("body = %q, want it to say %q", rec.Body.String(), wantBody)
}
}
+15 -6
View File
@@ -11,7 +11,6 @@ import (
"sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/session"
) )
@@ -33,17 +32,27 @@ func newBodyLimitTestRouter(
cfg := &config.Config{SigningKey: testSigningKey, Debug: true} cfg := &config.Config{SigningKey: testSigningKey, Debug: true}
keys, err := seal.DeriveKeys(testSigningKey) sessMgr, err := session.NewManager(testSigningKey)
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("session.NewManager() error = %v", err)
}
encGen, err := encurl.NewGenerator(testSigningKey)
if err != nil {
t.Fatalf("encurl.NewGenerator() error = %v", err)
}
protect, err := newCSRFProtect(testSigningKey, cfg.Debug)
if err != nil {
t.Fatalf("newCSRFProtect() error = %v", err)
} }
h := &Handlers{ h := &Handlers{
log: slog.New(slog.DiscardHandler), log: slog.New(slog.DiscardHandler),
config: cfg, config: cfg,
sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey), sessMgr: sessMgr,
encGen: encurl.NewGenerator(keys.EncryptedURLKey), encGen: encGen,
csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug), csrfProtect: protect,
} }
r := chi.NewRouter() r := chi.NewRouter()
@@ -1,278 +0,0 @@
package handlers
import (
"bytes"
"context"
"database/sql"
"database/sql/driver"
"image"
"image/jpeg"
"log/slog"
"net/http"
"net/http/httptest"
"path/filepath"
"sync/atomic"
"testing"
"testing/fstest"
"time"
"github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/signature"
)
// countingHost is the allowlisted host whose photoPath the server of
// newCountingDatabaseServer serves.
const countingHost = "counting.example.com"
// countingConnector opens connections to the SQLite database file name with
// sqliteDriver, each one counting in statements the statements run on it.
type countingConnector struct {
sqliteDriver driver.Driver
name string
statements *atomic.Int64
}
func (c countingConnector) Connect(context.Context) (driver.Conn, error) {
conn, err := c.sqliteDriver.Open(c.name)
if err != nil {
return nil, err
}
return countingConn{Conn: conn, statements: c.statements}, nil
}
func (c countingConnector) Driver() driver.Driver {
return c.sqliteDriver
}
// countingConn counts the statements run on its connection. It has only the
// methods of driver.Conn, so database/sql prepares every query and every
// other statement with Prepare, which counts it.
type countingConn struct {
driver.Conn
statements *atomic.Int64
}
func (c countingConn) Prepare(query string) (driver.Stmt, error) {
c.statements.Add(1)
return c.Conn.Prepare(query)
}
// newCountingDatabase returns a database with pixa's schema, in a file of
// its own and with one connection, and the count of the statements run on
// it once the schema is in place.
func newCountingDatabase(tb testing.TB) (*sql.DB, *atomic.Int64) {
tb.Helper()
path := filepath.Join(tb.TempDir(), "state.sqlite3")
schemaDB, err := sql.Open("sqlite", path)
if err != nil {
tb.Fatalf("opening the database: %v", err)
}
defer func() { _ = schemaDB.Close() }()
err = database.ApplyMigrations(tb.Context(), schemaDB, nil)
if err != nil {
tb.Fatalf("applying the migrations: %v", err)
}
statements := new(atomic.Int64)
db := sql.OpenDB(countingConnector{
sqliteDriver: schemaDB.Driver(),
name: path,
statements: statements,
})
db.SetMaxOpenConns(1)
tb.Cleanup(func() { _ = db.Close() })
return db, statements
}
// newCountingDatabaseServer returns a router for both image routes whose
// image cache uses newCountingDatabase, the count of the statements run on
// that database, and a URL on each route for a different image of
// photoPath on countingHost. Nothing writes the cache's access times in the
// background, so every statement counted comes from a request.
func newCountingDatabaseServer(
tb testing.TB,
) (http.Handler, *atomic.Int64, []string) {
tb.Helper()
db, statements := newCountingDatabase(tb)
var photo bytes.Buffer
err := jpeg.Encode(&photo, image.NewRGBA(image.Rect(0, 0, 100, 100)), nil)
if err != nil {
tb.Fatalf("encoding the photo: %v", err)
}
cache, err := imgcache.NewCache(db, imgcache.Config{
StateDir: tb.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
tb.Fatalf("imgcache.NewCache() error = %v", err)
}
svc, err := imageproxy.New(&imageproxy.Config{
Cache: cache,
Fetcher: newMockFetcher(fstest.MapFS{
countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()},
}),
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
MaxResponseSize: 50 << 20,
Logger: slog.New(slog.DiscardHandler),
}, prometheus.NewRegistry())
if err != nil {
tb.Fatalf("imageproxy.New() error = %v", err)
}
keys, err := seal.DeriveKeys(testSigningKey)
if err != nil {
tb.Fatalf("seal.DeriveKeys() error = %v", err)
}
encGen := encurl.NewGenerator(keys.EncryptedURLKey)
token, err := encGen.Generate(&imagerequest.Request{
Source: imagerequest.Source{Host: countingHost, Path: photoPath},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 40, Height: 40},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}, time.Time{})
if err != nil {
tb.Fatalf("encGen.Generate() error = %v", err)
}
h := &Handlers{
log: slog.New(slog.DiscardHandler),
imgSvc: svc,
encGen: encGen,
signer: signature.New(testSigningKey),
allowlist: allowlist.New([]string{countingHost}),
}
r := chi.NewRouter()
r.Get("/v1/image/*", h.HandleImage())
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
return r, statements, []string{
"/v1/image/" + countingHost + photoPath + "/50x50.jpeg",
"/v1/e/" + token + "/img.jpg",
}
}
// serveImage sends srv a GET for target and returns the response, failing
// the test unless it is a 200 with an image.
func serveImage(
tb testing.TB, srv http.Handler, target string,
) *httptest.ResponseRecorder {
tb.Helper()
rec := httptest.NewRecorder()
srv.ServeHTTP(rec,
httptest.NewRequestWithContext(tb.Context(), http.MethodGet, target, nil))
if rec.Code != http.StatusOK || rec.Body.Len() == 0 {
tb.Errorf("GET %s: %d with %d bytes, want 200 with an image",
target, rec.Code, rec.Body.Len())
}
return rec
}
// TestImageRoutes_CacheHitsMakeNoDatabaseCall serves an image on each image
// route once, which makes and caches it, then serves it again from the cache.
// Serving those cache hits must run no statement on the database.
func TestImageRoutes_CacheHitsMakeNoDatabaseCall(t *testing.T) {
t.Parallel()
srv, statements, targets := newCountingDatabaseServer(t)
for _, target := range targets {
before := statements.Load()
serveImage(t, srv, target)
afterMiss := statements.Load()
t.Logf("GET %s, a miss: %d statements", target, afterMiss-before)
if afterMiss == before {
t.Fatalf("GET %s, a miss: no statement counted, want some", target)
}
for range 3 {
rec := serveImage(t, srv, target)
cacheStatus := rec.Header().Get("X-Pixa-Cache")
if cacheStatus != string(imageproxy.CacheHit) {
t.Errorf("GET %s: X-Pixa-Cache = %q, want %s",
target, cacheStatus, imageproxy.CacheHit)
}
}
if hitStatements := statements.Load() - afterMiss; hitStatements != 0 {
t.Errorf("GET %s: 3 cache hits ran %d statements, want 0",
target, hitStatements)
}
}
}
// BenchmarkHandleImage_CacheHits serves one cached image through the
// /v1/image/ route from b.RunParallel's goroutines, and reports the cache
// hits served per second. It fails if serving them ran any statement on the
// database. script/bench runs it.
func BenchmarkHandleImage_CacheHits(b *testing.B) {
srv, statements, targets := newCountingDatabaseServer(b)
target := targets[0]
serveImage(b, srv, target)
before := statements.Load()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
rec := serveImage(b, srv, target)
cacheStatus := rec.Header().Get("X-Pixa-Cache")
if cacheStatus != string(imageproxy.CacheHit) {
b.Errorf("GET %s: X-Pixa-Cache = %q, want %s",
target, cacheStatus, imageproxy.CacheHit)
return
}
}
})
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "hits/s")
if hitStatements := statements.Load() - before; hitStatements != 0 {
b.Fatalf("%d cache hits ran %d statements, want 0", b.N, hitStatements)
}
}
@@ -0,0 +1,147 @@
package handlers
import (
"os"
"path/filepath"
"testing"
"go.uber.org/fx/fxtest"
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/globals"
"sneak.berlin/go/pixa/internal/logger"
)
// TestNewCacheConfigFromCacheMaxBytes checks the cache configuration
// built from cache_max_bytes: omitted, the cache works out the default
// limit; 0 turns the disk cache off; a positive value is the limit,
// unchanged.
func TestNewCacheConfigFromCacheMaxBytes(t *testing.T) {
t.Parallel()
const oneGiB = 1 << 30
cases := []struct {
name string
cacheMaxBytes int64
cacheMaxBytesExplicit bool
wantMaxBytes int64
wantUseDefaultMaxBytes bool
wantDisableDiskCache bool
}{
{
name: "cache_max_bytes omitted",
cacheMaxBytes: 0,
cacheMaxBytesExplicit: false,
wantMaxBytes: 0,
wantUseDefaultMaxBytes: true,
wantDisableDiskCache: false,
},
{
name: "cache_max_bytes: 0",
cacheMaxBytes: 0,
cacheMaxBytesExplicit: true,
wantMaxBytes: 0,
wantUseDefaultMaxBytes: false,
wantDisableDiskCache: true,
},
{
name: "cache_max_bytes: 1 GiB",
cacheMaxBytes: oneGiB,
cacheMaxBytesExplicit: true,
wantMaxBytes: oneGiB,
wantUseDefaultMaxBytes: false,
wantDisableDiskCache: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
cfg := &config.Config{
CacheMaxBytes: tc.cacheMaxBytes,
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
}
got := newCacheConfig(cfg, nil)
t.Logf("MaxBytes = %d, UseDefaultMaxBytes = %v, DisableDiskCache = %v",
got.MaxBytes, got.UseDefaultMaxBytes, got.DisableDiskCache)
if got.MaxBytes != tc.wantMaxBytes {
t.Errorf("MaxBytes = %d, want %d", got.MaxBytes, tc.wantMaxBytes)
}
if got.UseDefaultMaxBytes != tc.wantUseDefaultMaxBytes {
t.Errorf("UseDefaultMaxBytes = %v, want %v",
got.UseDefaultMaxBytes, tc.wantUseDefaultMaxBytes)
}
if got.DisableDiskCache != tc.wantDisableDiskCache {
t.Errorf("DisableDiskCache = %v, want %v",
got.DisableDiskCache, tc.wantDisableDiskCache)
}
})
}
}
// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes starts the handlers
// once with cache_max_bytes omitted and once with cache_max_bytes: 0,
// and checks by whether the cache directories were created that the
// disk cache is on in the first case and off in the second.
func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) {
t.Parallel()
cases := []struct {
name string
cacheMaxBytesExplicit bool
wantDiskCache bool
}{
{name: "cache_max_bytes omitted", cacheMaxBytesExplicit: false, wantDiskCache: true},
{name: "cache_max_bytes: 0", cacheMaxBytesExplicit: true, wantDiskCache: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
stateDir := t.TempDir()
cfg := &config.Config{
SigningKey: testSigningKey,
StateDir: stateDir,
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
CacheMaxBytes: 0,
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
}
lc := fxtest.NewLifecycle(t)
log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}})
if err != nil {
t.Fatalf("logger.New() error = %v", err)
}
db, err := database.New(lc, database.Params{Logger: log, Config: cfg})
if err != nil {
t.Fatalf("database.New() error = %v", err)
}
_, err = New(lc, Params{Logger: log, Database: db, Config: cfg})
if err != nil {
t.Fatalf("New() error = %v", err)
}
lc.RequireStart()
t.Cleanup(lc.RequireStop)
_, err = os.Stat(filepath.Join(stateDir, "cache", "variants"))
gotDiskCache := err == nil
if gotDiskCache != tc.wantDiskCache {
t.Errorf("cache directories created = %v, want %v",
gotDiskCache, tc.wantDiskCache)
}
})
}
}
@@ -1,177 +0,0 @@
package handlers
import (
"bytes"
"context"
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"slices"
"testing"
"time"
"github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// heldFetcher sends on started when a fetch begins, and holds the fetch until
// release is closed. It then fails the fetch as an upstream error.
type heldFetcher struct {
started chan struct{}
release chan struct{}
}
// Fetch sends on f.started, waits for f.release to be closed, and returns
// httpfetcher.ErrUpstreamError.
func (f heldFetcher) Fetch(
context.Context, imagerequest.Source,
) (*httpfetcher.FetchResult, error) {
f.started <- struct{}{}
<-f.release
return nil, httpfetcher.ErrUpstreamError
}
// readLogLines decodes the JSON log lines in logs.
func readLogLines(t *testing.T, logs *bytes.Buffer) []map[string]any {
t.Helper()
t.Logf("logged:\n%s", logs.String())
var lines []map[string]any
dec := json.NewDecoder(logs)
for dec.More() {
var line map[string]any
err := dec.Decode(&line)
if err != nil {
t.Fatalf("decoding log line: %v", err)
}
lines = append(lines, line)
}
return lines
}
// TestImageRoutes_ClientGoneWhileWaitingAnswers499 checks that both image
// routes answer 499 to a request whose client goes away while it waits for an
// image another request is making. A first request's fetch is held, so the
// image is being made, and a second request for the same image waits for it
// until its client goes away. The second request must be logged once at info
// level with its request ID and the source host and path. Nothing may be
// logged at error level.
func TestImageRoutes_ClientGoneWhileWaitingAnswers499(t *testing.T) {
t.Parallel()
const requestID = "test-request-id"
for route, imageURL := range photoURLsByRoute() {
t.Run(route, func(t *testing.T) {
t.Parallel()
var logs bytes.Buffer
fetcher := heldFetcher{started: make(chan struct{}), release: make(chan struct{})}
h, routes := newRoutesFetchingThrough(t, fetcher,
slog.New(slog.NewJSONHandler(&logs, nil)))
target := imageURL(t, h)
firstDone := make(chan struct{})
go func() {
defer close(firstDone)
sendGet(t, routes, target)
}()
<-fetcher.started
ctx, cancel := context.WithCancel(context.WithValue(t.Context(),
middleware.RequestIDKey, requestID))
rec := httptest.NewRecorder()
secondDone := make(chan struct{})
go func() {
defer close(secondDone)
routes.ServeHTTP(rec, httptest.NewRequestWithContext(
ctx, http.MethodGet, target, nil))
}()
// Give the second request time to start waiting for the first
// request's image. The fetch is still held, so the second request
// can end only because its client went away
time.Sleep(100 * time.Millisecond)
cancel()
<-secondDone
close(fetcher.release)
<-firstDone
if rec.Code != 499 {
t.Errorf("status = %d, want 499", rec.Code)
}
var clientGoneLines int
for _, line := range readLogLines(t, &logs) {
if line["level"] == slog.LevelError.String() {
t.Errorf("logged at error level: %v", line["msg"])
}
if line["level"] == slog.LevelInfo.String() &&
line["msg"] == "client went away before the image was ready" &&
line["request_id"] == requestID &&
line["host"] == signedHost &&
line["path"] == photoPath {
clientGoneLines++
}
}
if clientGoneLines != 1 {
t.Errorf("info line naming the request logged %d times, want 1",
clientGoneLines)
}
})
}
}
// TestImageRoutes_CanceledWithLiveRequestAnswers500 checks that both image
// routes answer 500, logged as "failed to get image" at error level, for an
// image service error that wraps context.Canceled while the request's own
// context is live. Its client has not gone away, so the cancellation is
// pixa's own failure.
func TestImageRoutes_CanceledWithLiveRequestAnswers500(t *testing.T) {
t.Parallel()
for route, imageURL := range photoURLsByRoute() {
t.Run(route, func(t *testing.T) {
t.Parallel()
var logs bytes.Buffer
h, routes := newRoutesFetchingThrough(t,
failingFetcher{err: context.Canceled},
slog.New(slog.NewJSONHandler(&logs, nil)))
rec := sendGet(t, routes, imageURL(t, h))
if rec.Code != http.StatusInternalServerError {
t.Errorf("status = %d, want %d",
rec.Code, http.StatusInternalServerError)
}
if !slices.ContainsFunc(readLogLines(t, &logs),
func(line map[string]any) bool {
return line["level"] == slog.LevelError.String() &&
line["msg"] == "failed to get image"
}) {
t.Error(`no "failed to get image" line logged at error level`)
}
})
}
}
+15 -5
View File
@@ -9,10 +9,15 @@ import (
"sneak.berlin/go/pixa/internal/seal" "sneak.berlin/go/pixa/internal/seal"
) )
// csrfKeySalt provides domain separation for the CSRF authentication key,
// derived from the signing key so tokens survive restarts without extra
// configuration and never reuse the session or encrypted-URL key material.
const csrfKeySalt = "pixa-csrf-v1"
// newCSRFProtect builds the CSRF-protection middleware for the // newCSRFProtect builds the CSRF-protection middleware for the
// state-mutating HTML form routes. The token lives in its own cookie, // state-mutating HTML form routes. The token lives in its own cookie,
// independent of the session cookie, so it also protects the login POST // independent of the session cookie, so it also protects the login POST
// where no session exists yet (login CSRF). key signs that cookie. // where no session exists yet (login CSRF).
// //
// When plaintext is true (local HTTP development), requests are marked // When plaintext is true (local HTTP development), requests are marked
// plaintext so the library neither demands an https Referer nor sets a // plaintext so the library neither demands an https Referer nor sets a
@@ -20,8 +25,13 @@ import (
// service runs behind a TLS-terminating proxy, so plaintext is false and // service runs behind a TLS-terminating proxy, so plaintext is false and
// the library enforces its https Referer origin check. // the library enforces its https Referer origin check.
func newCSRFProtect( func newCSRFProtect(
key [seal.KeySize]byte, plaintext bool, signingKey string, plaintext bool,
) func(http.Handler) http.Handler { ) (func(http.Handler) http.Handler, error) {
key, err := seal.DeriveKey([]byte(signingKey), csrfKeySalt)
if err != nil {
return nil, err
}
protect := csrf.Protect( protect := csrf.Protect(
key[:], key[:],
csrf.Path("/"), csrf.Path("/"),
@@ -30,7 +40,7 @@ func newCSRFProtect(
) )
if !plaintext { if !plaintext {
return protect return protect, nil
} }
return func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler {
@@ -39,7 +49,7 @@ func newCSRFProtect(
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
protected.ServeHTTP(w, csrf.PlaintextHTTPRequest(r)) protected.ServeHTTP(w, csrf.PlaintextHTTPRequest(r))
}) })
} }, nil
} }
// CSRF returns the CSRF-protection middleware for the login and generator // CSRF returns the CSRF-protection middleware for the login and generator
@@ -0,0 +1,61 @@
package handlers
import (
"net/http"
"net/netip"
"path/filepath"
"testing"
"time"
"github.com/go-chi/chi/v5"
"go.uber.org/fx"
"go.uber.org/fx/fxtest"
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/globals"
"sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/logger"
)
// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as
// pixad does, in an fx app that provides no fetcher, and requests an image
// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL
// check accepts that address; only the dialer that refuses internal
// addresses checks blocked_networks, so the answer is 403 only if the
// fetcher the handlers build from the config connects with that dialer. Any
// other dialer would try to connect until the upstream fetch timeout, which
// is short so that the test then fails quickly.
func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) {
t.Parallel()
const host = "192.0.2.10"
stateDir := t.TempDir()
cfg := &config.Config{
SigningKey: testSigningKey,
StateDir: stateDir,
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
AllowlistHosts: []string{host},
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
UpstreamFetchTimeout: 2 * time.Second,
// With no connection slots, the fetch would fail before dialing.
UpstreamConnections: config.DefaultUpstreamConnections,
}
var h *Handlers
app := fxtest.New(t,
fx.Supply(cfg),
fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New),
fx.Populate(&h),
)
app.RequireStart()
t.Cleanup(app.RequireStop)
r := chi.NewRouter()
r.Get("/v1/image/*", h.HandleImage())
rec := sendGet(t, r, photoURL(host))
checkErrorBody(t, rec, http.StatusForbidden, "forbidden")
}
@@ -1,203 +0,0 @@
package handlers
import (
"bytes"
"encoding/json"
"image/color"
"log/slog"
"net/http"
"slices"
"testing"
"testing/fstest"
"time"
"github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/signature"
)
// mismatchedSourcePath is the path of the source that
// TestFirstBytesNotMatchingContentType serves on allowlistedHost. The mock
// fetcher names a file's Content-Type from its extension, so the source is
// sent as image/png.
const mismatchedSourcePath = "/images/source.png"
// newRoutesFetchingThrough returns both image routes, and the Handlers behind
// them, whose image service fetches through fetcher, and which have
// allowlistedHost on their allowlist. The handlers and the image service log
// to log.
func newRoutesFetchingThrough(
t *testing.T, fetcher httpfetcher.Fetcher, log *slog.Logger,
) (*Handlers, http.Handler) {
t.Helper()
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err)
}
svc, err := imageproxy.New(&imageproxy.Config{
Cache: cache,
Fetcher: fetcher,
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
MaxResponseSize: 50 << 20,
Logger: log,
}, prometheus.NewRegistry())
if err != nil {
t.Fatalf("imageproxy.New() error = %v", err)
}
keys, err := seal.DeriveKeys(testSigningKey)
if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err)
}
encGen := encurl.NewGenerator(keys.EncryptedURLKey)
h := &Handlers{
log: log,
imgSvc: svc,
encGen: encGen,
signer: signature.New(testSigningKey),
allowlist: allowlist.New([]string{allowlistedHost}),
}
r := chi.NewRouter()
r.Get("/v1/image/*", h.HandleImage())
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
return h, r
}
// mismatchedSourcePlainURL returns the /v1/image/ URL of mismatchedSourcePath
// on allowlistedHost at 50x50 as a JPEG.
func mismatchedSourcePlainURL(*testing.T, *Handlers) string {
return "/v1/image/" + allowlistedHost + mismatchedSourcePath + "/50x50.jpeg"
}
// mismatchedSourceEncURL returns an encrypted /v1/e/ URL, made with h's
// generator, of mismatchedSourcePath on allowlistedHost at 50x50 as a JPEG.
func mismatchedSourceEncURL(t *testing.T, h *Handlers) string {
t.Helper()
token, err := h.encGen.Generate(&imagerequest.Request{
Source: imagerequest.Source{
Host: allowlistedHost,
Path: mismatchedSourcePath,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
},
}, time.Time{})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
return "/v1/e/" + token + "/img.jpg"
}
// loggedLevels reads the JSON log lines in logs and returns the messages
// logged at each level.
func loggedLevels(t *testing.T, logs *bytes.Buffer) map[string][]string {
t.Helper()
t.Logf("logged:\n%s", logs.String())
messages := make(map[string][]string)
dec := json.NewDecoder(logs)
for dec.More() {
var line struct {
Level string `json:"level"`
Msg string `json:"msg"`
}
err := dec.Decode(&line)
if err != nil {
t.Fatalf("decoding log line: %v", err)
}
messages[line.Level] = append(messages[line.Level], line.Msg)
}
return messages
}
// TestFirstBytesNotMatchingContentType checks that both image routes answer
// 415 for an upstream body whose first bytes are not those of the format its
// Content-Type names, and log it as a refused source, not as an error of
// pixa's. The next request for that source must answer 502 without fetching
// it again, as after an upstream error status.
func TestFirstBytesNotMatchingContentType(t *testing.T) {
t.Parallel()
bodies := map[string][]byte{
"JPEG sent as image/png": generateTestJPEG(t, 10, 10, color.Black),
"text sent as image/png": []byte("This is just text, not a PNG"),
"body too short to name a format": []byte("short"),
}
// The source's URL on each image route.
sourceURLs := map[string]func(*testing.T, *Handlers) string{
"plain URL": mismatchedSourcePlainURL,
"encrypted URL": mismatchedSourceEncURL,
}
for bodyName, body := range bodies {
for urlName, sourceURL := range sourceURLs {
t.Run(urlName+", "+bodyName, func(t *testing.T) {
t.Parallel()
fetcher := &countingFetcher{
Fetcher: httpfetcher.NewMock(fstest.MapFS{
allowlistedHost + mismatchedSourcePath: &fstest.MapFile{
Data: body,
},
}),
}
var logs bytes.Buffer
h, routes := newRoutesFetchingThrough(t, fetcher,
slog.New(slog.NewJSONHandler(&logs, nil)))
target := sourceURL(t, h)
checkErrorBody(t, sendGet(t, routes, target),
http.StatusUnsupportedMediaType, "unsupported source format")
levels := loggedLevels(t, &logs)
if len(levels[slog.LevelError.String()]) != 0 {
t.Errorf("logged at error level: %q",
levels[slog.LevelError.String()])
}
if !slices.Contains(levels[slog.LevelWarn.String()], "source refused") {
t.Error(`no "source refused" line logged at warning level`)
}
checkErrorBody(t, sendGet(t, routes, target),
http.StatusBadGateway, "upstream error")
if n := fetcher.fetches.Load(); n != 1 {
t.Errorf("upstream fetched %d times, want 1", n)
}
})
}
}
}
+23 -22
View File
@@ -8,7 +8,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
) )
// Errors for an Accept header that an auto URL cannot be served for. // Errors for an Accept header that an auto URL cannot be served for.
@@ -18,17 +18,16 @@ var (
"image/jxl, image/avif, image/webp or image/jpeg") "image/jxl, image/avif, image/webp or image/jpeg")
) )
// chooseAutoFormat returns the format to serve for the requested format: for // chooseAutoFormat replaces the format auto in req with the format
// the format auto, the format formatForAccept chooses from r's Accept header, // formatForAccept chooses from r's Accept header, and adds Vary: Accept to the
// and any other format as it is. For auto it adds Vary: Accept to the
// response, which then depends on that header. It answers 400 for an Accept // response, which then depends on that header. It answers 400 for an Accept
// header that is not valid and 406 for one that allows none of the formats, // header that is not valid and 406 for one that allows none of the formats,
// and reports whether the image can be served. // and reports whether req can be served. Any other format is left as it is.
func (s *Handlers) chooseAutoFormat( func (s *Handlers) chooseAutoFormat(
w http.ResponseWriter, r *http.Request, requested imagerequest.Format, w http.ResponseWriter, r *http.Request, req *imgcache.ImageRequest,
) (imagerequest.Format, bool) { ) bool {
if requested != imagerequest.FormatAuto { if req.Format != imgcache.FormatAuto {
return requested, true return true
} }
w.Header().Add("Vary", "Accept") w.Header().Add("Vary", "Accept")
@@ -37,16 +36,18 @@ func (s *Handlers) chooseAutoFormat(
if errors.Is(err, errNotAcceptable) { if errors.Is(err, errNotAcceptable) {
s.respondError(w, err.Error(), http.StatusNotAcceptable) s.respondError(w, err.Error(), http.StatusNotAcceptable)
return "", false return false
} }
if err != nil { if err != nil {
s.respondError(w, err.Error(), http.StatusBadRequest) s.respondError(w, err.Error(), http.StatusBadRequest)
return "", false return false
} }
return format, true req.Format = format
return true
} }
// formatForAccept returns the format an auto URL is served in for the Accept // formatForAccept returns the format an auto URL is served in for the Accept
@@ -55,30 +56,30 @@ func (s *Handlers) chooseAutoFormat(
// specific entry of image/jpeg, image/* and */* allows it, or when it names // specific entry of image/jpeg, image/* and */* allows it, or when it names
// nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as // nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as
// clients that cannot show them send image/* and */* too. // clients that cannot show them send image/* and */* too.
func formatForAccept(accept string) (imagerequest.Format, error) { func formatForAccept(accept string) (imgcache.ImageFormat, error) {
qualities, err := parseAccept(accept) qualities, err := parseAccept(accept)
if err != nil { if err != nil {
return "", err return "", err
} }
if len(qualities) == 0 { if len(qualities) == 0 {
return imagerequest.FormatJPEG, nil return imgcache.FormatJPEG, nil
} }
if qualities[imagerequest.FormatJXL.MIMEType()] > 0 { if qualities["image/jxl"] > 0 {
return imagerequest.FormatJXL, nil return imgcache.FormatJXL, nil
} }
if qualities[imagerequest.FormatAVIF.MIMEType()] > 0 { if qualities["image/avif"] > 0 {
return imagerequest.FormatAVIF, nil return imgcache.FormatAVIF, nil
} }
if qualities[imagerequest.FormatWebP.MIMEType()] > 0 { if qualities["image/webp"] > 0 {
return imagerequest.FormatWebP, nil return imgcache.FormatWebP, nil
} }
// For JPEG, the most specific entry the header has decides // For JPEG, the most specific entry the header has decides
quality, named := qualities[imagerequest.FormatJPEG.MIMEType()] quality, named := qualities["image/jpeg"]
if !named { if !named {
quality, named = qualities["image/*"] quality, named = qualities["image/*"]
} }
@@ -88,7 +89,7 @@ func formatForAccept(accept string) (imagerequest.Format, error) {
} }
if quality > 0 { if quality > 0 {
return imagerequest.FormatJPEG, nil return imgcache.FormatJPEG, nil
} }
return "", errNotAcceptable return "", errNotAcceptable
+39 -44
View File
@@ -12,7 +12,8 @@ import (
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imgcache"
) )
// The content types the tests below expect. // The content types the tests below expect.
@@ -32,24 +33,24 @@ func TestFormatForAccept(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
accept string accept string
want imagerequest.Format want imgcache.ImageFormat
wantErr error wantErr error
}{ }{
{"AVIF-capable browser", {"AVIF-capable browser",
"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
imagerequest.FormatAVIF, nil}, imgcache.FormatAVIF, nil},
{"WebP-capable browser", {"WebP-capable browser",
"image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5", "image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5",
imagerequest.FormatWebP, nil}, imgcache.FormatWebP, nil},
{"WebP only", webpType, imagerequest.FormatWebP, nil}, {"WebP only", webpType, imgcache.FormatWebP, nil},
{"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imagerequest.FormatJPEG, nil}, {"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imgcache.FormatJPEG, nil},
{"wildcard only", "*/*", imagerequest.FormatJPEG, nil}, {"wildcard only", "*/*", imgcache.FormatJPEG, nil},
{"image wildcard only", "image/*", imagerequest.FormatJPEG, nil}, {"image wildcard only", "image/*", imgcache.FormatJPEG, nil},
{"absent", "", imagerequest.FormatJPEG, nil}, {"absent", "", imgcache.FormatJPEG, nil},
{"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imagerequest.FormatWebP, nil}, {"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imgcache.FormatWebP, nil},
{"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*", {"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*",
imagerequest.FormatJPEG, nil}, imgcache.FormatJPEG, nil},
{"upper case and spaces", " Image/AVIF ; Q=0.5 ", imagerequest.FormatAVIF, nil}, {"upper case and spaces", " Image/AVIF ; Q=0.5 ", imgcache.FormatAVIF, nil},
{"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable}, {"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable},
{"q=0 on everything", "*/*;q=0", "", errNotAcceptable}, {"q=0 on everything", "*/*;q=0", "", errNotAcceptable},
{"PNG only", "image/png", "", errNotAcceptable}, {"PNG only", "image/png", "", errNotAcceptable},
@@ -75,33 +76,28 @@ func TestFormatForAccept(t *testing.T) {
// autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL, // autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL,
// both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in // both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in
// the format auto. The first is signed with testSigningKey, the key the // the format auto, made with h's image service and generator.
// handlers that newSignedHostServer builds check signatures with. The second
// is made with h's generator.
func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) { func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) {
t.Helper() t.Helper()
signedURL := generateSignedURL(testSigningKey, &imagerequest.Request{ signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Size: imgcache.Size{Width: 50, Height: 50},
}, Format: imgcache.FormatAuto,
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatAuto,
},
}, time.Minute) }, time.Minute)
if err != nil {
t.Fatalf("GenerateSignedURL() error = %v", err)
}
token, err := h.encGen.Generate(&imagerequest.Request{ token, err := h.encGen.Generate(&encurl.Payload{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{ Format: imgcache.FormatAuto,
Size: imagerequest.Size{Width: 50, Height: 50}, ExpiresAt: time.Now().Add(time.Minute).Unix(),
Format: imagerequest.FormatAuto, })
},
}, time.Now().Add(time.Minute))
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
@@ -186,18 +182,17 @@ func TestFormatAuto_ChosenFromAccept(t *testing.T) {
func TestFormatAuto_SignatureCoversAuto(t *testing.T) { func TestFormatAuto_SignatureCoversAuto(t *testing.T) {
t.Parallel() t.Parallel()
_, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
signedForAVIF := generateSignedURL(testSigningKey, &imagerequest.Request{ signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Size: imgcache.Size{Width: 50, Height: 50},
}, Format: imgcache.FormatAVIF,
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatAVIF,
},
}, time.Minute) }, time.Minute)
if err != nil {
t.Fatalf("GenerateSignedURL() error = %v", err)
}
target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1) target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1)
if target == signedForAVIF { if target == signedForAVIF {
+128 -21
View File
@@ -4,6 +4,7 @@ package handlers
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"log/slog" "log/slog"
"net/http" "net/http"
"time" "time"
@@ -11,13 +12,13 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/config" "sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/healthcheck" "sneak.berlin/go/pixa/internal/healthcheck"
"sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/logger" "sneak.berlin/go/pixa/internal/logger"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session" "sneak.berlin/go/pixa/internal/session"
"sneak.berlin/go/pixa/internal/signature"
) )
// Params defines dependencies for Handlers. // Params defines dependencies for Handlers.
@@ -26,48 +27,70 @@ type Params struct {
Logger *logger.Logger Logger *logger.Logger
Healthcheck *healthcheck.Healthcheck Healthcheck *healthcheck.Healthcheck
Database *database.Database
Config *config.Config Config *config.Config
Keys seal.Keys
ImageService *imageproxy.Proxy // Fetcher, when provided, fetches upstream images in place of the
SessionManager *session.Manager // fetcher the handlers build from the config. Only tests provide one;
EncryptedURLGenerator *encurl.Generator // pixad does not.
Fetcher httpfetcher.Fetcher `optional:"true"`
} }
// Handlers provides HTTP request handlers. // Handlers provides HTTP request handlers.
type Handlers struct { type Handlers struct {
log *slog.Logger log *slog.Logger
hc *healthcheck.Healthcheck hc *healthcheck.Healthcheck
db *database.Database
config *config.Config config *config.Config
imgSvc *imageproxy.Proxy fetcher httpfetcher.Fetcher
imgSvc *imgcache.Service
imgCache *imgcache.Cache
sessMgr *session.Manager sessMgr *session.Manager
encGen *encurl.Generator encGen *encurl.Generator
csrfProtect func(http.Handler) http.Handler csrfProtect func(http.Handler) http.Handler
// signer checks the signature of a /v1/image/ URL with signing_key.
signer *signature.Signer
// allowlist matches the hosts of allowlist_hosts, which a /v1/image/
// URL may name without a signature.
allowlist *allowlist.HostAllowList
// refererBlocklist matches the hosts of referer_blocklist; its IsAllowed // refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
// reports whether a URL's host is on that list. // reports whether a URL's host is on that list.
refererBlocklist *allowlist.HostAllowList refererBlocklist *allowlist.HostAllowList
} }
// New creates a new Handlers instance. // New creates a new Handlers instance.
func New(_ fx.Lifecycle, params Params) *Handlers { func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
return &Handlers{ csrfProtect, err := newCSRFProtect(params.Config.SigningKey, params.Config.Debug)
if err != nil {
return nil, err
}
s := &Handlers{
log: params.Logger.Get(), log: params.Logger.Get(),
hc: params.Healthcheck, hc: params.Healthcheck,
db: params.Database,
config: params.Config, config: params.Config,
imgSvc: params.ImageService, fetcher: params.Fetcher,
sessMgr: params.SessionManager, csrfProtect: csrfProtect,
encGen: params.EncryptedURLGenerator,
csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug),
signer: signature.New(params.Config.SigningKey),
allowlist: allowlist.New(params.Config.AllowlistHosts),
refererBlocklist: allowlist.New(params.Config.RefererBlocklist), refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
} }
lc.Append(fx.Hook{
//nolint:contextcheck // the cache's goroutines outlive OnStart; OnStop stops them
OnStart: func(_ context.Context) error {
return s.initImageService()
},
// The pending counts are written here, before the database's stop
// hook closes it.
OnStop: func(ctx context.Context) error {
if s.imgCache == nil {
return nil
}
return errors.Join(
s.imgCache.StopEviction(ctx),
s.imgCache.StopPendingCountWrites(ctx),
)
},
})
return s, nil
} }
// WaitForProcessing waits until no image is being processed, or until ctx // WaitForProcessing waits until no image is being processed, or until ctx
@@ -76,6 +99,90 @@ func (s *Handlers) WaitForProcessing(ctx context.Context) int {
return s.imgSvc.WaitForProcessing(ctx) return s.imgSvc.WaitForProcessing(ctx)
} }
// newCacheConfig builds the image cache's configuration from cfg.
// cache_max_bytes: 0 disables the disk cache entirely; any other value
// is the eviction limit in bytes; when it is omitted, the cache works
// out the default limit itself.
func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig {
return imgcache.CacheConfig{
StateDir: cfg.StateDir,
NegativeTTL: imgcache.DefaultNegativeTTL,
MaxBytes: cfg.CacheMaxBytes,
UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit,
DisableDiskCache: cfg.CacheMaxBytesExplicit && cfg.CacheMaxBytes == 0,
Logger: log,
}
}
// initImageService initializes the image cache and service.
func (s *Handlers) initImageService() error {
cache, err := imgcache.NewCache(s.db.DB(), newCacheConfig(s.config, s.log))
if err != nil {
return err
}
s.imgCache = cache
// Background eviction: startup reconciliation, then periodic and
// write-pressure passes. No-op when the disk cache is disabled.
cache.StartEviction(imgcache.DefaultEvictionInterval)
// Writes the counts requests could not write by their deadline
cache.StartPendingCountWrites()
// Create the fetcher config
fetcherCfg := httpfetcher.DefaultConfig()
fetcherCfg.AllowHTTP = s.config.AllowHTTP
fetcherCfg.Timeout = s.config.UpstreamFetchTimeout
fetcherCfg.MaxResponseSize = s.config.UpstreamMaxResponseSize
if s.config.UpstreamConnectionsPerHost > 0 {
fetcherCfg.MaxConnectionsPerHost = s.config.UpstreamConnectionsPerHost
}
fetcherCfg.MaxConnections = s.config.UpstreamConnections
fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
// Create the service. With no fetcher provided, it builds its own from
// fetcherCfg.
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache,
FetcherConfig: fetcherCfg,
Fetcher: s.fetcher,
SigningKey: s.config.SigningKey,
Allowlist: s.config.AllowlistHosts,
MaxConcurrentProcessing: s.config.MaxConcurrentProcessing,
Logger: s.log,
})
if err != nil {
return err
}
s.imgSvc = svc
s.log.Info("image service initialized")
// Initialize session manager (signing key is validated at config load
// time). Session cookies are always Secure/HttpOnly/SameSite=Strict.
sessMgr, err := session.NewManager(s.config.SigningKey)
if err != nil {
return err
}
s.sessMgr = sessMgr
// Initialize encrypted URL generator
encGen, err := encurl.NewGenerator(s.config.SigningKey)
if err != nil {
return err
}
s.encGen = encGen
s.log.Info("session manager and URL generator initialized")
return nil
}
func (s *Handlers) respondJSON(w http.ResponseWriter, data any, status int) { func (s *Handlers) respondJSON(w http.ResponseWriter, data any, status int) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status) w.WriteHeader(status)
+12 -21
View File
@@ -18,20 +18,15 @@ import (
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/database" "sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/signature"
) )
// testFixtures contains test data. // testFixtures contains test data.
type testFixtures struct { type testFixtures struct {
handler *Handlers handler *Handlers
service *imgcache.Service
goodHost string goodHost string
} }
@@ -50,37 +45,32 @@ func setupTestHandler(t *testing.T) *testFixtures {
tmpDir := t.TempDir() tmpDir := t.TempDir()
db := setupTestDB(t) db := setupTestDB(t)
cache, err := imgcache.NewCache(db, imgcache.Config{ cache, err := imgcache.NewCache(db, imgcache.CacheConfig{
StateDir: tmpDir, StateDir: tmpDir,
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30, })
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("failed to create cache: %v", err) t.Fatalf("failed to create cache: %v", err)
} }
svc, err := imageproxy.New(&imageproxy.Config{ svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache, Cache: cache,
Fetcher: newMockFetcher(mockFS), Fetcher: newMockFetcher(mockFS),
Processor: imageprocessor.New(imageprocessor.Params{ SigningKey: "test-signing-key-must-be-32-chars",
MaxInputBytes: 50 << 20, Allowlist: []string{goodHost},
MaxConcurrentProcessing: 2, })
}),
MaxResponseSize: 50 << 20,
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("failed to create proxy: %v", err) t.Fatalf("failed to create service: %v", err)
} }
h := &Handlers{ h := &Handlers{
imgSvc: svc, imgSvc: svc,
log: slog.Default(), log: slog.Default(),
signer: signature.New("test-signing-key-must-be-32-chars"),
allowlist: allowlist.New([]string{goodHost}),
} }
return &testFixtures{ return &testFixtures{
handler: h, handler: h,
service: svc,
goodHost: goodHost, goodHost: goodHost,
} }
} }
@@ -131,9 +121,10 @@ func newMockFetcher(fs fs.FS) *mockFetcher {
} }
func (f *mockFetcher) Fetch( func (f *mockFetcher) Fetch(
_ context.Context, source imagerequest.Source, _ context.Context, url string,
) (*httpfetcher.FetchResult, error) { ) (*httpfetcher.FetchResult, error) {
path := source.Host + source.Path // Remove https:// prefix
path := url[8:] // Remove "https://"
data, err := fs.ReadFile(f.fs, path) data, err := fs.ReadFile(f.fs, path)
if err != nil { if err != nil {
+104 -180
View File
@@ -1,31 +1,22 @@
package handlers package handlers
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
"strings"
"time" "time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/magic"
) )
// statusClientClosedRequest is the status nginx calls "client closed
// request". The image routes answer it when the client goes away before its
// image is ready, unless its request is the one making the image. Nobody reads
// it; it is for logs and metrics.
const statusClientClosedRequest = 499
// HandleImage handles the main image proxy route: // HandleImage handles the main image proxy route:
// /v1/image/<host>/<path>/<width>x<height>.<format>, or with no format // /v1/image/<host>/<path>/<width>x<height>.<format>, or with no format
// /v1/image/<host>/<path>/<width>x<height> // /v1/image/<host>/<path>/<width>x<height>
@@ -35,17 +26,17 @@ func (s *Handlers) HandleImage() http.HandlerFunc {
return return
} }
req, sig, expires, ok := s.parseImageRequest(w, r) req, ok := s.parseImageRequest(w, r)
if !ok { if !ok {
return return
} }
// Validate signature if required // Validate signature if required
err := s.validateSignature(req, sig, expires) err := s.imgSvc.ValidateRequest(req)
if err != nil { if err != nil {
s.log.Warn("signature validation failed", s.log.Warn("signature validation failed",
"host", req.Source.Host, "host", req.SourceHost,
"path", req.Source.Path, "path", req.SourcePath,
"error", err, "error", err,
) )
s.respondError(w, "unauthorized", http.StatusUnauthorized) s.respondError(w, "unauthorized", http.StatusUnauthorized)
@@ -53,7 +44,27 @@ func (s *Handlers) HandleImage() http.HandlerFunc {
return return
} }
s.serveImage(w, r, req, expires) // The signature covers the format auto, not the format chosen
if !s.chooseAutoFormat(w, r, req) {
return
}
// Get cache key for logging
cacheKey := imgcache.CacheKey(req)
// Get the image (from cache or fetch/process)
startTime := time.Now()
resp, err := s.imgSvc.Get(r.Context(), req)
if err != nil {
s.respondImageError(w, req, err)
return
}
defer func() { _ = resp.Content.Close() }()
s.writeImageResponse(w, r, req, resp, cacheKey, startTime)
} }
} }
@@ -69,28 +80,30 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc {
} }
} }
// parseImageRequest parses the image URL's path and query parameters into // parseImageRequest parses the wildcard path and query parameters into
// an image request, and returns it with the URL's signature and expiry. On // an ImageRequest. On invalid input it writes an error response and
// invalid input it writes an error response and returns false. // returns false.
func (s *Handlers) parseImageRequest( func (s *Handlers) parseImageRequest(
w http.ResponseWriter, r *http.Request, w http.ResponseWriter, r *http.Request,
) (*imagerequest.Request, string, time.Time, bool) { ) (*imgcache.ImageRequest, bool) {
// The path is parsed decoded once, as r.URL.Path holds it, and not from // Get the wildcard path from chi
// chi's wildcard: chi routes on the path as the client wrote it whenever pathParam := chi.URLParam(r, "*")
// that is not Go's own encoding of it, so its wildcard would still hold
// the %3F before a source query when, say, a "/" in the query is written // Parse the URL path
// as %2F parsed, err := imgcache.ParseImagePath(pathParam)
req, err := imagerequest.ParseImageURL(r.URL.Path)
if err != nil { if err != nil {
s.log.Warn("failed to parse image URL", s.log.Warn("failed to parse image URL",
"path", r.URL.Path, "path", pathParam,
"error", err, "error", err,
) )
s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest) s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
// Convert to ImageRequest
req := parsed.ToImageRequest()
// Parse signature params from query string. r.URL.Query() would silently // Parse signature params from query string. r.URL.Query() would silently
// drop a pair it cannot decode, such as q=80%, so that q would be served // drop a pair it cannot decode, such as q=80%, so that q would be served
// at 85; a query string that cannot be decoded is refused instead. A // at 85; a query string that cannot be decoded is refused instead. A
@@ -101,7 +114,7 @@ func (s *Handlers) parseImageRequest(
s.respondError(w, fmt.Sprintf("invalid query string %q: %v", s.respondError(w, fmt.Sprintf("invalid query string %q: %v",
r.URL.RawQuery, err), http.StatusBadRequest) r.URL.RawQuery, err), http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
for name, values := range query { for name, values := range query {
@@ -109,15 +122,17 @@ func (s *Handlers) parseImageRequest(
s.respondError(w, fmt.Sprintf("invalid %s: given more than once", s.respondError(w, fmt.Sprintf("invalid %s: given more than once",
name), http.StatusBadRequest) name), http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
} }
expires, err := parseExpires(query) req.Signature = query.Get("sig")
req.Expires, err = parseExpires(query)
if err != nil { if err != nil {
s.respondError(w, err.Error(), http.StatusBadRequest) s.respondError(w, err.Error(), http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
// Parse optional quality and fit params. Only a q missing from the URL is // Parse optional quality and fit params. Only a q missing from the URL is
@@ -129,76 +144,51 @@ func (s *Handlers) parseImageRequest(
s.respondError(w, `invalid q: not a number, got ""`, s.respondError(w, `invalid q: not a number, got ""`,
http.StatusBadRequest) http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
req.Transform.Quality, err = parseFormInt(query, "q", req.Quality, err = parseFormInt(query, "q",
encurl.DefaultQuality, minQuality, maxQuality) encurl.DefaultQuality, minQuality, maxQuality)
if err != nil { if err != nil {
s.respondError(w, fmt.Sprintf("%v, got %q", err, qStr), s.respondError(w, fmt.Sprintf("%v, got %q", err, qStr),
http.StatusBadRequest) http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
// Only a fit missing from the URL is cover. A fit in the URL that is not a // Only a fit missing from the URL is cover. A fit in the URL that is not a
// fit mode is refused by the fit-mode check below; that check would take an // fit mode is refused by the fit-mode check below; that check would take an
// empty fit as missing, so an empty one is refused here. // empty fit as missing, so an empty one is refused here.
req.Transform.FitMode = imagerequest.Fit(query.Get("fit")) req.FitMode = imgcache.FitMode(query.Get("fit"))
if query.Has("fit") && req.Transform.FitMode == "" { if query.Has("fit") && req.FitMode == "" {
s.respondError(w, `invalid fit: not a fit mode, got ""`, http.StatusBadRequest) s.respondError(w, `invalid fit: not a fit mode, got ""`, http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
// Default fit mode if not set // Default fit mode if not set
if req.Transform.FitMode == "" { if req.FitMode == "" {
req.Transform.FitMode = imagerequest.FitCover req.FitMode = imgcache.FitCover
} }
// Enforce dimension and fit-mode bounds, shared with the encrypted-URL // Enforce dimension and fit-mode bounds, shared with the encrypted-URL
// route. Dimensions are already bounded by the path parser above; this // route. Dimensions are already bounded by the path parser above; this
// also rejects an unrecognized fit mode with 400 instead of letting it // also rejects an unrecognized fit mode with 400 instead of letting it
// reach the processor as a 500. // reach the processor as a 500.
err = imagerequest.ValidateImageRequest(req) err = imgcache.ValidateImageRequest(req)
if err != nil { if err != nil {
s.respondError(w, "invalid image request: "+err.Error(), s.respondError(w, "invalid image request: "+err.Error(),
http.StatusBadRequest) http.StatusBadRequest)
return nil, "", time.Time{}, false return nil, false
} }
return req, query.Get("sig"), expires, true return req, true
}
// validateSignature returns nil when the source host of req is on
// allowlist_hosts and names no port. Otherwise it verifies sig, the URL's
// signature of req, and expires, the URL's expiry.
func (s *Handlers) validateSignature(
req *imagerequest.Request, sig string, expires time.Time,
) error {
// An allowlist entry names no port, so a host followed by ":", with or
// without a port, needs a signature. Go's URL type reads the port as the
// fetcher does: in an IPv6 address written without brackets, such as
// 2001:db8::1:8443, it is what follows the last ":". This is checked here
// and not in allowlist.IsAllowed, which also matches referer_blocklist,
// where a Referer must be blocked whatever its port. Port and IsAllowed
// read only the host of a URL.
sourceURL := &url.URL{Host: req.Source.Host}
hostHasPort := sourceURL.Port() != "" ||
strings.HasSuffix(sourceURL.Host, ":")
if !hostHasPort && s.allowlist.IsAllowed(sourceURL) {
return nil
}
// Signature required for non-allowed hosts
return s.signer.Verify(req, sig, expires)
} }
// parseExpires reads the exp query parameter, a Unix time in seconds. An exp // parseExpires reads the exp query parameter, a Unix time in seconds. An exp
// missing from the URL gives the zero time, which the signature check refuses // missing from the URL gives the zero time, which the signature check takes
// as a missing exp. An exp in the URL that is not a whole number, an empty // as no expiration. An exp in the URL that is not a whole number, an empty
// one included, is an error naming exp and the value. // one included, is an error naming exp and the value.
func parseExpires(query url.Values) (time.Time, error) { func parseExpires(query url.Values) (time.Time, error) {
if !query.Has("exp") { if !query.Has("exp") {
@@ -226,73 +216,54 @@ func (s *Handlers) HandleTooManyRequests() http.HandlerFunc {
} }
} }
// respondImageError answers err, the image service's error for req, on both // respondImageError maps image retrieval errors to HTTP responses.
// image routes. It logs a 500 at error level, as pixa's own failure. It logs
// the other answers at warn level, since the client, the upstream or a full
// limit caused them. The 415 for a source refused for its first bytes is not
// logged here, because the image service logs it.
func (s *Handlers) respondImageError( func (s *Handlers) respondImageError(
w http.ResponseWriter, req *imagerequest.Request, err error, w http.ResponseWriter, req *imgcache.ImageRequest, err error,
) { ) {
// An upstream body whose first bytes are not those of the format its s.log.Error("failed to get image",
// Content-Type names is the upstream's error, not pixa's: the image "host", req.SourceHost,
// service has logged it as a refused source. "path", req.SourcePath,
if errors.Is(err, magic.ErrMagicByteMismatch) || "error", err,
errors.Is(err, magic.ErrUnknownFormat) || )
errors.Is(err, magic.ErrNotEnoughData) {
// Check for specific error types
if errors.Is(err, httpfetcher.ErrSSRFBlocked) {
s.respondError(w, "forbidden", http.StatusForbidden)
return
}
if errors.Is(err, httpfetcher.ErrUpstreamError) {
s.respondError(w, "upstream error", http.StatusBadGateway)
return
}
if errors.Is(err, httpfetcher.ErrTooManyConnections) ||
errors.Is(err, imageprocessor.ErrTooManyImages) {
s.respondError(w, "server busy, try again later",
http.StatusServiceUnavailable)
return
}
if errors.Is(err, httpfetcher.ErrInvalidContentType) ||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat) {
s.respondError(w, "unsupported source format", s.respondError(w, "unsupported source format",
http.StatusUnsupportedMediaType) http.StatusUnsupportedMediaType)
return return
} }
var ( if errors.Is(err, imageprocessor.ErrOutputTooLarge) {
status int s.respondError(w, fmt.Sprintf("size too large for the source: the side "+
message string
)
switch {
case errors.Is(err, httpfetcher.ErrSSRFBlocked):
status, message = http.StatusForbidden, "forbidden"
case errors.Is(err, httpfetcher.ErrUpstreamError):
status, message = http.StatusBadGateway, "upstream error"
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
status, message = http.StatusGatewayTimeout, "upstream timeout"
case errors.Is(err, httpfetcher.ErrTooManyConnections),
errors.Is(err, imageprocessor.ErrTooManyImages),
errors.Is(err, imageproxy.ErrProcessingTimedOut):
status = http.StatusServiceUnavailable
message = "server busy, try again later"
case errors.Is(err, httpfetcher.ErrInvalidContentType),
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
status = http.StatusUnsupportedMediaType
message = "unsupported source format"
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
status = http.StatusBadRequest
message = fmt.Sprintf("size too large for the source: the side "+
"computed from its proportions is above %d", "computed from its proportions is above %d",
imageprocessor.MaxOutputDimension) imageprocessor.MaxOutputDimension), http.StatusBadRequest)
case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge):
status = http.StatusBadRequest
message = fmt.Sprintf("size too large for AVIF: the output "+
"would be above %d pixels", imageprocessor.MaxAVIFOutputPixels)
default:
s.log.Error("failed to get image",
"host", req.Source.Host,
"path", req.Source.Path,
"error", err,
)
s.respondError(w, "internal error", http.StatusInternalServerError)
return return
} }
s.log.Warn("failed to get image", s.respondError(w, "internal error", http.StatusInternalServerError)
"host", req.Source.Host,
"path", req.Source.Path,
"error", err,
)
s.respondError(w, message, status)
} }
// cacheControl returns the Cache-Control header for an image served through a // cacheControl returns the Cache-Control header for an image served through a
@@ -349,50 +320,13 @@ func notModified(w http.ResponseWriter, r *http.Request, etag string) bool {
return true return true
} }
// serveImage serves the image that req asks for. Both image routes call it // writeImageResponse writes headers and streams the image content,
// once they have the request and its URL's expiry, which ends the // handling conditional and HEAD requests.
// Cache-Control max-age. It chooses the format for the format auto, answers func (s *Handlers) writeImageResponse(
// If-None-Match with 304 and HEAD with the headers only. It answers 499 when
// the image service stopped because the client went away, and any other error
// of the image service with respondImageError.
func (s *Handlers) serveImage(
w http.ResponseWriter, r *http.Request, w http.ResponseWriter, r *http.Request,
req *imagerequest.Request, expires time.Time, req *imgcache.ImageRequest, resp *imgcache.ImageResponse,
cacheKey imgcache.VariantKey, startTime time.Time,
) { ) {
// The signature or the token covers the format auto, not the format chosen
format, ok := s.chooseAutoFormat(w, r, req.Transform.Format)
if !ok {
return
}
req.Transform.Format = format
startTime := time.Now()
resp, err := s.imgSvc.Get(r.Context(), req)
// The client went away before its image was ready, which is not pixa's
// failure. A cancellation while the request's context is live is pixa's,
// and respondImageError answers it 500.
if errors.Is(err, context.Canceled) && r.Context().Err() != nil {
s.log.Info("client went away before the image was ready",
"request_id", middleware.GetReqID(r.Context()),
"host", req.Source.Host,
"path", req.Source.Path,
)
w.WriteHeader(statusClientClosedRequest)
return
}
if err != nil {
s.respondImageError(w, req, err)
return
}
defer func() { _ = resp.Content.Close() }()
// Set response headers // Set response headers
w.Header().Set("Content-Type", resp.ContentType) w.Header().Set("Content-Type", resp.ContentType)
@@ -401,7 +335,7 @@ func (s *Handlers) serveImage(
} }
// Cache control headers // Cache control headers
w.Header().Set("Cache-Control", cacheControl(expires)) w.Header().Set("Cache-Control", cacheControl(req.Expires))
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus)) w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
if notModified(w, r, resp.ETag) { if notModified(w, r, resp.ETag) {
@@ -418,31 +352,21 @@ func (s *Handlers) serveImage(
// Stream the response // Stream the response
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
// A failed write is most often a client that has gone away, not pixa's
// failure, so it is logged at warn level
servedBytes, err := io.Copy(w, resp.Content) servedBytes, err := io.Copy(w, resp.Content)
if err != nil { if err != nil {
s.log.Warn("failed to write response", s.log.Error("failed to write response",
"request_id", middleware.GetReqID(r.Context()),
"host", req.Source.Host,
"path", req.Source.Path,
"served_bytes", servedBytes,
"error", err, "error", err,
) )
return
} }
// Log cache status and timing after serving // Log cache status and timing after serving
duration := time.Since(startTime) duration := time.Since(startTime)
s.log.Info("image served", s.log.Info("image served",
"request_id", middleware.GetReqID(r.Context()), "request_id", middleware.GetReqID(r.Context()),
"cache_key", imagerequest.CacheKey(req), "cache_key", cacheKey,
"host", req.Source.Host,
"path", req.Source.Path,
"cache_status", resp.CacheStatus, "cache_status", resp.CacheStatus,
"duration_ms", duration.Milliseconds(), "duration_ms", duration.Milliseconds(),
"format", req.Transform.Format, "format", req.Format,
"served_bytes", servedBytes, "served_bytes", servedBytes,
"fetched_bytes", resp.FetchedBytes, "fetched_bytes", resp.FetchedBytes,
) )
@@ -12,16 +12,9 @@ import (
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/signature"
) )
// photoPath is the path of the JPEG that newSignedHostServer serves. // photoPath is the path of the JPEG that newSignedHostServer serves.
@@ -36,46 +29,37 @@ func newSignedHostServer(
) (*Handlers, http.Handler) { ) (*Handlers, http.Handler) {
t.Helper() t.Helper()
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(), StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30, })
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err) t.Fatalf("imgcache.NewCache() error = %v", err)
} }
jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}) jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255})
svc, err := imageproxy.New(&imageproxy.Config{ svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache, Cache: cache,
Fetcher: newMockFetcher(fstest.MapFS{ Fetcher: newMockFetcher(fstest.MapFS{
signedHost + photoPath: &fstest.MapFile{Data: jpegData}, signedHost + photoPath: &fstest.MapFile{Data: jpegData},
}), }),
Processor: imageprocessor.New(imageprocessor.Params{ SigningKey: testSigningKey,
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
MaxResponseSize: 50 << 20,
Logger: log, Logger: log,
}, prometheus.NewRegistry()) })
if err != nil { if err != nil {
t.Fatalf("imageproxy.New() error = %v", err) t.Fatalf("imgcache.NewService() error = %v", err)
} }
keys, err := seal.DeriveKeys(testSigningKey) encGen, err := encurl.NewGenerator(testSigningKey)
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("encurl.NewGenerator() error = %v", err)
} }
encGen := encurl.NewGenerator(keys.EncryptedURLKey)
h := &Handlers{ h := &Handlers{
log: log, log: log,
imgSvc: svc, imgSvc: svc,
encGen: encGen, encGen: encGen,
signer: signature.New(testSigningKey),
allowlist: allowlist.New(nil),
} }
r := chi.NewRouter() r := chi.NewRouter()
@@ -121,18 +105,17 @@ func getMaxAge(t *testing.T, srv http.Handler, target string) int {
func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) { func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) {
t.Parallel() t.Parallel()
_, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
signedURL := generateSignedURL(testSigningKey, &imagerequest.Request{ signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Size: imgcache.Size{Width: 50, Height: 50},
}, Format: imgcache.FormatJPEG,
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
},
}, time.Minute) }, time.Minute)
if err != nil {
t.Fatalf("GenerateSignedURL() error = %v", err)
}
maxAge := getMaxAge(t, srv, signedURL) maxAge := getMaxAge(t, srv, signedURL)
if maxAge < 50 || maxAge > 60 { if maxAge < 50 || maxAge > 60 {
@@ -185,13 +168,13 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
expires time.Time expiresAt int64
wantAtLeast int wantAtLeast int
wantAtMost int wantAtMost int
}{ }{
{"60 second TTL", time.Now().Add(time.Minute), 50, 60}, {"60 second TTL", time.Now().Add(time.Minute).Unix(), 50, 60},
{"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour), 31536000, 31536000}, {"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour).Unix(), 31536000, 31536000},
{"no TTL", time.Time{}, 31536000, 31536000}, {"no TTL", 0, 31536000, 31536000},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -200,16 +183,14 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
token, err := h.encGen.Generate(&imagerequest.Request{ token, err := h.encGen.Generate(&encurl.Payload{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{ Format: imgcache.FormatJPEG,
Size: imagerequest.Size{Width: 50, Height: 50}, ExpiresAt: tt.expiresAt,
Format: imagerequest.FormatJPEG, })
},
}, tt.expires)
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
+21 -342
View File
@@ -1,29 +1,19 @@
package handlers package handlers
import ( import (
"bytes"
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"image/color" "image/color"
"io"
"log/slog" "log/slog"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"slices"
"strings" "strings"
"testing" "testing"
"testing/fstest" "testing/fstest"
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/signature" "sneak.berlin/go/pixa/internal/signature"
) )
@@ -33,38 +23,29 @@ import (
const allowlistedHost = "allowed.example.com" const allowlistedHost = "allowed.example.com"
// newImageRoute returns the image route of a Handlers whose service fetches // newImageRoute returns the image route of a Handlers whose service fetches
// with fetcher, and which checks signatures with testSigningKey. // with fetcher and checks signatures with testSigningKey.
func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler { func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler {
t.Helper() t.Helper()
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(), StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30, })
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("failed to create cache: %v", err) t.Fatalf("failed to create cache: %v", err)
} }
svc, err := imageproxy.New(&imageproxy.Config{ svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache, Cache: cache,
Fetcher: fetcher, Fetcher: fetcher,
Processor: imageprocessor.New(imageprocessor.Params{ SigningKey: testSigningKey,
MaxInputBytes: 50 << 20, Allowlist: []string{allowlistedHost},
MaxConcurrentProcessing: 2, })
}),
MaxResponseSize: 50 << 20,
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("failed to create proxy: %v", err) t.Fatalf("failed to create service: %v", err)
} }
h := &Handlers{ h := &Handlers{imgSvc: svc, log: slog.New(slog.DiscardHandler)}
imgSvc: svc,
log: slog.New(slog.DiscardHandler),
signer: signature.New(testSigningKey),
allowlist: allowlist.New([]string{allowlistedHost}),
}
r := chi.NewRouter() r := chi.NewRouter()
r.Get("/v1/image/*", h.HandleImage()) r.Get("/v1/image/*", h.HandleImage())
@@ -73,7 +54,7 @@ func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler {
} }
// newPhotoFetcher returns a mock fetcher that serves a JPEG at photoPath on // newPhotoFetcher returns a mock fetcher that serves a JPEG at photoPath on
// each of hosts, and answers any other source with an upstream error. // each of hosts, and answers any other URL with an upstream error.
func newPhotoFetcher(t *testing.T, hosts ...string) *httpfetcher.MockFetcher { func newPhotoFetcher(t *testing.T, hosts ...string) *httpfetcher.MockFetcher {
t.Helper() t.Helper()
@@ -100,27 +81,19 @@ func photoURLWithSig(host, sig string, expires time.Time) string {
return fmt.Sprintf("%s?sig=%s&exp=%d", photoURL(host), sig, expires.Unix()) return fmt.Sprintf("%s?sig=%s&exp=%d", photoURL(host), sig, expires.Unix())
} }
// photoRequest returns the image request of photoURL(host) at the default
// quality and fit.
func photoRequest(host string) *imagerequest.Request {
return &imagerequest.Request{
Source: imagerequest.Source{
Host: host,
Path: photoPath,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
}
// photoSignature returns the signature of photoURL(host) at the default // photoSignature returns the signature of photoURL(host) at the default
// quality and fit, made with key and expiring at expires. // quality and fit, made with key and expiring at expires.
func photoSignature(key, host string, expires time.Time) string { func photoSignature(key, host string, expires time.Time) string {
return signature.New(key).Sign(photoRequest(host), expires) return signature.New(key).Sign(&signature.Request{
SourceHost: host,
SourcePath: photoPath,
Width: 50,
Height: 50,
Format: string(imgcache.FormatJPEG),
Quality: 85,
FitMode: string(imgcache.FitCover),
Expires: expires,
})
} }
// sendGet sends a GET for target to route and returns the response. // sendGet sends a GET for target to route and returns the response.
@@ -191,12 +164,7 @@ func TestHandleImage_ErrorAnswers(t *testing.T) {
photos := newPhotoFetcher(t, photos := newPhotoFetcher(t,
signedHost, parentHost, siblingHost, subdomainHost, appendedHost) signedHost, parentHost, siblingHost, subdomainHost, appendedHost)
// The real fetcher refuses localhost before any lookup or connection. // The real fetcher refuses localhost before any lookup or connection.
realFetcher := httpfetcher.New(&httpfetcher.Config{ realFetcher := httpfetcher.New(httpfetcher.DefaultConfig())
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
})
exp := time.Now().Add(time.Hour) exp := time.Now().Add(time.Hour)
expired := time.Now().Add(-time.Hour) expired := time.Now().Add(-time.Hour)
@@ -296,292 +264,3 @@ func TestHandleImage_AllowlistOrSignature(t *testing.T) {
}) })
} }
} }
// TestHandleImage_InvalidHost_Returns400 sends image URLs whose host no URL
// host can be, and checks that each is refused with 400 naming the host as
// decoded: allowlistedHost with a character after it that a URL host cannot
// hold, percent-encoded, an IPv6 address without brackets, and brackets
// around something that is not an IPv6 address. Later steps put the host
// unparsed into the allowlist check, the cache key and the upstream URL, so
// such a host is refused first.
func TestHandleImage_InvalidHost_Returns400(t *testing.T) {
t.Parallel()
photos := newPhotoFetcher(t, allowlistedHost)
tests := []struct {
name string
writtenHost string
decodedHost string
}{
{"? written as %3F", allowlistedHost + "%3Fx=1", allowlistedHost + "?x=1"},
{"# written as %23", allowlistedHost + "%23x", allowlistedHost + "#x"},
{"% written as %25", allowlistedHost + "%253F", allowlistedHost + "%3F"},
{`\ written as %5C`, allowlistedHost + "%5Cx", allowlistedHost + `\x`},
{"space written as %20", allowlistedHost + "%20", allowlistedHost + " "},
{"IPv6 address without brackets", "2001:db8::1", "2001:db8::1"},
{"IPv4 address in brackets", "[203.0.113.5]", "[203.0.113.5]"},
{"no address in brackets", "[abc]", "[abc]"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
rec := sendGet(t, newImageRoute(t, photos), photoURL(tt.writtenHost))
checkErrorBody(t, rec, http.StatusBadRequest,
fmt.Sprintf("invalid image URL: invalid source host: %q",
tt.decodedHost))
})
}
}
// TestHandleImage_HostForms_Served checks that the image route serves, with
// a valid signature, the image at photoPath on a host name, an IPv4 address,
// an IPv6 address in brackets, and a host followed by ":" with or without a
// port.
func TestHandleImage_HostForms_Served(t *testing.T) {
t.Parallel()
hosts := []string{
allowlistedHost,
"img_1.example.com",
"203.0.113.5",
"[2001:db8::1]",
allowlistedHost + ":8443",
"[2001:db8::1]:8443",
allowlistedHost + ":",
}
photos := newPhotoFetcher(t, hosts...)
exp := time.Now().Add(time.Hour)
for _, host := range hosts {
t.Run(host, func(t *testing.T) {
t.Parallel()
sig := photoSignature(testSigningKey, host, exp)
rec := sendGet(t, newImageRoute(t, photos),
photoURLWithSig(host, sig, exp))
if rec.Code != http.StatusOK {
t.Errorf("status = %d, want %d: %s",
rec.Code, http.StatusOK, rec.Body)
}
})
}
}
// TestHandleImage_HashInSourcePathOrQuery_Returns400 sends image URLs whose
// source path or source query holds a "#", written as %23, and checks that
// each is refused with 400, the one signed for that source path included.
// A "#" starts a URL's fragment, so neither the source path nor the source
// query may hold one.
func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) {
t.Parallel()
photos := newPhotoFetcher(t, allowlistedHost, signedHost)
exp := time.Now().Add(time.Hour)
sig := signature.New(testSigningKey).Sign(&imagerequest.Request{
Source: imagerequest.Source{
Host: signedHost,
Path: photoPath + "#x",
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}, exp)
tests := []struct {
name string
target string
}{
{"# in the source path",
"/v1/image/" + allowlistedHost + photoPath + "%23x/50x50.jpeg"},
{"# in the source query",
"/v1/image/" + allowlistedHost + photoPath + "%3Fv=1%23x/50x50.jpeg"},
{"# in the source path, signed",
fmt.Sprintf("/v1/image/%s%s%%23x/50x50.jpeg?sig=%s&exp=%d",
signedHost, photoPath, sig, exp.Unix())},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
rec := sendGet(t, newImageRoute(t, photos), tt.target)
checkErrorBody(t, rec, http.StatusBadRequest,
`invalid image URL: invalid image path: `+
`the source path or query holds a "#"`)
})
}
}
// failingFetcher answers every fetch with its error.
type failingFetcher struct {
err error
}
// Fetch returns f's error.
func (f failingFetcher) Fetch(
context.Context, imagerequest.Source,
) (*httpfetcher.FetchResult, error) {
return nil, f.err
}
// photoURLsByRoute returns, for each image route, the function that makes the
// URL of the JPEG at photoPath on signedHost on that route.
func photoURLsByRoute() map[string]func(*testing.T, *Handlers) string {
return map[string]func(*testing.T, *Handlers) string{
"/v1/image/": signedPhotoURL,
"/v1/e/": encPhotoURL,
}
}
// TestImageRoutes_SameErrorAnswers checks that both image routes answer each
// upstream fetch error below with the same status. They log it as "failed to
// get image" at the same level: error for the 500 and warn for the others.
// Each request gets routes of its own, so no request finds another's failed
// fetch in the negative cache.
func TestImageRoutes_SameErrorAnswers(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err error
wantStatus int
wantLevel slog.Level
}{
{"blocked upstream address", httpfetcher.ErrSSRFBlocked,
http.StatusForbidden, slog.LevelWarn},
{"upstream error status",
fmt.Errorf("%w: status 404", httpfetcher.ErrUpstreamError),
http.StatusBadGateway, slog.LevelWarn},
{"no response headers within the fetch timeout",
httpfetcher.ErrUpstreamTimeout, http.StatusGatewayTimeout,
slog.LevelWarn},
{"no free upstream connection", httpfetcher.ErrTooManyConnections,
http.StatusServiceUnavailable, slog.LevelWarn},
{"Content-Type not a source format",
fmt.Errorf("%w: image/svg+xml", httpfetcher.ErrInvalidContentType),
http.StatusUnsupportedMediaType, slog.LevelWarn},
{"any other failure", io.ErrUnexpectedEOF,
http.StatusInternalServerError, slog.LevelError},
}
for _, tt := range tests {
for route, imageURL := range photoURLsByRoute() {
t.Run(route+" "+tt.name, func(t *testing.T) {
t.Parallel()
var logs bytes.Buffer
h, routes := newRoutesFetchingThrough(t,
failingFetcher{err: tt.err},
slog.New(slog.NewJSONHandler(&logs, nil)))
rec := sendGet(t, routes, imageURL(t, h))
if rec.Code != tt.wantStatus {
t.Errorf("status = %d, want %d: %s",
rec.Code, tt.wantStatus, rec.Body)
}
levels := loggedLevels(t, &logs)
if !slices.Contains(levels[tt.wantLevel.String()],
"failed to get image") {
t.Errorf(`no "failed to get image" line logged at level %s`,
tt.wantLevel)
}
if tt.wantLevel != slog.LevelError &&
len(levels[slog.LevelError.String()]) != 0 {
t.Errorf("logged at error level: %q",
levels[slog.LevelError.String()])
}
})
}
}
}
// failingResponseWriter is a response writer whose Write always fails, as when
// the client has gone away.
type failingResponseWriter struct {
*httptest.ResponseRecorder
}
// Write writes nothing and returns io.ErrClosedPipe.
func (failingResponseWriter) Write([]byte) (int, error) {
return 0, io.ErrClosedPipe
}
// TestImageRoutes_FailedWriteLoggedOnce checks that both image routes log a
// failed write of the image once, at warn level, with the request's ID, the
// source host and path, the bytes served and the error. Neither may also log
// the image as served.
func TestImageRoutes_FailedWriteLoggedOnce(t *testing.T) {
t.Parallel()
const requestID = "test-request-id"
for route, imageURL := range photoURLsByRoute() {
t.Run(route, func(t *testing.T) {
t.Parallel()
var logs bytes.Buffer
h, srv := newSignedHostServer(t,
slog.New(slog.NewJSONHandler(&logs, nil)))
ctx := context.WithValue(t.Context(),
middleware.RequestIDKey, requestID)
srv.ServeHTTP(failingResponseWriter{httptest.NewRecorder()},
httptest.NewRequestWithContext(
ctx, http.MethodGet, imageURL(t, h), nil))
t.Logf("logged:\n%s", logs.String())
var failedWrites []map[string]any
dec := json.NewDecoder(&logs)
for dec.More() {
var line map[string]any
err := dec.Decode(&line)
if err != nil {
t.Fatalf("decoding log line: %v", err)
}
switch line["msg"] {
case "image served":
t.Error(`"image served" logged after a failed write`)
case "failed to write response":
failedWrites = append(failedWrites, line)
}
}
if len(failedWrites) != 1 {
t.Fatalf(`"failed to write response" logged %d times, want 1`,
len(failedWrites))
}
want := map[string]any{
"level": slog.LevelWarn.String(),
"request_id": requestID,
"host": signedHost,
"path": photoPath,
"served_bytes": float64(0),
"error": io.ErrClosedPipe.Error(),
}
for key, wantValue := range want {
if got := failedWrites[0][key]; got != wantValue {
t.Errorf("%s = %v, want %v", key, got, wantValue)
}
}
})
}
}
@@ -5,12 +5,11 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url"
"testing" "testing"
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/signature" "sneak.berlin/go/pixa/internal/signature"
) )
@@ -37,47 +36,6 @@ func getImage(t *testing.T, fix *testFixtures, target string) int {
return rec.Code return rec.Code
} }
// generateSignedURL returns the /v1/image/ URL of req, signed with key and
// expiring after ttl. It signs a copy of req, so req itself is not changed.
// An unset quality or fit in the copy is set to 85 or cover, the values the
// route uses when a URL has no q or fit. The URL carries q and fit next to sig
// and exp, so the route verifies it for the quality and fit it was signed
// with.
func generateSignedURL(
key string, req *imagerequest.Request, ttl time.Duration,
) string {
signed := *req
if signed.Transform.Quality == 0 {
signed.Transform.Quality = 85
}
if signed.Transform.FitMode == "" {
signed.Transform.FitMode = imagerequest.FitCover
}
expires := time.Now().Add(ttl)
sig := signature.New(key).Sign(&signed, expires)
// A source query follows the source path after a "?". The image route
// reads the path decoded once, so the whole path is written in Go's own
// percent-encoding of a URL path, which writes "?" as %3F and "%" as %25,
// and leaves "/", ";", ",", "=" and "&" as they are.
sourcePathAndQuery := signed.Source.Path
if signed.Source.Query != "" {
sourcePathAndQuery += "?" + signed.Source.Query
}
unescapedPath := fmt.Sprintf("/v1/image/%s%s/%dx%d.%s",
signed.Source.Host, sourcePathAndQuery,
signed.Transform.Size.Width, signed.Transform.Size.Height,
signed.Transform.Format)
return fmt.Sprintf("%s?sig=%s&exp=%d&q=%d&fit=%s",
(&url.URL{Path: unescapedPath}).EscapedPath(), sig, expires.Unix(),
signed.Transform.Quality, signed.Transform.FitMode)
}
// TestHandleImage_SignatureCoversQualityAndFit signs a URL for quality 85 // TestHandleImage_SignatureCoversQualityAndFit signs a URL for quality 85
// and fit cover, the values the route uses when a URL has no q or fit, and // and fit cover, the values the route uses when a URL has no q or fit, and
// sends that signature with each q and fit below. // sends that signature with each q and fit below.
@@ -86,18 +44,16 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) {
expires := time.Now().Add(time.Hour) expires := time.Now().Add(time.Hour)
signer := signature.New("test-signing-key-must-be-32-chars") signer := signature.New("test-signing-key-must-be-32-chars")
sig := signer.Sign(&imagerequest.Request{ sig := signer.Sign(&signature.Request{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: "/images/photo.jpg",
Path: "/images/photo.jpg", Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{ Format: string(imgcache.FormatJPEG),
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: string(imgcache.FitCover),
}, Expires: expires,
}, expires) })
signedURL := fmt.Sprintf("/v1/image/%s/images/photo.jpg/50x50.jpeg?sig=%s&exp=%d", signedURL := fmt.Sprintf("/v1/image/%s/images/photo.jpg/50x50.jpeg?sig=%s&exp=%d",
signedHost, sig, expires.Unix()) signedHost, sig, expires.Unix())
@@ -124,17 +80,17 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) {
} }
} }
// TestHandleImage_GeneratedSignedURLVerifies sends URLs built by // TestHandleImage_GeneratedSignedURLVerifies sends URLs built by the
// generateSignedURL to the route. // service's signed-URL generator to the route.
func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) { func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) {
t.Parallel() t.Parallel()
tests := []struct { tests := []struct {
name string name string
quality int quality int
fitMode imagerequest.Fit fitMode imgcache.FitMode
}{ }{
{"quality 40 and fit contain", 40, imagerequest.FitContain}, {"quality 40 and fit contain", 40, imgcache.FitContain},
{"quality and fit unset", 0, ""}, {"quality and fit unset", 0, ""},
} }
@@ -142,21 +98,21 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
t.Parallel() t.Parallel()
signedURL := generateSignedURL("test-signing-key-must-be-32-chars", fix := setupTestHandler(t)
&imagerequest.Request{
Source: imagerequest.Source{ signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{
Host: signedHost, SourceHost: signedHost,
Path: "/images/photo.jpg", SourcePath: "/images/photo.jpg",
}, Size: imgcache.Size{Width: 50, Height: 50},
Transform: imagerequest.Transform{ Format: imgcache.FormatJPEG,
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: tt.quality, Quality: tt.quality,
FitMode: tt.fitMode, FitMode: tt.fitMode,
},
}, time.Hour) }, time.Hour)
if err != nil {
t.Fatalf("GenerateSignedURL() error = %v", err)
}
status := getImage(t, setupTestHandler(t), signedURL) status := getImage(t, fix, signedURL)
if status != http.StatusBadGateway { if status != http.StatusBadGateway {
t.Errorf("status = %d, want %d", status, http.StatusBadGateway) t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
} }
@@ -213,170 +169,3 @@ func TestHandleImage_InvalidExp_Returns400(t *testing.T) {
}) })
} }
} }
// TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies sends URLs built
// by generateSignedURL for source URLs with a query to the route, through a
// chi router: chi routes on the path as the client wrote it whenever that is
// not Go's own encoding of it. A 502 means the signature passed, so the route
// read the source path and query that were signed.
func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) {
t.Parallel()
tests := []struct {
name string
query string
}{
{"slash and colon", "url=https://origin.example.com/cat.jpg"},
{"semicolon", "a=1;b=2"},
{"comma", "crop=0,0,50,50"},
{"equals and ampersand", "a=1&b=2"},
{"percent-escape", "name=my%20cat"},
{"question mark", "next=/a?b=1"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
signedURL := generateSignedURL("test-signing-key-must-be-32-chars",
&imagerequest.Request{
Source: imagerequest.Source{
Host: signedHost,
Path: photoPath,
Query: tt.query,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
},
}, time.Hour)
status := getImage(t, setupTestHandler(t), signedURL)
if status != http.StatusBadGateway {
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
}
})
}
}
// TestHandleImage_SourceQueryEncodedAnyWay signs one source URL with a query
// and writes it into the image URL in ways the README allows: the "?" before
// the query and each "%" in it percent-encoded, and every other character as
// itself or percent-encoded. A 502 means the signature passed.
func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) {
t.Parallel()
expires := time.Now().Add(time.Hour)
signer := signature.New("test-signing-key-must-be-32-chars")
sig := signer.Sign(&imagerequest.Request{
Source: imagerequest.Source{
Host: signedHost,
Path: photoPath,
Query: "url=https://origin.example.com/my%20cat.jpg",
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}, expires)
tests := []struct {
name string
writtenQuery string
}{
{"only ? and % encoded",
"%3Furl=https://origin.example.com/my%2520cat.jpg"},
{"every reserved character encoded",
"%3Furl%3Dhttps%3A%2F%2Forigin.example.com%2Fmy%2520cat.jpg"},
{"lower-case hex digits",
"%3furl=https://origin.example.com/my%2520cat.jpg"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
target := fmt.Sprintf("/v1/image/%s%s%s/50x50.jpeg?sig=%s&exp=%d",
signedHost, photoPath, tt.writtenQuery, sig, expires.Unix())
status := getImage(t, setupTestHandler(t), target)
if status != http.StatusBadGateway {
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
}
})
}
}
// TestHandleImage_SourceURLHoldingPercent25Verifies signs a source path and a
// source query that each hold %25, the percent-escape of a literal "%", and
// sends each to the route written as the README says, with that "%" written as
// %25, and as generateSignedURL writes it. A 502 means the signature passed.
func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) {
t.Parallel()
tests := []struct {
name string
sourcePath string
sourceQuery string
// writtenPathAndQuery is the source path and query as the README says
// to write them into the image URL
writtenPathAndQuery string
}{
{"path", "/sale/50%25off.jpg", "", "/sale/50%2525off.jpg"},
{"query", photoPath, "discount=50%25", photoPath + "%3Fdiscount=50%2525"},
}
for _, tt := range tests {
t.Run(tt.name+" written by the README rule", func(t *testing.T) {
t.Parallel()
expires := time.Now().Add(time.Hour)
signer := signature.New("test-signing-key-must-be-32-chars")
sig := signer.Sign(&imagerequest.Request{
Source: imagerequest.Source{
Host: signedHost,
Path: tt.sourcePath,
Query: tt.sourceQuery,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}, expires)
target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d",
signedHost, tt.writtenPathAndQuery, sig, expires.Unix())
status := getImage(t, setupTestHandler(t), target)
if status != http.StatusBadGateway {
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
}
})
t.Run(tt.name+" written by generateSignedURL", func(t *testing.T) {
t.Parallel()
signedURL := generateSignedURL("test-signing-key-must-be-32-chars",
&imagerequest.Request{
Source: imagerequest.Source{
Host: signedHost,
Path: tt.sourcePath,
Query: tt.sourceQuery,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
},
}, time.Hour)
status := getImage(t, setupTestHandler(t), signedURL)
if status != http.StatusBadGateway {
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
}
})
}
}
+117 -19
View File
@@ -2,13 +2,19 @@ package handlers
import ( import (
"errors" "errors"
"fmt"
"io"
"net/http" "net/http"
"strconv"
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imgcache"
) )
// HandleImageEnc handles requests to /v1/e/{token}/* for encrypted // HandleImageEnc handles requests to /v1/e/{token}/* for encrypted
@@ -20,59 +26,151 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
return return
} }
req, expires, ok := s.parseImageEncRequest(w, r) ctx := r.Context()
if !ok { start := time.Now()
req, ok := s.parseImageEncRequest(w, r)
if !ok || !s.chooseAutoFormat(w, r, req) {
return return
} }
s.serveImage(w, r, req, expires) // Log the request
s.log.Debug("encrypted image request",
"host", req.SourceHost,
"path", req.SourcePath,
"dimensions", fmt.Sprintf("%dx%d", req.Size.Width, req.Size.Height),
"format", req.Format,
)
// Fetch and process the image (no signature validation
// needed - encrypted URL is trusted)
resp, err := s.imgSvc.Get(ctx, req)
if err != nil {
s.handleImageError(w, err)
return
}
defer func() { _ = resp.Content.Close() }()
// Set response headers
w.Header().Set("Content-Type", resp.ContentType)
if resp.ContentLength > 0 {
w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
}
// Cache headers: max-age ends at the URL's expiry
w.Header().Set("Cache-Control", cacheControl(req.Expires))
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
if notModified(w, r, resp.ETag) {
return
}
// A HEAD request gets the headers only
if r.Method == http.MethodHead {
w.WriteHeader(http.StatusOK)
return
}
// Stream the response
written, err := io.Copy(w, resp.Content)
if err != nil {
s.log.Error("failed to write response", "error", err)
return
}
// Log completion
duration := time.Since(start)
s.log.Info("image served",
"request_id", middleware.GetReqID(ctx),
"cache_key", imgcache.CacheKey(req),
"host", req.SourceHost,
"path", req.SourcePath,
"format", req.Format,
"cache_status", resp.CacheStatus,
"served_bytes", written,
"duration_ms", duration.Milliseconds(),
)
} }
} }
// parseImageEncRequest decrypts the token of an encrypted image URL into an // parseImageEncRequest decrypts the token of an encrypted image URL into an
// image request and checks it, and returns it with the token's expiry. On a // ImageRequest and checks it. On a token that is missing, does not decrypt,
// token that is missing, does not decrypt, has expired or asks for something // has expired or asks for something not valid, it writes an error response
// not valid, it writes an error response and returns false. // and returns false.
func (s *Handlers) parseImageEncRequest( func (s *Handlers) parseImageEncRequest(
w http.ResponseWriter, r *http.Request, w http.ResponseWriter, r *http.Request,
) (*imagerequest.Request, time.Time, bool) { ) (*imgcache.ImageRequest, bool) {
// Extract token from URL // Extract token from URL
token := chi.URLParam(r, "token") token := chi.URLParam(r, "token")
if token == "" { if token == "" {
s.respondError(w, "missing token", http.StatusBadRequest) s.respondError(w, "missing token", http.StatusBadRequest)
return nil, time.Time{}, false return nil, false
} }
// Decrypt and validate the payload // Decrypt and validate the payload
req, expires, err := s.encGen.Parse(token) payload, err := s.encGen.Parse(token)
if err != nil { if err != nil {
if errors.Is(err, encurl.ErrExpired) { if errors.Is(err, encurl.ErrExpired) {
s.log.Debug("encrypted URL expired", "error", err) s.log.Debug("encrypted URL expired", "error", err)
s.respondError(w, "URL has expired", http.StatusGone) s.respondError(w, "URL has expired", http.StatusGone)
return nil, time.Time{}, false return nil, false
} }
s.log.Debug("invalid encrypted URL", "error", err) s.log.Debug("failed to decrypt URL", "error", err)
s.respondError(w, "invalid encrypted URL", http.StatusBadRequest) s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
return nil, time.Time{}, false return nil, false
} }
// Convert payload to ImageRequest
req := payload.ToImageRequest()
// Apply the same dimension and fit-mode bounds as the plain image // Apply the same dimension and fit-mode bounds as the plain image
// route: a sealed payload is trusted for its origin, not for staying // route: a sealed payload is trusted for its origin, not for staying
// within limits. This refuses an over-limit size before the source is // within limits, so an over-limit size or unknown fit mode is a 400
// fetched. An unknown fit mode is a 400 here rather than a 500 from the // here rather than an out-of-memory or a 500 from the processor.
// processor. err = imgcache.ValidateImageRequest(req)
err = imagerequest.ValidateImageRequest(req)
if err != nil { if err != nil {
s.log.Debug("encrypted URL failed validation", "error", err) s.log.Debug("encrypted URL failed validation", "error", err)
s.respondError(w, "invalid encrypted URL: "+err.Error(), s.respondError(w, "invalid encrypted URL: "+err.Error(),
http.StatusBadRequest) http.StatusBadRequest)
return nil, time.Time{}, false return nil, false
} }
return req, expires, true return req, true
}
// handleImageError converts image service errors to HTTP responses.
func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, httpfetcher.ErrSSRFBlocked):
s.respondError(w, "forbidden", http.StatusForbidden)
case errors.Is(err, httpfetcher.ErrUpstreamError):
s.respondError(w, "upstream error", http.StatusBadGateway)
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
s.respondError(w, "upstream timeout", http.StatusGatewayTimeout)
case errors.Is(err, httpfetcher.ErrTooManyConnections),
errors.Is(err, imageprocessor.ErrTooManyImages):
s.respondError(w, "server busy, try again later",
http.StatusServiceUnavailable)
case errors.Is(err, httpfetcher.ErrInvalidContentType),
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
s.respondError(w, "unsupported source format",
http.StatusUnsupportedMediaType)
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
s.respondError(w, fmt.Sprintf("size too large for the source: the side "+
"computed from its proportions is above %d",
imageprocessor.MaxOutputDimension), http.StatusBadRequest)
default:
s.log.Error("image request failed", "error", err)
s.respondError(w, "internal error", http.StatusInternalServerError)
}
} }
+16 -26
View File
@@ -6,13 +6,11 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
"time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
) )
// newEncTestServer builds a router serving the encrypted-URL route with a // newEncTestServer builds a router serving the encrypted-URL route with a
@@ -22,13 +20,11 @@ import (
func newEncTestServer(t *testing.T) (*encurl.Generator, http.Handler) { func newEncTestServer(t *testing.T) (*encurl.Generator, http.Handler) {
t.Helper() t.Helper()
keys, err := seal.DeriveKeys(testSigningKey) encGen, err := encurl.NewGenerator(testSigningKey)
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("encurl.NewGenerator() error = %v", err)
} }
encGen := encurl.NewGenerator(keys.EncryptedURLKey)
h := &Handlers{ h := &Handlers{
log: slog.New(slog.DiscardHandler), log: slog.New(slog.DiscardHandler),
encGen: encGen, encGen: encGen,
@@ -58,15 +54,12 @@ func TestHandleImageEnc_OverLimitDimension_Returns400(t *testing.T) {
encGen, srv := newEncTestServer(t) encGen, srv := newEncTestServer(t)
token, err := encGen.Generate(&imagerequest.Request{ token, err := encGen.Generate(&encurl.Payload{
Source: imagerequest.Source{ SourceHost: "cdn.example.com",
Host: "cdn.example.com", SourcePath: "/photo.jpg",
Path: "/photo.jpg", Width: 100000,
}, Height: 100000,
Transform: imagerequest.Transform{ })
Size: imagerequest.Size{Width: 100000, Height: 100000},
},
}, time.Time{})
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
@@ -86,16 +79,13 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) {
encGen, srv := newEncTestServer(t) encGen, srv := newEncTestServer(t)
token, err := encGen.Generate(&imagerequest.Request{ token, err := encGen.Generate(&encurl.Payload{
Source: imagerequest.Source{ SourceHost: "cdn.example.com",
Host: "cdn.example.com", SourcePath: "/photo.jpg",
Path: "/photo.jpg", Width: 800,
}, Height: 600,
Transform: imagerequest.Transform{ FitMode: imgcache.FitMode("bogus"),
Size: imagerequest.Size{Width: 800, Height: 600}, })
FitMode: imagerequest.Fit("bogus"),
},
}, time.Time{})
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
@@ -9,8 +9,6 @@ import (
"time" "time"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/seal"
) )
// requireServedPhoto requires that rec answers 200 with the JPEG at photoPath // requireServedPhoto requires that rec answers 200 with the JPEG at photoPath
@@ -61,34 +59,32 @@ func TestHandleImageEnc_RejectedToken(t *testing.T) {
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
photo := imagerequest.Request{ photo := encurl.Payload{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
},
} }
valid, err := h.encGen.Generate(&photo, time.Time{}) valid, err := h.encGen.Generate(&photo)
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
expired, err := h.encGen.Generate(&photo, time.Now().Add(-time.Minute)) expiredPhoto := photo
expiredPhoto.ExpiresAt = time.Now().Add(-time.Minute).Unix()
expired, err := h.encGen.Generate(&expiredPhoto)
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
keys, err := seal.DeriveKeys("another-signing-key-fedcba9876543210") otherGen, err := encurl.NewGenerator("another-signing-key-fedcba9876543210")
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("encurl.NewGenerator() error = %v", err)
} }
otherGen := encurl.NewGenerator(keys.EncryptedURLKey) otherKey, err := otherGen.Generate(&photo)
otherKey, err := otherGen.Generate(&photo, time.Time{})
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
@@ -14,7 +14,7 @@ import (
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/signature" "sneak.berlin/go/pixa/internal/signature"
) )
@@ -58,18 +58,16 @@ func TestImageWithoutFormat_SignedAsJXL(t *testing.T) {
route := newImageRoute(t, newPhotoFetcher(t, signedHost)) route := newImageRoute(t, newPhotoFetcher(t, signedHost))
expires := time.Now().Add(time.Hour) expires := time.Now().Add(time.Hour)
sig := signature.New(testSigningKey).Sign(&imagerequest.Request{ sig := signature.New(testSigningKey).Sign(&signature.Request{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{ Format: string(imgcache.FormatJXL),
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJXL,
Quality: encurl.DefaultQuality, Quality: encurl.DefaultQuality,
FitMode: imagerequest.FitCover, FitMode: string(imgcache.FitCover),
}, Expires: expires,
}, expires) })
query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix()) query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix())
for _, size := range []string{"50x50.jxl", "50x50"} { for _, size := range []string{"50x50.jxl", "50x50"} {
@@ -85,15 +83,12 @@ func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) {
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler)) h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
token, err := h.encGen.Generate(&imagerequest.Request{ token, err := h.encGen.Generate(&encurl.Payload{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{ })
Size: imagerequest.Size{Width: 50, Height: 50},
},
}, time.Time{})
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
+6 -6
View File
@@ -7,7 +7,7 @@ import (
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
) )
// jxlType is the content type of JPEG XL. // jxlType is the content type of JPEG XL.
@@ -22,14 +22,14 @@ func TestFormatForAccept_JPEGXL(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
accept string accept string
want imagerequest.Format want imgcache.ImageFormat
}{ }{
{"JPEG XL-capable browser", {"JPEG XL-capable browser",
"image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imagerequest.FormatJXL}, "image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imgcache.FormatJXL},
{"JPEG XL only", jxlType, imagerequest.FormatJXL}, {"JPEG XL only", jxlType, imgcache.FormatJXL},
{"JPEG XL with a lower q than AVIF", "image/avif,image/jxl;q=0.5", {"JPEG XL with a lower q than AVIF", "image/avif,image/jxl;q=0.5",
imagerequest.FormatJXL}, imgcache.FormatJXL},
{"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imagerequest.FormatAVIF}, {"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imgcache.FormatAVIF},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -10,16 +10,10 @@ import (
"time" "time"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/allowlist" "sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/encurl" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/signature"
) )
// blockedReferer is a page on leech.example, which newRefererRoutes puts on // blockedReferer is a page on leech.example, which newRefererRoutes puts on
@@ -35,11 +29,11 @@ type countingFetcher struct {
// Fetch counts the fetch and passes it on. // Fetch counts the fetch and passes it on.
func (f *countingFetcher) Fetch( func (f *countingFetcher) Fetch(
ctx context.Context, source imagerequest.Source, ctx context.Context, url string,
) (*httpfetcher.FetchResult, error) { ) (*httpfetcher.FetchResult, error) {
f.fetches.Add(1) f.fetches.Add(1)
return f.Fetcher.Fetch(ctx, source) return f.Fetcher.Fetch(ctx, url)
} }
// newRefererRoutes returns both image routes of a Handlers whose // newRefererRoutes returns both image routes of a Handlers whose
@@ -53,41 +47,33 @@ func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher)
Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost), Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost),
} }
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
StateDir: t.TempDir(), StateDir: t.TempDir(),
NegativeTTL: 5 * time.Minute, NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30, })
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err) t.Fatalf("imgcache.NewCache() error = %v", err)
} }
svc, err := imageproxy.New(&imageproxy.Config{ svc, err := imgcache.NewService(&imgcache.ServiceConfig{
Cache: cache, Cache: cache,
Fetcher: fetcher, Fetcher: fetcher,
Processor: imageprocessor.New(imageprocessor.Params{ SigningKey: testSigningKey,
MaxInputBytes: 50 << 20, Allowlist: []string{allowlistedHost},
MaxConcurrentProcessing: 2, })
}),
MaxResponseSize: 50 << 20,
}, prometheus.NewRegistry())
if err != nil { if err != nil {
t.Fatalf("imageproxy.New() error = %v", err) t.Fatalf("imgcache.NewService() error = %v", err)
} }
keys, err := seal.DeriveKeys(testSigningKey) encGen, err := encurl.NewGenerator(testSigningKey)
if err != nil { if err != nil {
t.Fatalf("seal.DeriveKeys() error = %v", err) t.Fatalf("encurl.NewGenerator() error = %v", err)
} }
encGen := encurl.NewGenerator(keys.EncryptedURLKey)
h := &Handlers{ h := &Handlers{
log: slog.New(slog.DiscardHandler), log: slog.New(slog.DiscardHandler),
imgSvc: svc, imgSvc: svc,
encGen: encGen, encGen: encGen,
signer: signature.New(testSigningKey),
allowlist: allowlist.New([]string{allowlistedHost}),
refererBlocklist: allowlist.New( refererBlocklist: allowlist.New(
[]string{"leech.example", ".hotlinker.example"}), []string{"leech.example", ".hotlinker.example"}),
} }
+21 -24
View File
@@ -13,26 +13,26 @@ import (
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/imgcache"
) )
// signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the // signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the
// JPEG at photoPath on signedHost at 50x50. It is signed with testSigningKey, // JPEG at photoPath on signedHost at 50x50, made with h's image service.
// the key the handlers that newSignedHostServer builds check signatures with. func signedPhotoURL(t *testing.T, h *Handlers) string {
// It takes a Handlers it does not use, so that it has the type of encPhotoURL.
func signedPhotoURL(t *testing.T, _ *Handlers) string {
t.Helper() t.Helper()
return generateSignedURL(testSigningKey, &imagerequest.Request{ signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Size: imgcache.Size{Width: 50, Height: 50},
}, Format: imgcache.FormatJPEG,
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
},
}, time.Minute) }, time.Minute)
if err != nil {
t.Fatalf("GenerateSignedURL() error = %v", err)
}
return signedURL
} }
// encPhotoURL returns an encrypted /v1/e/ URL, which never expires, for the // encPhotoURL returns an encrypted /v1/e/ URL, which never expires, for the
@@ -40,16 +40,13 @@ func signedPhotoURL(t *testing.T, _ *Handlers) string {
func encPhotoURL(t *testing.T, h *Handlers) string { func encPhotoURL(t *testing.T, h *Handlers) string {
t.Helper() t.Helper()
token, err := h.encGen.Generate(&imagerequest.Request{ token, err := h.encGen.Generate(&encurl.Payload{
Source: imagerequest.Source{ SourceHost: signedHost,
Host: signedHost, SourcePath: photoPath,
Path: photoPath, Width: 50,
}, Height: 50,
Transform: imagerequest.Transform{ Format: imgcache.FormatJPEG,
Size: imagerequest.Size{Width: 50, Height: 50}, })
Format: imagerequest.FormatJPEG,
},
}, time.Time{})
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
} }
+17 -42
View File
@@ -1,7 +1,6 @@
package handlers package handlers
import ( import (
"context"
"fmt" "fmt"
"log/slog" "log/slog"
"net/http" "net/http"
@@ -11,64 +10,40 @@ import (
"sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imageproxy" "sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestServerBusyAnswers503 checks that respondImageError, which answers the // TestServerBusyAnswers503 checks that both image routes answer 503 with
// errors of both image routes, answers 503 with a clear error when the image // a clear error when the image service gives up waiting for a free upstream
// service gives up waiting for a free upstream connection or processing slot, // connection or processing slot, wrapped as the service wraps them.
// wrapped as the service wraps them.
func TestServerBusyAnswers503(t *testing.T) { func TestServerBusyAnswers503(t *testing.T) {
t.Parallel() t.Parallel()
h := &Handlers{log: slog.New(slog.DiscardHandler)} h := &Handlers{log: slog.New(slog.DiscardHandler)}
req := &imagerequest.Request{ req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.jpg"}
Source: imagerequest.Source{Host: "img.example.com", Path: "/a.jpg"},
}
for _, err := range []error{ for _, err := range []error{
fmt.Errorf("upstream fetch failed: %w", httpfetcher.ErrTooManyConnections), fmt.Errorf("upstream fetch failed: %w", httpfetcher.ErrTooManyConnections),
fmt.Errorf("image processing failed: %w", imageprocessor.ErrTooManyImages), fmt.Errorf("image processing failed: %w", imageprocessor.ErrTooManyImages),
} { } {
rec := httptest.NewRecorder() plain := httptest.NewRecorder()
h.respondImageError(rec, req, err) h.respondImageError(plain, req, err)
encrypted := httptest.NewRecorder()
h.handleImageError(encrypted, err)
for route, rec := range map[string]*httptest.ResponseRecorder{
"/v1/image/": plain, "/v1/e/": encrypted,
} {
if rec.Code != http.StatusServiceUnavailable { if rec.Code != http.StatusServiceUnavailable {
t.Errorf("for %v: status = %d, want %d", t.Errorf("%s for %v: status = %d, want %d",
err, rec.Code, http.StatusServiceUnavailable) route, err, rec.Code, http.StatusServiceUnavailable)
} }
if !strings.Contains(rec.Body.String(), "server busy, try again later") { if !strings.Contains(rec.Body.String(), "server busy, try again later") {
t.Errorf("for %v: body = %q, want the server busy error", t.Errorf("%s for %v: body = %q, want the server busy error",
err, rec.Body.String()) route, err, rec.Body.String())
} }
} }
} }
// TestProcessingTimedOutAnswers503 checks that respondImageError, which
// answers the errors of both image routes, answers 503 with the server busy
// error when the image service reports that processing stopped at the
// deadline of the request doing it, wrapped as the service wraps it. A request
// that waited for that processing gets this error while its own deadline may
// have time left. The request's source only goes into a log line, so it is
// left empty.
func TestProcessingTimedOutAnswers503(t *testing.T) {
t.Parallel()
h := &Handlers{log: slog.New(slog.DiscardHandler)}
err := fmt.Errorf("%w: failed to encode: %w",
imageproxy.ErrProcessingTimedOut, context.DeadlineExceeded)
rec := httptest.NewRecorder()
h.respondImageError(rec, &imagerequest.Request{}, err)
t.Logf("status = %d, body = %q", rec.Code, rec.Body.String())
if rec.Code != http.StatusServiceUnavailable {
t.Errorf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable)
}
if !strings.Contains(rec.Body.String(), "server busy, try again later") {
t.Errorf("body = %q, want the server busy error", rec.Body.String())
}
} }
@@ -10,40 +10,45 @@ import (
"sneak.berlin/go/pixa/internal/httpfetcher" "sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor" "sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest" "sneak.berlin/go/pixa/internal/imgcache"
) )
// TestUnsupportedSourceFormatAnswers415 checks that respondImageError, which // TestUnsupportedSourceFormatAnswers415 checks that both image routes answer
// answers the errors of both image routes, answers 415 with a clear error when // 415 with a clear error when the upstream answers with a content type pixa
// the upstream answers with a content type pixa does not accept, such as SVG, // does not accept, such as SVG, and when orig is asked of a source in a
// and when libvips does not read a source as the format its first bytes // format the processor cannot name, wrapped as the service wraps them.
// show, wrapped as the service wraps them.
func TestUnsupportedSourceFormatAnswers415(t *testing.T) { func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
t.Parallel() t.Parallel()
h := &Handlers{log: slog.New(slog.DiscardHandler)} h := &Handlers{log: slog.New(slog.DiscardHandler)}
req := &imagerequest.Request{ req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.svg"}
Source: imagerequest.Source{Host: "img.example.com", Path: "/a.svg"},
}
for _, err := range []error{ for _, err := range []error{
fmt.Errorf("upstream fetch failed: %w: image/svg+xml", fmt.Errorf("upstream fetch failed: %w: image/svg+xml",
httpfetcher.ErrInvalidContentType), httpfetcher.ErrInvalidContentType),
fmt.Errorf("image processing failed: %w: libvips does not read the input as avif", fmt.Errorf("image processing failed: %w: unknown",
imageprocessor.ErrUnsupportedInputFormat), imageprocessor.ErrUnsupportedInputFormat),
} { } {
rec := httptest.NewRecorder() plain := httptest.NewRecorder()
h.respondImageError(rec, req, err) h.respondImageError(plain, req, err)
t.Logf("for %v: %d %s", err, rec.Code, rec.Body.String())
encrypted := httptest.NewRecorder()
h.handleImageError(encrypted, err)
for route, rec := range map[string]*httptest.ResponseRecorder{
"HandleImage": plain, "HandleImageEnc": encrypted,
} {
t.Logf("%s for %v: %d %s", route, err, rec.Code, rec.Body.String())
if rec.Code != http.StatusUnsupportedMediaType { if rec.Code != http.StatusUnsupportedMediaType {
t.Errorf("for %v: status = %d, want %d", t.Errorf("%s for %v: status = %d, want %d",
err, rec.Code, http.StatusUnsupportedMediaType) route, err, rec.Code, http.StatusUnsupportedMediaType)
} }
if !strings.Contains(rec.Body.String(), "unsupported source format") { if !strings.Contains(rec.Body.String(), "unsupported source format") {
t.Errorf("for %v: body = %q, want the unsupported source format error", t.Errorf("%s for %v: body = %q, want the unsupported source format error",
err, rec.Body.String()) route, err, rec.Body.String())
}
} }
} }
} }
@@ -1,211 +0,0 @@
package handlers
import (
"testing"
"time"
"sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/signature"
)
// Addresses the port tests below put on the allowlist.
const (
allowlistedIPv4 = "203.0.113.5"
allowlistedIPv6 = "2001:db8::1"
)
// newSignatureChecker returns Handlers that check signatures with
// testSigningKey and have allowlistHosts on their allowlist, as New builds
// them from signing_key and allowlist_hosts.
func newSignatureChecker(allowlistHosts ...string) *Handlers {
return &Handlers{
signer: signature.New(testSigningKey),
allowlist: allowlist.New(allowlistHosts),
}
}
func TestValidateSignature_NonAllowlistedHost_NoSignature(t *testing.T) {
t.Parallel()
h := newSignatureChecker(allowlistedHost)
// Should fail validation - not allowlisted and no signature
err := h.validateSignature(photoRequest(signedHost), "", time.Time{})
if err == nil {
t.Error("validateSignature() expected error for non-allowlisted " +
"host without signature")
}
}
func TestValidateSignature_NonAllowlistedHost_ValidSignature(t *testing.T) {
t.Parallel()
h := newSignatureChecker(allowlistedHost)
req := photoRequest(signedHost)
// Generate a valid signature
expires := time.Now().Add(time.Hour)
sig := signature.New(testSigningKey).Sign(req, expires)
// Should pass validation
err := h.validateSignature(req, sig, expires)
if err != nil {
t.Errorf("validateSignature() error = %v", err)
}
}
func TestValidateSignature_NonAllowlistedHost_ExpiredSignature(t *testing.T) {
t.Parallel()
h := newSignatureChecker(allowlistedHost)
req := photoRequest(signedHost)
// Generate an expired signature
expires := time.Now().Add(-time.Hour)
sig := signature.New(testSigningKey).Sign(req, expires)
// Should fail validation
err := h.validateSignature(req, sig, expires)
if err == nil {
t.Error("validateSignature() expected error for expired signature")
}
}
func TestValidateSignature_NonAllowlistedHost_InvalidSignature(t *testing.T) {
t.Parallel()
h := newSignatureChecker(allowlistedHost)
// Should fail validation
err := h.validateSignature(photoRequest(signedHost), "invalid-signature",
time.Now().Add(time.Hour))
if err == nil {
t.Error("validateSignature() expected error for invalid signature")
}
}
// TestValidateSignature_NoAllowlist checks that with no host on the
// allowlist, a request without a signature fails validation.
func TestValidateSignature_NoAllowlist(t *testing.T) {
t.Parallel()
h := newSignatureChecker()
err := h.validateSignature(photoRequest(signedHost), "", time.Time{})
if err == nil {
t.Error("validateSignature() expected error when no host is allowlisted")
}
}
// TestValidateSignature_SignatureExactHostMatch verifies that
// validateSignature enforces exact host matching for signatures. A
// signature for one host must not verify for a different host, even
// if they share a domain suffix.
func TestValidateSignature_SignatureExactHostMatch(t *testing.T) {
t.Parallel()
h := newSignatureChecker()
// Sign a request for signedHost
signedReq := photoRequest(signedHost)
expires := time.Now().Add(time.Hour)
sig := signature.New(testSigningKey).Sign(signedReq, expires)
// The original request should pass validation
t.Run("exact host passes", func(t *testing.T) {
t.Parallel()
err := h.validateSignature(signedReq, sig, expires)
if err != nil {
t.Errorf("validateSignature() exact host failed: %v", err)
}
})
// Try to reuse the signature with different hosts
tests := []struct {
name string
host string
}{
{"parent domain", "example.com"},
{"sibling subdomain", "images.example.com"},
{"deeper subdomain", "a." + signedHost},
{"evil suffix domain", signedHost + ".evil.com"},
{"prefixed host", "evil" + signedHost},
}
for _, tt := range tests {
t.Run(tt.name+" rejected", func(t *testing.T) {
t.Parallel()
err := h.validateSignature(photoRequest(tt.host), sig, expires)
if err == nil {
t.Errorf("validateSignature() should reject signature for "+
"host %q (signed for %q)", tt.host, signedHost)
}
})
}
}
// TestValidateSignature_HostWithPortNeedsSignature verifies that a host
// followed by a port, or by a colon with no port, matches no allowlist entry,
// so that a request for it without a signature is refused, while the same
// host with no port is allowlisted.
func TestValidateSignature_HostWithPortNeedsSignature(t *testing.T) {
t.Parallel()
h := newSignatureChecker(allowlistedHost, allowlistedIPv4, allowlistedIPv6)
tests := []struct {
host string
allowlisted bool
}{
{allowlistedHost, true},
{allowlistedHost + ":8443", false},
{allowlistedHost + ":443", false},
{allowlistedHost + ":", false},
{allowlistedIPv4, true},
{allowlistedIPv4 + ":8080", false},
{"[" + allowlistedIPv6 + "]", true},
{"[" + allowlistedIPv6 + "]:8443", false},
}
for _, tt := range tests {
t.Run(tt.host, func(t *testing.T) {
t.Parallel()
err := h.validateSignature(photoRequest(tt.host), "", time.Time{})
t.Logf("validateSignature() for host %q with no signature: %v",
tt.host, err)
if allowlisted := err == nil; allowlisted != tt.allowlisted {
t.Errorf("host %q served without a signature = %v, want %v",
tt.host, allowlisted, tt.allowlisted)
}
})
}
}
// TestValidateSignature_UnbracketedIPv6PortNeedsSignature verifies that an
// allowlisted IPv6 address written without brackets and followed by a port,
// or by a colon with no port, needs a signature. Go's URL type reads
// 2001:db8::1:8443 as the host 2001:db8::1 with the port 8443.
func TestValidateSignature_UnbracketedIPv6PortNeedsSignature(t *testing.T) {
t.Parallel()
h := newSignatureChecker(allowlistedIPv6)
for _, host := range []string{allowlistedIPv6 + ":8443", allowlistedIPv6 + ":"} {
t.Run(host, func(t *testing.T) {
t.Parallel()
err := h.validateSignature(photoRequest(host), "", time.Time{})
t.Logf("validateSignature() for host %q with no signature: %v",
host, err)
if err == nil {
t.Errorf("host %q served without a signature, want refused",
host)
}
})
}
}
@@ -7,7 +7,6 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"testing" "testing"
"time"
) )
// TestIsPrivateIPBlocksSpecialRanges covers the internal and special-use // TestIsPrivateIPBlocksSpecialRanges covers the internal and special-use
@@ -71,15 +70,12 @@ func transportOf(t *testing.T, f *HTTPFetcher) *http.Transport {
func TestDialerEnforcesBlockedNetworks(t *testing.T) { func TestDialerEnforcesBlockedNetworks(t *testing.T) {
t.Parallel() t.Parallel()
transport := transportOf(t, New(&Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
// TEST-NET-2 (198.51.100.0/24) is public to the built-in check, so // TEST-NET-2 (198.51.100.0/24) is public to the built-in check, so
// blocking it can only come from the operator-supplied list. // blocking it can only come from the operator-supplied list.
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}, cfg.BlockedNetworks = []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}
}))
transport := transportOf(t, New(cfg))
blocked := []string{ blocked := []string{
"198.51.100.5:80", // operator-supplied range "198.51.100.5:80", // operator-supplied range
@@ -4,7 +4,6 @@ import (
"errors" "errors"
"net" "net"
"testing" "testing"
"time"
) )
// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built // TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built
@@ -14,12 +13,7 @@ func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) {
t.Parallel() t.Parallel()
srv := startUpstream(t) srv := startUpstream(t)
transport := transportOf(t, New(&Config{ transport := transportOf(t, New(DefaultConfig()))
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
}))
addr := srv.Listener.Addr().String() addr := srv.Listener.Addr().String()
@@ -38,25 +32,21 @@ func TestDialContextReplacesOnlyTheDialer(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
dialer := &recordingDialer{target: srv.Listener.Addr().String()} dialer := &recordingDialer{target: srv.Listener.Addr().String()}
f := New(&Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.AllowHTTP = true
MaxResponseSize: 50 << 20, cfg.DialContext = dialer.dialContext
AllowHTTP: true, f := New(cfg)
MaxConnectionsPerHost: 20,
MaxConnections: 64,
DialContext: dialer.dialContext,
})
if body := fetchBody(t, f, "/image"); body != imagePayload { if body := fetchBody(t, f, "/image"); body != imagePayload {
t.Errorf("body = %q, want %q", body, imagePayload) t.Errorf("body = %q, want %q", body, imagePayload)
} }
_, err := f.Fetch(testContext(t), imageSourceOn("127.0.0.1")) _, err := f.Fetch(testContext(t), "http://127.0.0.1/image")
if !errors.Is(err, ErrSSRFBlocked) { if !errors.Is(err, ErrSSRFBlocked) {
t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err) t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err)
} }
_, err = f.Fetch(testContext(t), upstreamSource("/redirect/private")) _, err = f.Fetch(testContext(t), upstreamURL("/redirect/private"))
if !errors.Is(err, ErrSSRFBlocked) { if !errors.Is(err, ErrSSRFBlocked) {
t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err) t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err)
} }
+18 -225
View File
@@ -9,13 +9,10 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"slices" "slices"
"strconv"
"strings" "strings"
"sync" "sync"
"testing" "testing"
"time" "time"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// testPublicHost is a TEST-NET-1 (RFC 5737) literal. isPrivateIP treats it as // testPublicHost is a TEST-NET-1 (RFC 5737) literal. isPrivateIP treats it as
@@ -31,9 +28,9 @@ const imagePayload = "fake-jpeg-bytes"
// would mean SSRF protection let a forbidden target reach the transport. // would mean SSRF protection let a forbidden target reach the transport.
var errUnexpectedDial = errors.New("unexpected dial target") var errUnexpectedDial = errors.New("unexpected dial target")
// upstreamSource returns the source at path on the fake public host. // upstreamURL builds a fetch URL on the fake public host for the given path.
func upstreamSource(path string) imagerequest.Source { func upstreamURL(path string) string {
return imagerequest.Source{Host: testPublicHost, Path: path} return "http://" + testPublicHost + path
} }
// recordingDialer records every address the transport asks it to dial and // recordingDialer records every address the transport asks it to dial and
@@ -83,7 +80,7 @@ func startUpstream(t *testing.T) *httptest.Server {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/image", func(w http.ResponseWriter, _ *http.Request) { mux.HandleFunc("/image", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType()) w.Header().Set("Content-Type", contentTypeJPEG)
_, _ = io.WriteString(w, imagePayload) _, _ = io.WriteString(w, imagePayload)
}) })
mux.HandleFunc("/status/500", func(w http.ResponseWriter, _ *http.Request) { mux.HandleFunc("/status/500", func(w http.ResponseWriter, _ *http.Request) {
@@ -122,12 +119,7 @@ func newServerFetcher(
t.Helper() t.Helper()
if cfg == nil { if cfg == nil {
cfg = &Config{ cfg = DefaultConfig()
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
}
} }
cfg.AllowHTTP = true cfg.AllowHTTP = true
@@ -160,7 +152,7 @@ func testContext(t *testing.T) context.Context {
func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult { func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult {
t.Helper() t.Helper()
res, err := f.Fetch(testContext(t), upstreamSource(path)) res, err := f.Fetch(testContext(t), upstreamURL(path))
if err != nil { if err != nil {
t.Fatalf("Fetch(%s) error = %v", path, err) t.Fatalf("Fetch(%s) error = %v", path, err)
} }
@@ -172,7 +164,7 @@ func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult {
func fetchExpectError(t *testing.T, f *HTTPFetcher, path string) error { func fetchExpectError(t *testing.T, f *HTTPFetcher, path string) error {
t.Helper() t.Helper()
res, err := f.Fetch(testContext(t), upstreamSource(path)) res, err := f.Fetch(testContext(t), upstreamURL(path))
if err == nil { if err == nil {
_ = res.Content.Close() _ = res.Content.Close()
@@ -224,7 +216,7 @@ func TestFetchRedirectToPrivateIPBlocked(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
f, dialer := newServerFetcher(t, srv, nil) f, dialer := newServerFetcher(t, srv, nil)
_, err := f.Fetch(testContext(t), upstreamSource("/redirect/private")) _, err := f.Fetch(testContext(t), upstreamURL("/redirect/private"))
if !errors.Is(err, ErrSSRFBlocked) { if !errors.Is(err, ErrSSRFBlocked) {
t.Fatalf("Fetch() error = %v, want ErrSSRFBlocked", err) t.Fatalf("Fetch() error = %v, want ErrSSRFBlocked", err)
} }
@@ -287,12 +279,8 @@ func TestFetchMaxResponseSizeEnforced(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxResponseSize = 8
MaxResponseSize: 8,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
@@ -309,154 +297,13 @@ func TestFetchMaxResponseSizeEnforced(t *testing.T) {
} }
} }
// fetchWithMaxResponseSize serves handler as the fake upstream and fetches
// its /image route with MaxResponseSize set to maxResponseSize.
func fetchWithMaxResponseSize(
t *testing.T,
maxResponseSize int64,
handler http.HandlerFunc,
) *FetchResult {
t.Helper()
srv := httptest.NewServer(handler)
t.Cleanup(srv.Close)
cfg := &Config{
Timeout: 30 * time.Second,
MaxResponseSize: maxResponseSize,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
}
f, _ := newServerFetcher(t, srv, cfg)
res := fetchImage(t, f, "/image")
t.Cleanup(func() { _ = res.Content.Close() })
return res
}
// TestFetchContentLengthBodyAtMaxResponseSize checks that a body sent with a
// Content-Length is read whole at exactly MaxResponseSize bytes and refused
// at one byte more.
func TestFetchContentLengthBodyAtMaxResponseSize(t *testing.T) {
t.Parallel()
const maxResponseSize = 8
for _, tc := range []struct {
size int
wantErr error
}{
{maxResponseSize, nil},
{maxResponseSize + 1, ErrResponseTooLarge},
} {
t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) {
t.Parallel()
body := strings.Repeat("x", tc.size)
res := fetchWithMaxResponseSize(t, maxResponseSize,
func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType())
w.Header().Set("Content-Length", strconv.Itoa(tc.size))
_, _ = io.WriteString(w, body)
})
if res.ContentLength != int64(tc.size) {
t.Fatalf("ContentLength = %d, want %d", res.ContentLength, tc.size)
}
data, err := io.ReadAll(res.Content)
t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err)
if !errors.Is(err, tc.wantErr) {
t.Fatalf("read error = %v, want %v", err, tc.wantErr)
}
if err == nil && string(data) != body {
t.Errorf("read %q, want %q", data, body)
}
})
}
}
// TestFetchChunkedBodyAtMaxResponseSize checks that a chunked body is read
// whole at exactly MaxResponseSize bytes and refused at one byte more. The
// upstream sends the end of the body only once the test has read
// MaxResponseSize bytes, so the read after them comes before the end of the
// body has arrived, as it can from a real upstream.
func TestFetchChunkedBodyAtMaxResponseSize(t *testing.T) {
t.Parallel()
const maxResponseSize = 8
for _, tc := range []struct {
size int
wantErr error
}{
{maxResponseSize, nil},
{maxResponseSize + 1, ErrResponseTooLarge},
} {
t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) {
t.Parallel()
body := strings.Repeat("x", tc.size)
endBody := make(chan struct{})
res := fetchWithMaxResponseSize(t, maxResponseSize,
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType())
_, _ = io.WriteString(w, body)
// With no Content-Length, flushing sends the body
// chunked; its end is sent when the handler returns.
_ = http.NewResponseController(w).Flush()
select {
case <-endBody:
case <-r.Context().Done():
}
})
if res.ContentLength != -1 {
t.Fatalf("ContentLength = %d, want -1 (chunked)", res.ContentLength)
}
head := make([]byte, maxResponseSize)
_, err := io.ReadFull(res.Content, head)
if err != nil {
t.Fatalf("read the first %d bytes: %v", maxResponseSize, err)
}
close(endBody)
rest, err := io.ReadAll(res.Content)
data := string(head) + string(rest)
t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err)
if !errors.Is(err, tc.wantErr) {
t.Fatalf("read error = %v, want %v", err, tc.wantErr)
}
if err == nil && data != body {
t.Errorf("read %q, want %q", data, body)
}
})
}
}
func TestFetchSemaphoreReleasedOnError(t *testing.T) { func TestFetchSemaphoreReleasedOnError(t *testing.T) {
t.Parallel() t.Parallel()
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxConnectionsPerHost = 1
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 1,
MaxConnections: 64,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
@@ -485,12 +332,8 @@ func assertSlotReleasedByClose(
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxConnectionsPerHost = 1
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 1,
MaxConnections: 64,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
@@ -547,7 +390,7 @@ func TestFetchSemaphoreReleasedOnPartialReadClose(t *testing.T) {
}) })
} }
// The dial-time re-resolution in dialSSRFSafe is what closes the DNS // The dial-time re-resolution in ssrfSafeDialer is what closes the DNS
// rebinding window: even if validateURL saw a public answer earlier, the // rebinding window: even if validateURL saw a public answer earlier, the
// dialer independently re-checks the address it is about to connect to. A full // dialer independently re-checks the address it is about to connect to. A full
// rebinding simulation (a resolver returning public, then private) would mean // rebinding simulation (a resolver returning public, then private) would mean
@@ -567,9 +410,9 @@ func TestSSRFSafeDialerBlocksPrivateTarget(t *testing.T) {
t.Run(addr, func(t *testing.T) { t.Run(addr, func(t *testing.T) {
t.Parallel() t.Parallel()
_, err := dialSSRFSafe(context.Background(), "tcp", addr, nil) _, err := ssrfSafeDialer(context.Background(), "tcp", addr)
if !errors.Is(err, ErrSSRFBlocked) { if !errors.Is(err, ErrSSRFBlocked) {
t.Errorf("dialSSRFSafe(%q) = %v, want ErrSSRFBlocked", addr, err) t.Errorf("ssrfSafeDialer(%q) = %v, want ErrSSRFBlocked", addr, err)
} }
}) })
} }
@@ -583,7 +426,7 @@ func TestSSRFSafeDialerAllowsPublicTarget(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
_, err := dialSSRFSafe(ctx, "tcp", testPublicHost+":80", nil) _, err := ssrfSafeDialer(ctx, "tcp", testPublicHost+":80")
if err == nil { if err == nil {
t.Fatal("expected a dial error for an unreachable public target") t.Fatal("expected a dial error for an unreachable public target")
} }
@@ -592,53 +435,3 @@ func TestSSRFSafeDialerAllowsPublicTarget(t *testing.T) {
t.Errorf("public target was SSRF-blocked: %v", err) t.Errorf("public target was SSRF-blocked: %v", err)
} }
} }
// TestFetchSchemeFollowsAllowHTTP checks that the fetcher fetches a source
// over https, on port 443, unless AllowHTTP is set, and then over http, on
// port 80. The test server speaks plain HTTP, so only the fetch over http
// succeeds.
func TestFetchSchemeFollowsAllowHTTP(t *testing.T) {
t.Parallel()
tests := []struct {
allowHTTP bool
wantAddr string
}{
{false, net.JoinHostPort(testPublicHost, "443")},
{true, net.JoinHostPort(testPublicHost, "80")},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("AllowHTTP %v", tt.allowHTTP), func(t *testing.T) {
t.Parallel()
srv := startUpstream(t)
dialer := &recordingDialer{target: srv.Listener.Addr().String()}
f := New(&Config{
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
AllowHTTP: tt.allowHTTP,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
DialContext: dialer.dialContext,
})
res, err := f.Fetch(testContext(t), upstreamSource("/image"))
if err == nil {
_ = res.Content.Close()
}
dialed := dialer.dialedAddrs()
t.Logf("Fetch(): error %v, dialed %v", err, dialed)
if served := err == nil; served != tt.allowHTTP {
t.Errorf("served = %v, want %v", served, tt.allowHTTP)
}
if len(dialed) != 1 || dialed[0] != tt.wantAddr {
t.Errorf("dialed %v, want only %s", dialed, tt.wantAddr)
}
})
}
}
+105 -95
View File
@@ -20,24 +20,36 @@ import (
"time" "time"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// Fetcher configuration constants. // Fetcher configuration constants.
const ( const (
DefaultFetchTimeout = 30 * time.Second
DefaultMaxResponseSize = 50 << 20 // 50MB
DefaultTLSTimeout = 10 * time.Second DefaultTLSTimeout = 10 * time.Second
DefaultMaxIdleConns = 100 DefaultMaxIdleConns = 100
DefaultIdleConnTimeout = 90 * time.Second DefaultIdleConnTimeout = 90 * time.Second
DefaultMaxRedirects = 10 DefaultMaxRedirects = 10
DefaultMaxConnectionsPerHost = 20
DefaultMaxConnections = 64
) )
// userAgent is the User-Agent header sent to upstream servers.
const userAgent = "pixa/1.0"
// ConnectionWaitTimeout is how long Fetch waits for a free connection when // ConnectionWaitTimeout is how long Fetch waits for a free connection when
// MaxConnections fetches are already in progress. // MaxConnections fetches are already in progress.
const ConnectionWaitTimeout = 10 * time.Second const ConnectionWaitTimeout = 10 * time.Second
// MIME content types.
const (
contentTypeJPEG = "image/jpeg"
contentTypePNG = "image/png"
contentTypeGIF = "image/gif"
contentTypeWebP = "image/webp"
contentTypeAVIF = "image/avif"
contentTypeJXL = "image/jxl"
contentTypeSVG = "image/svg+xml"
contentTypeOctetStream = "application/octet-stream"
)
// Loopback addresses blocked by SSRF protection. // Loopback addresses blocked by SSRF protection.
const ( const (
localhostIPv4 = "127.0.0.1" localhostIPv4 = "127.0.0.1"
@@ -47,9 +59,8 @@ const (
// builtinBlockedPrefixes are internal or special-use ranges that Go's // builtinBlockedPrefixes are internal or special-use ranges that Go's
// net.IP predicates (IsPrivate, IsLinkLocalUnicast, and the like) do not // net.IP predicates (IsPrivate, IsLinkLocalUnicast, and the like) do not
// already cover. They are always blocked, in addition to any // already cover. They are always blocked, in addition to any
// operator-supplied networks. IPv4-mapped and IPv4-compatible IPv6 // operator-supplied networks. IPv4-mapped IPv6 addresses are unmapped
// addresses are read as the IPv4 address they hold before matching, so // before matching, so these IPv4 ranges are caught in both forms.
// these IPv4 ranges are caught in all three forms.
// //
//nolint:gochecknoglobals // immutable built-in blocklist //nolint:gochecknoglobals // immutable built-in blocklist
var builtinBlockedPrefixes = []netip.Prefix{ var builtinBlockedPrefixes = []netip.Prefix{
@@ -57,7 +68,6 @@ var builtinBlockedPrefixes = []netip.Prefix{
netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments
netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range
netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4) netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4)
netip.MustParsePrefix("64:ff9b:1::/48"), // RFC 8215 local-use NAT64 (maps onto IPv4)
} }
// Fetcher errors. // Fetcher errors.
@@ -80,8 +90,8 @@ var (
// Fetcher retrieves content from upstream origins. // Fetcher retrieves content from upstream origins.
type Fetcher interface { type Fetcher interface {
// Fetch retrieves the content of the source image. // Fetch retrieves content from the given URL.
Fetch(ctx context.Context, source imagerequest.Source) (*FetchResult, error) Fetch(ctx context.Context, url string) (*FetchResult, error)
} }
// FetchResult contains the result of fetching from upstream. // FetchResult contains the result of fetching from upstream.
@@ -108,29 +118,17 @@ type FetchResult struct {
TLSCipherSuite string TLSCipherSuite string
} }
// allowedContentTypes returns the content types an upstream response may // Config holds configuration for the upstream fetcher.
// have, which Fetch also sends as its Accept header: the MIME types of the
// formats pixa reads.
func allowedContentTypes() []string {
return []string{
imagerequest.FormatJPEG.MIMEType(),
imagerequest.FormatPNG.MIMEType(),
imagerequest.FormatGIF.MIMEType(),
imagerequest.FormatWebP.MIMEType(),
imagerequest.FormatAVIF.MIMEType(),
imagerequest.FormatJXL.MIMEType(),
}
}
// Config holds configuration for the upstream fetcher. Every field but
// AllowHTTP, BlockedNetworks and DialContext must be set.
type Config struct { type Config struct {
// Timeout for upstream requests. // Timeout for upstream requests.
Timeout time.Duration Timeout time.Duration
// MaxResponseSize is the maximum allowed response body size. // MaxResponseSize is the maximum allowed response body size.
MaxResponseSize int64 MaxResponseSize int64
// AllowHTTP makes the fetcher fetch sources over http instead of https, // UserAgent to send to upstream servers.
// and follow redirects to http (for testing only). UserAgent string
// AllowedContentTypes is an allow list of MIME types to accept.
AllowedContentTypes []string
// AllowHTTP allows non-TLS connections (for testing only).
AllowHTTP bool AllowHTTP bool
// MaxConnectionsPerHost limits concurrent connections to each upstream host. // MaxConnectionsPerHost limits concurrent connections to each upstream host.
MaxConnectionsPerHost int MaxConnectionsPerHost int
@@ -147,6 +145,26 @@ type Config struct {
DialContext func(ctx context.Context, network, addr string) (net.Conn, error) DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
} }
// DefaultConfig returns a Config with sensible defaults.
func DefaultConfig() *Config {
return &Config{
Timeout: DefaultFetchTimeout,
MaxResponseSize: DefaultMaxResponseSize,
UserAgent: "pixa/1.0",
AllowedContentTypes: []string{
contentTypeJPEG,
contentTypePNG,
contentTypeGIF,
contentTypeWebP,
contentTypeAVIF,
contentTypeJXL,
},
AllowHTTP: false,
MaxConnectionsPerHost: DefaultMaxConnectionsPerHost,
MaxConnections: DefaultMaxConnections,
}
}
// HTTPFetcher implements Fetcher with SSRF protection and connection limits // HTTPFetcher implements Fetcher with SSRF protection and connection limits
// per host and for all hosts together. // per host and for all hosts together.
type HTTPFetcher struct { type HTTPFetcher struct {
@@ -174,6 +192,10 @@ type hostSemaphore struct {
// New creates a new HTTPFetcher with SSRF protection. // New creates a new HTTPFetcher with SSRF protection.
func New(config *Config) *HTTPFetcher { func New(config *Config) *HTTPFetcher {
if config == nil {
config = DefaultConfig()
}
// Unless config.DialContext replaces it, the transport connects with // Unless config.DialContext replaces it, the transport connects with
// the SSRF-safe dialer, which re-resolves and re-checks at connect time // the SSRF-safe dialer, which re-resolves and re-checks at connect time
// (closing the DNS-rebinding window) against both the built-in ranges // (closing the DNS-rebinding window) against both the built-in ranges
@@ -202,7 +224,7 @@ func New(config *Config) *HTTPFetcher {
} }
// Validate the redirect target // Validate the redirect target
err := validateURL(req.Context(), req.URL, config.AllowHTTP) err := validateURL(req.Context(), req.URL.String(), config.AllowHTTP)
if err != nil { if err != nil {
return fmt.Errorf("redirect blocked: %w", err) return fmt.Errorf("redirect blocked: %w", err)
} }
@@ -220,22 +242,18 @@ func New(config *Config) *HTTPFetcher {
} }
} }
// Fetch retrieves the source image with SSRF protection, over https, or over // Fetch retrieves content from the given URL with SSRF protection. When
// http when AllowHTTP is set. When MaxConnections fetches are already in // MaxConnections fetches are already in progress, it waits up to
// progress, it waits up to ConnectionWaitTimeout for one to finish, then // ConnectionWaitTimeout for one to finish, then fails with
// fails with ErrTooManyConnections. // ErrTooManyConnections.
func (f *HTTPFetcher) Fetch( func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, error) {
ctx context.Context, source imagerequest.Source,
) (*FetchResult, error) {
sourceURL := source.URL(f.config.AllowHTTP)
// Validate URL before making request // Validate URL before making request
err := validateURL(ctx, sourceURL, f.config.AllowHTTP) err := validateURL(ctx, url, f.config.AllowHTTP)
if err != nil { if err != nil {
return nil, err return nil, err
} }
release, err := f.acquireConnection(ctx, source.Host) release, err := f.acquireConnection(ctx, extractHost(url))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -249,14 +267,19 @@ func (f *HTTPFetcher) Fetch(
} }
}() }()
parsedURL, err := neturl.Parse(url)
if err != nil {
return nil, fmt.Errorf("failed to parse URL: %w", err)
}
req := &http.Request{ req := &http.Request{
Method: http.MethodGet, Method: http.MethodGet,
URL: sourceURL, URL: parsedURL,
Header: make(http.Header), Header: make(http.Header),
} }
req.Header.Set("User-Agent", userAgent) req.Header.Set("User-Agent", f.config.UserAgent)
req.Header.Set("Accept", strings.Join(allowedContentTypes(), ", ")) req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", "))
// The ID of the request this fetch serves, so the fetch can be found in // The ID of the request this fetch serves, so the fetch can be found in
// the upstream host's logs // the upstream host's logs
@@ -435,7 +458,7 @@ func (f *HTTPFetcher) isAllowedContentType(contentType string) bool {
// Extract the MIME type without parameters // Extract the MIME type without parameters
mediaType := strings.TrimSpace(strings.Split(contentType, ";")[0]) mediaType := strings.TrimSpace(strings.Split(contentType, ";")[0])
for _, allowed := range allowedContentTypes() { for _, allowed := range f.config.AllowedContentTypes {
if strings.EqualFold(mediaType, allowed) { if strings.EqualFold(mediaType, allowed) {
return true return true
} }
@@ -445,18 +468,23 @@ func (f *HTTPFetcher) isAllowedContentType(contentType string) bool {
} }
// validateURL checks if a URL is safe to fetch (not internal/private). // validateURL checks if a URL is safe to fetch (not internal/private).
func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) error { func validateURL(ctx context.Context, rawURL string, allowHTTP bool) error {
if !allowHTTP && fetchURL.Scheme != "https" { if !allowHTTP && !strings.HasPrefix(rawURL, "https://") {
return ErrUnsupportedScheme return ErrUnsupportedScheme
} }
// Hostname drops any port, and the brackets of an IPv6 address whether // Parse to extract host
// or not a port follows them host := extractHost(rawURL)
host := fetchURL.Hostname()
if host == "" { if host == "" {
return ErrInvalidHost return ErrInvalidHost
} }
// Remove port if present
h, _, err := net.SplitHostPort(host)
if err == nil {
host = h
}
// Block obvious localhost patterns // Block obvious localhost patterns
if isLocalhost(host) { if isLocalhost(host) {
return ErrSSRFBlocked return ErrSSRFBlocked
@@ -478,6 +506,25 @@ func validateURL(ctx context.Context, fetchURL *neturl.URL, allowHTTP bool) erro
return nil return nil
} }
// extractHost extracts the host from a URL string.
func extractHost(rawURL string) string {
// Simple extraction without full URL parsing
url := rawURL
if idx := strings.Index(url, "://"); idx != -1 {
url = url[idx+3:]
}
if idx := strings.Index(url, "/"); idx != -1 {
url = url[:idx]
}
if idx := strings.Index(url, "?"); idx != -1 {
url = url[:idx]
}
return url
}
// isLocalhost checks if the host is localhost. // isLocalhost checks if the host is localhost.
func isLocalhost(host string) bool { func isLocalhost(host string) bool {
host = strings.ToLower(host) host = strings.ToLower(host)
@@ -496,9 +543,6 @@ func isPrivateIP(ip net.IP) bool {
return true return true
} }
// An IPv4-compatible IPv6 address is judged as the IPv4 address it holds
ip = ipv4CompatibleToIPv4(ip)
// Check for loopback // Check for loopback
if ip.IsLoopback() { if ip.IsLoopback() {
return true return true
@@ -550,12 +594,7 @@ func isPrivateIP(ip net.IP) bool {
} }
// isBlockedIP reports whether ip is refused, either by the built-in // isBlockedIP reports whether ip is refused, either by the built-in
// internal-range check or by one of the operator-supplied prefixes. An // internal-range check or by one of the operator-supplied prefixes.
// IPv4-compatible address (in ::/96) is refused when a prefix contains the
// address as written or the IPv4 address it holds. An IPv4-mapped address (in
// ::ffff:0:0/96) is matched only as the IPv4 address it holds: Go keeps a
// plain IPv4 address in that form too, and an IPv6 prefix such as ::/0 must
// not refuse every IPv4 address.
func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool { func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
if isPrivateIP(ip) { if isPrivateIP(ip) {
return true return true
@@ -568,26 +607,16 @@ func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
addr = addr.Unmap() addr = addr.Unmap()
// ip read as IPv4 where it is or holds an IPv4 address, otherwise the
// zero netip.Addr, which no prefix contains. Matching asIPv4 as well as
// addr refuses more only for an IPv4-compatible address.
asIPv4, _ := netip.AddrFromSlice(ipv4CompatibleToIPv4(ip).To4())
return slices.ContainsFunc(blocked, func(prefix netip.Prefix) bool { return slices.ContainsFunc(blocked, func(prefix netip.Prefix) bool {
return prefix.Contains(addr) || prefix.Contains(asIPv4) return prefix.Contains(addr)
}) })
} }
// ipv4CompatibleToIPv4 returns the IPv4 address that an IPv4-compatible IPv6 // ssrfSafeDialer validates IP addresses against the built-in blocked ranges
// address (in ::/96, such as ::127.0.0.1) holds in its last four bytes, and // before connecting. New wraps dialSSRFSafe with the operator-supplied
// any other address unchanged. // blocklist; this entry point enforces the built-in ranges alone.
func ipv4CompatibleToIPv4(ip net.IP) net.IP { func ssrfSafeDialer(ctx context.Context, network, addr string) (net.Conn, error) {
addr, ok := netip.AddrFromSlice(ip) return dialSSRFSafe(ctx, network, addr, nil)
if ok && netip.MustParsePrefix("::/96").Contains(addr) {
return ip[12:]
}
return ip
} }
// dialSSRFSafe re-resolves addr and refuses to connect to any built-in // dialSSRFSafe re-resolves addr and refuses to connect to any built-in
@@ -631,33 +660,14 @@ func dialSSRFSafe(
return nil, fmt.Errorf("%w to %s", errConnectFailed, host) return nil, fmt.Errorf("%w to %s", errConnectFailed, host)
} }
// limitedReader wraps a reader and refuses a body longer than remaining // limitedReader wraps a reader and limits the number of bytes read.
// bytes with ErrResponseTooLarge. remaining goes below 0 once the body is
// found to be longer.
type limitedReader struct { type limitedReader struct {
reader io.Reader reader io.Reader
remaining int64 remaining int64
} }
// Read reads at most the remaining bytes. Once they are read, it reads one
// more byte to tell the end of the body from more data: at the end it
// returns the wrapped reader's error, io.EOF; with more data it returns
// ErrResponseTooLarge, as does every later Read.
func (r *limitedReader) Read(p []byte) (int, error) { func (r *limitedReader) Read(p []byte) (int, error) {
if r.remaining < 0 { if r.remaining <= 0 {
return 0, ErrResponseTooLarge
}
if r.remaining == 0 {
var extra [1]byte
n, err := r.reader.Read(extra[:])
r.remaining -= int64(n)
if n == 0 {
return 0, err
}
return 0, ErrResponseTooLarge return 0, ErrResponseTooLarge
} }
@@ -5,34 +5,70 @@ import (
"errors" "errors"
"io" "io"
"net" "net"
"net/url"
"testing" "testing"
"testing/fstest" "testing/fstest"
"time"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// testHost is the hostname used by mock fetch tests. // testHost is the hostname used by mock fetch tests.
const testHost = "example.com" const testHost = "example.com"
func TestDefaultConfig(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
if cfg.Timeout != DefaultFetchTimeout {
t.Errorf("Timeout = %v, want %v", cfg.Timeout, DefaultFetchTimeout)
}
if cfg.MaxResponseSize != DefaultMaxResponseSize {
t.Errorf("MaxResponseSize = %d, want %d", cfg.MaxResponseSize, DefaultMaxResponseSize)
}
if cfg.MaxConnectionsPerHost != DefaultMaxConnectionsPerHost {
t.Errorf("MaxConnectionsPerHost = %d, want %d",
cfg.MaxConnectionsPerHost, DefaultMaxConnectionsPerHost)
}
if cfg.AllowHTTP {
t.Error("AllowHTTP should default to false")
}
if len(cfg.AllowedContentTypes) == 0 {
t.Error("AllowedContentTypes should not be empty")
}
}
func TestNewWithNilConfigUsesDefaults(t *testing.T) {
t.Parallel()
f := New(nil)
if f == nil {
t.Fatal("New(nil) returned nil")
}
if f.config == nil {
t.Fatal("config should be populated from DefaultConfig")
}
if f.config.Timeout != DefaultFetchTimeout {
t.Errorf("Timeout = %v, want %v", f.config.Timeout, DefaultFetchTimeout)
}
}
func TestIsAllowedContentType(t *testing.T) { func TestIsAllowedContentType(t *testing.T) {
t.Parallel() t.Parallel()
f := New(&Config{ f := New(DefaultConfig())
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
})
tests := []struct { tests := []struct {
contentType string contentType string
want bool want bool
}{ }{
{imagerequest.FormatJPEG.MIMEType(), true}, {contentTypeJPEG, true},
{imagerequest.FormatPNG.MIMEType(), true}, {contentTypePNG, true},
{imagerequest.FormatWebP.MIMEType(), true}, {contentTypeWebP, true},
{"image/jpeg; charset=utf-8", true}, {"image/jpeg; charset=utf-8", true},
{"IMAGE/JPEG", true}, {"IMAGE/JPEG", true},
{"text/html", false}, {"text/html", false},
@@ -52,6 +88,33 @@ func TestIsAllowedContentType(t *testing.T) {
} }
} }
func TestExtractHost(t *testing.T) {
t.Parallel()
tests := []struct {
url string
want string
}{
{"https://example.com/path", testHost},
{"http://example.com:8080/path", "example.com:8080"},
{"https://example.com", testHost},
{"https://example.com?q=1", testHost},
{"example.com/path", testHost},
{"", ""},
}
for _, tc := range tests {
t.Run(tc.url, func(t *testing.T) {
t.Parallel()
got := extractHost(tc.url)
if got != tc.want {
t.Errorf("extractHost(%q) = %q, want %q", tc.url, got, tc.want)
}
})
}
}
func TestIsLocalhost(t *testing.T) { func TestIsLocalhost(t *testing.T) {
t.Parallel() t.Parallel()
@@ -123,48 +186,10 @@ func TestIsPrivateIP(t *testing.T) {
} }
} }
// TestIsPrivateIPJudgesEmbeddedIPv4Address checks IPv6 addresses that hold an
// IPv4 address in their last four bytes: the IPv4-compatible form (::/96) and
// the NAT64 form (64:ff9b::/96). One holding a loopback, private or
// link-local IPv4 address is refused as that address is, and one holding a
// public IPv4 address in the IPv4-compatible form passes.
func TestIsPrivateIPJudgesEmbeddedIPv4Address(t *testing.T) {
t.Parallel()
tests := []struct {
ip string
want bool
}{
{"::127.0.0.1", true}, // loopback
{"::10.0.0.1", true}, // private
{"::169.254.169.254", true}, // link-local
{"64:ff9b::7f00:1", true}, // NAT64 form of 127.0.0.1
{"::8.8.8.8", false}, // public
}
for _, tc := range tests {
t.Run(tc.ip, func(t *testing.T) {
t.Parallel()
ip := net.ParseIP(tc.ip)
if ip == nil {
t.Fatalf("failed to parse IP %q", tc.ip)
}
got := isPrivateIP(ip)
t.Logf("isPrivateIP(%s) = %v", ip, got)
if got != tc.want {
t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want)
}
})
}
}
func TestValidateURL_RejectsNonHTTPS(t *testing.T) { func TestValidateURL_RejectsNonHTTPS(t *testing.T) {
t.Parallel() t.Parallel()
err := validateURL(t.Context(), &url.URL{Scheme: "http", Host: testHost}, false) err := validateURL(t.Context(), "http://example.com/path", false)
if !errors.Is(err, ErrUnsupportedScheme) { if !errors.Is(err, ErrUnsupportedScheme) {
t.Errorf("validateURL http = %v, want ErrUnsupportedScheme", err) t.Errorf("validateURL http = %v, want ErrUnsupportedScheme", err)
} }
@@ -174,8 +199,7 @@ func TestValidateURL_AllowsHTTPWhenConfigured(t *testing.T) {
t.Parallel() t.Parallel()
// Use a host that won't resolve (explicit .invalid TLD) so we don't hit DNS. // Use a host that won't resolve (explicit .invalid TLD) so we don't hit DNS.
err := validateURL(t.Context(), err := validateURL(t.Context(), "http://nonexistent.invalid/path", true)
&url.URL{Scheme: "http", Host: "nonexistent.invalid"}, true)
// We expect a host resolution error, not ErrUnsupportedScheme. // We expect a host resolution error, not ErrUnsupportedScheme.
if errors.Is(err, ErrUnsupportedScheme) { if errors.Is(err, ErrUnsupportedScheme) {
t.Error("validateURL with AllowHTTP should not return ErrUnsupportedScheme") t.Error("validateURL with AllowHTTP should not return ErrUnsupportedScheme")
@@ -185,7 +209,7 @@ func TestValidateURL_AllowsHTTPWhenConfigured(t *testing.T) {
func TestValidateURL_RejectsLocalhost(t *testing.T) { func TestValidateURL_RejectsLocalhost(t *testing.T) {
t.Parallel() t.Parallel()
err := validateURL(t.Context(), &url.URL{Scheme: "https", Host: "localhost"}, false) err := validateURL(t.Context(), "https://localhost/path", false)
if !errors.Is(err, ErrSSRFBlocked) { if !errors.Is(err, ErrSSRFBlocked) {
t.Errorf("validateURL localhost = %v, want ErrSSRFBlocked", err) t.Errorf("validateURL localhost = %v, want ErrSSRFBlocked", err)
} }
@@ -194,7 +218,7 @@ func TestValidateURL_RejectsLocalhost(t *testing.T) {
func TestValidateURL_EmptyHost(t *testing.T) { func TestValidateURL_EmptyHost(t *testing.T) {
t.Parallel() t.Parallel()
err := validateURL(t.Context(), &url.URL{Scheme: "https"}, false) err := validateURL(t.Context(), "https:///path", false)
if !errors.Is(err, ErrInvalidHost) { if !errors.Is(err, ErrInvalidHost) {
t.Errorf("validateURL empty host = %v, want ErrInvalidHost", err) t.Errorf("validateURL empty host = %v, want ErrInvalidHost", err)
} }
@@ -209,15 +233,14 @@ func TestMockFetcher_FetchesFile(t *testing.T) {
m := NewMock(mockFS) m := NewMock(mockFS)
result, err := m.Fetch(context.Background(), result, err := m.Fetch(context.Background(), "https://example.com/images/photo.jpg")
imagerequest.Source{Host: testHost, Path: "/images/photo.jpg"})
if err != nil { if err != nil {
t.Fatalf("Fetch() error = %v", err) t.Fatalf("Fetch() error = %v", err)
} }
defer func() { _ = result.Content.Close() }() defer func() { _ = result.Content.Close() }()
if result.ContentType != imagerequest.FormatJPEG.MIMEType() { if result.ContentType != contentTypeJPEG {
t.Errorf("ContentType = %q, want image/jpeg", result.ContentType) t.Errorf("ContentType = %q, want image/jpeg", result.ContentType)
} }
@@ -241,8 +264,7 @@ func TestMockFetcher_MissingFileReturnsUpstreamError(t *testing.T) {
mockFS := fstest.MapFS{} mockFS := fstest.MapFS{}
m := NewMock(mockFS) m := NewMock(mockFS)
_, err := m.Fetch(context.Background(), _, err := m.Fetch(context.Background(), "https://example.com/missing.jpg")
imagerequest.Source{Host: testHost, Path: "/missing.jpg"})
if !errors.Is(err, ErrUpstreamError) { if !errors.Is(err, ErrUpstreamError) {
t.Errorf("Fetch() error = %v, want ErrUpstreamError", err) t.Errorf("Fetch() error = %v, want ErrUpstreamError", err)
} }
@@ -259,7 +281,7 @@ func TestMockFetcher_RespectsContextCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
_, err := m.Fetch(ctx, imagerequest.Source{Host: testHost, Path: "/photo.jpg"}) _, err := m.Fetch(ctx, "https://example.com/photo.jpg")
if !errors.Is(err, context.Canceled) { if !errors.Is(err, context.Canceled) {
t.Errorf("Fetch() error = %v, want context.Canceled", err) t.Errorf("Fetch() error = %v, want context.Canceled", err)
} }
@@ -272,13 +294,13 @@ func TestDetectContentTypeFromPath(t *testing.T) {
path string path string
want string want string
}{ }{
{"foo/bar.jpg", imagerequest.FormatJPEG.MIMEType()}, {"foo/bar.jpg", contentTypeJPEG},
{"foo/bar.JPG", imagerequest.FormatJPEG.MIMEType()}, {"foo/bar.JPG", contentTypeJPEG},
{"foo/bar.jpeg", imagerequest.FormatJPEG.MIMEType()}, {"foo/bar.jpeg", contentTypeJPEG},
{"foo/bar.png", imagerequest.FormatPNG.MIMEType()}, {"foo/bar.png", contentTypePNG},
{"foo/bar.gif", imagerequest.FormatGIF.MIMEType()}, {"foo/bar.gif", contentTypeGIF},
{"foo/bar.webp", imagerequest.FormatWebP.MIMEType()}, {"foo/bar.webp", contentTypeWebP},
{"foo/bar.avif", imagerequest.FormatAVIF.MIMEType()}, {"foo/bar.avif", contentTypeAVIF},
{"foo/bar.svg", contentTypeSVG}, {"foo/bar.svg", contentTypeSVG},
{"foo/bar.bin", contentTypeOctetStream}, {"foo/bar.bin", contentTypeOctetStream},
{"foo/bar", contentTypeOctetStream}, {"foo/bar", contentTypeOctetStream},
+4 -14
View File
@@ -1,26 +1,16 @@
package httpfetcher package httpfetcher
import ( import "testing"
"testing"
"time"
)
// TestJPEGXLContentType verifies that the fetcher accepts an upstream answer // TestJPEGXLContentType verifies that the fetcher accepts an upstream answer
// of type image/jxl, and that the mock fetcher serves a .jxl file as // of type image/jxl by default, and that the mock fetcher serves a .jxl file
// image/jxl. // as image/jxl.
func TestJPEGXLContentType(t *testing.T) { func TestJPEGXLContentType(t *testing.T) {
t.Parallel() t.Parallel()
const jxlType = "image/jxl" const jxlType = "image/jxl"
f := New(&Config{ if !New(DefaultConfig()).isAllowedContentType(jxlType) {
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
})
if !f.isAllowedContentType(jxlType) {
t.Errorf("isAllowedContentType(%q) = false, want true", jxlType) t.Errorf("isAllowedContentType(%q) = false, want true", jxlType)
} }
@@ -8,20 +8,22 @@ import (
"sync" "sync"
"testing" "testing"
"time" "time"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// imageSourceOn is the fake upstream's image route on host. // imageURLOnPort is the fake upstream's image route on testPublicHost at
func imageSourceOn(host string) imagerequest.Source {
return imagerequest.Source{Host: host, Path: "/image"}
}
// imageSourceOnPort is the fake upstream's image route on testPublicHost at
// port. Each port is a different host to the per-host limit, while the test // port. Each port is a different host to the per-host limit, while the test
// dialer sends every port to the one test server. // dialer sends every port to the one test server.
func imageSourceOnPort(port int) imagerequest.Source { func imageURLOnPort(port int) string {
return imageSourceOn(net.JoinHostPort(testPublicHost, strconv.Itoa(port))) return "http://" + net.JoinHostPort(testPublicHost, strconv.Itoa(port)) +
"/image"
}
func TestDefaultConfigMaxConnections(t *testing.T) {
t.Parallel()
if got := DefaultConfig().MaxConnections; got != DefaultMaxConnections {
t.Errorf("MaxConnections = %d, want %d", got, DefaultMaxConnections)
}
} }
// TestFetchLimitsConnectionsToAllHostsTogether checks that MaxConnections // TestFetchLimitsConnectionsToAllHostsTogether checks that MaxConnections
@@ -35,22 +37,18 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxConnections = 2
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 2,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
f.connectionWaitTimeout = 100 * time.Millisecond f.connectionWaitTimeout = 100 * time.Millisecond
first, err := f.Fetch(testContext(t), imageSourceOnPort(81)) first, err := f.Fetch(testContext(t), imageURLOnPort(81))
if err != nil { if err != nil {
t.Fatalf("first Fetch() error = %v", err) t.Fatalf("first Fetch() error = %v", err)
} }
second, err := f.Fetch(testContext(t), imageSourceOnPort(82)) second, err := f.Fetch(testContext(t), imageURLOnPort(82))
if err != nil { if err != nil {
t.Fatalf("second Fetch() error = %v", err) t.Fatalf("second Fetch() error = %v", err)
} }
@@ -59,7 +57,7 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) {
start := time.Now() start := time.Now()
_, err = f.Fetch(testContext(t), imageSourceOnPort(83)) _, err = f.Fetch(testContext(t), imageURLOnPort(83))
if !errors.Is(err, ErrTooManyConnections) { if !errors.Is(err, ErrTooManyConnections) {
t.Fatalf("third Fetch() error = %v, want ErrTooManyConnections", err) t.Fatalf("third Fetch() error = %v, want ErrTooManyConnections", err)
} }
@@ -78,7 +76,7 @@ func TestFetchLimitsConnectionsToAllHostsTogether(t *testing.T) {
t.Fatalf("close first body: %v", err) t.Fatalf("close first body: %v", err)
} }
third, err := f.Fetch(testContext(t), imageSourceOnPort(83)) third, err := f.Fetch(testContext(t), imageURLOnPort(83))
if err != nil { if err != nil {
t.Fatalf("Fetch() after a response was closed: error = %v", err) t.Fatalf("Fetch() after a response was closed: error = %v", err)
} }
@@ -96,16 +94,12 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxConnections = 1
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 1,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
first, err := f.Fetch(testContext(t), imageSourceOnPort(81)) first, err := f.Fetch(testContext(t), imageURLOnPort(81))
if err != nil { if err != nil {
t.Fatalf("first Fetch() error = %v", err) t.Fatalf("first Fetch() error = %v", err)
} }
@@ -115,7 +109,7 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel() defer cancel()
_, err = f.Fetch(ctx, imageSourceOnPort(82)) _, err = f.Fetch(ctx, imageURLOnPort(82))
if !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("second Fetch() error = %v, want context.DeadlineExceeded", err) t.Fatalf("second Fetch() error = %v, want context.DeadlineExceeded", err)
} }
@@ -140,7 +134,7 @@ func TestFetchRemovesIdleHostSemaphores(t *testing.T) {
for i := range 100 { for i := range 100 {
wg.Go(func() { wg.Go(func() {
res, err := f.Fetch(ctx, imageSourceOnPort(1+i%50)) res, err := f.Fetch(ctx, imageURLOnPort(1+i%50))
if err != nil { if err != nil {
t.Errorf("Fetch() error = %v", err) t.Errorf("Fetch() error = %v", err)
@@ -169,22 +163,19 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxConnections = 1
MaxResponseSize: 50 << 20, cfg.MaxConnectionsPerHost = 1
MaxConnectionsPerHost: 1,
MaxConnections: 1,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
f.connectionWaitTimeout = 100 * time.Millisecond f.connectionWaitTimeout = 100 * time.Millisecond
open, err := f.Fetch(testContext(t), imageSourceOnPort(81)) open, err := f.Fetch(testContext(t), imageURLOnPort(81))
if err != nil { if err != nil {
t.Fatalf("first Fetch() error = %v", err) t.Fatalf("first Fetch() error = %v", err)
} }
_, err = f.Fetch(testContext(t), imageSourceOnPort(82)) _, err = f.Fetch(testContext(t), imageURLOnPort(82))
if !errors.Is(err, ErrTooManyConnections) { if !errors.Is(err, ErrTooManyConnections) {
t.Fatalf("Fetch() from another host: error = %v, "+ t.Fatalf("Fetch() from another host: error = %v, "+
"want ErrTooManyConnections", err) "want ErrTooManyConnections", err)
@@ -193,7 +184,7 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel() defer cancel()
_, err = f.Fetch(ctx, imageSourceOnPort(81)) _, err = f.Fetch(ctx, imageURLOnPort(81))
if !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Fetch() from the busy host: error = %v, "+ t.Fatalf("Fetch() from the busy host: error = %v, "+
"want context.DeadlineExceeded", err) "want context.DeadlineExceeded", err)
@@ -205,7 +196,7 @@ func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) {
ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond) ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel() defer cancel()
_, err = f.Fetch(ctx, imageSourceOnPort(83)) _, err = f.Fetch(ctx, imageURLOnPort(83))
if !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Fetch() from a host with nothing open: error = %v, "+ t.Fatalf("Fetch() from a host with nothing open: error = %v, "+
"want context.DeadlineExceeded", err) "want context.DeadlineExceeded", err)
@@ -229,14 +220,14 @@ func TestFetchReleasesConnectionOnError(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
source imagerequest.Source url string
want error want error
}{ }{
{"upstream answers 500", upstreamSource("/status/500"), ErrUpstreamError}, {"upstream answers 500", upstreamURL("/status/500"), ErrUpstreamError},
{"upstream sends HTML", upstreamSource("/html"), ErrInvalidContentType}, {"upstream sends HTML", upstreamURL("/html"), ErrInvalidContentType},
// 198.51.100.7 (TEST-NET-2) passes the SSRF checks, and the test // 198.51.100.7 (TEST-NET-2) passes the SSRF checks, and the test
// dialer refuses every host but testPublicHost. // dialer refuses every host but testPublicHost.
{"connecting fails", imageSourceOn("198.51.100.7"), errUnexpectedDial}, {"connecting fails", "http://198.51.100.7/image", errUnexpectedDial},
} }
for _, tc := range cases { for _, tc := range cases {
@@ -245,17 +236,13 @@ func TestFetchReleasesConnectionOnError(t *testing.T) {
srv := startUpstream(t) srv := startUpstream(t)
cfg := &Config{ cfg := DefaultConfig()
Timeout: 30 * time.Second, cfg.MaxConnections = 1
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 1,
}
f, _ := newServerFetcher(t, srv, cfg) f, _ := newServerFetcher(t, srv, cfg)
f.connectionWaitTimeout = 100 * time.Millisecond f.connectionWaitTimeout = 100 * time.Millisecond
_, err := f.Fetch(testContext(t), tc.source) _, err := f.Fetch(testContext(t), tc.url)
if !errors.Is(err, tc.want) { if !errors.Is(err, tc.want) {
t.Fatalf("Fetch() error = %v, want %v", err, tc.want) t.Fatalf("Fetch() error = %v, want %v", err, tc.want)
} }
+42 -20
View File
@@ -7,14 +7,14 @@ import (
"io/fs" "io/fs"
"net/http" "net/http"
"strings" "strings"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// errEmptyURLPath is returned when a mock URL has no usable path.
var errEmptyURLPath = errors.New("empty URL path")
// MockFetcher implements Fetcher using an embedded filesystem. // MockFetcher implements Fetcher using an embedded filesystem.
// Files are organized as: hostname/path/to/file.ext // Files are organized as: hostname/path/to/file.ext
// The source with host example.com and path /images/photo.jpg maps to // URLs like https://example.com/images/photo.jpg map to example.com/images/photo.jpg.
// example.com/images/photo.jpg; the source query is ignored.
type MockFetcher struct { type MockFetcher struct {
fs fs.FS fs fs.FS
} }
@@ -24,10 +24,8 @@ func NewMock(fsys fs.FS) *MockFetcher {
return &MockFetcher{fs: fsys} return &MockFetcher{fs: fsys}
} }
// Fetch retrieves the source's content from the mock filesystem. // Fetch retrieves content from the mock filesystem.
func (m *MockFetcher) Fetch( func (m *MockFetcher) Fetch(ctx context.Context, url string) (*FetchResult, error) {
ctx context.Context, source imagerequest.Source,
) (*FetchResult, error) {
// Check context cancellation // Check context cancellation
select { select {
case <-ctx.Done(): case <-ctx.Done():
@@ -35,7 +33,11 @@ func (m *MockFetcher) Fetch(
default: default:
} }
path := source.Host + source.Path // Parse URL to get filesystem path
path, err := urlToFSPath(url)
if err != nil {
return nil, err
}
// Open the file // Open the file
f, err := m.fs.Open(path) f, err := m.fs.Open(path)
@@ -66,11 +68,31 @@ func (m *MockFetcher) Fetch(
}, nil }, nil
} }
// The content types the mock fetcher gives files in no format pixa reads. // urlToFSPath converts a URL to a filesystem path.
const ( // https://example.com/images/photo.jpg -> example.com/images/photo.jpg
contentTypeSVG = "image/svg+xml" func urlToFSPath(rawURL string) (string, error) {
contentTypeOctetStream = "application/octet-stream" // Strip scheme
) url := rawURL
if idx := strings.Index(url, "://"); idx != -1 {
url = url[idx+3:]
}
// Remove query string
if idx := strings.Index(url, "?"); idx != -1 {
url = url[:idx]
}
// Remove fragment
if idx := strings.Index(url, "#"); idx != -1 {
url = url[:idx]
}
if url == "" {
return "", errEmptyURLPath
}
return url, nil
}
// detectContentTypeFromPath returns the MIME type based on file extension. // detectContentTypeFromPath returns the MIME type based on file extension.
func detectContentTypeFromPath(path string) string { func detectContentTypeFromPath(path string) string {
@@ -78,17 +100,17 @@ func detectContentTypeFromPath(path string) string {
switch { switch {
case strings.HasSuffix(path, ".jpg"), strings.HasSuffix(path, ".jpeg"): case strings.HasSuffix(path, ".jpg"), strings.HasSuffix(path, ".jpeg"):
return imagerequest.FormatJPEG.MIMEType() return contentTypeJPEG
case strings.HasSuffix(path, ".png"): case strings.HasSuffix(path, ".png"):
return imagerequest.FormatPNG.MIMEType() return contentTypePNG
case strings.HasSuffix(path, ".gif"): case strings.HasSuffix(path, ".gif"):
return imagerequest.FormatGIF.MIMEType() return contentTypeGIF
case strings.HasSuffix(path, ".webp"): case strings.HasSuffix(path, ".webp"):
return imagerequest.FormatWebP.MIMEType() return contentTypeWebP
case strings.HasSuffix(path, ".avif"): case strings.HasSuffix(path, ".avif"):
return imagerequest.FormatAVIF.MIMEType() return contentTypeAVIF
case strings.HasSuffix(path, ".jxl"): case strings.HasSuffix(path, ".jxl"):
return imagerequest.FormatJXL.MIMEType() return contentTypeJXL
case strings.HasSuffix(path, ".svg"): case strings.HasSuffix(path, ".svg"):
return contentTypeSVG return contentTypeSVG
default: default:
@@ -8,7 +8,6 @@ import (
"testing" "testing"
"github.com/go-chi/chi/v5/middleware" "github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestFetchSendsRequestID verifies that a fetch sends the ID of the request // TestFetchSendsRequestID verifies that a fetch sends the ID of the request
@@ -26,7 +25,7 @@ func TestFetchSendsRequestID(t *testing.T) {
func(w http.ResponseWriter, r *http.Request) { func(w http.ResponseWriter, r *http.Request) {
received <- r.Header.Get("X-Request-Id") received <- r.Header.Get("X-Request-Id")
w.Header().Set("Content-Type", imagerequest.FormatJPEG.MIMEType()) w.Header().Set("Content-Type", contentTypeJPEG)
_, _ = io.WriteString(w, imagePayload) _, _ = io.WriteString(w, imagePayload)
})) }))
t.Cleanup(srv.Close) t.Cleanup(srv.Close)
@@ -35,7 +34,7 @@ func TestFetchSendsRequestID(t *testing.T) {
ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID) ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID)
res, err := f.Fetch(ctx, upstreamSource("/image")) res, err := f.Fetch(ctx, upstreamURL("/image"))
if err != nil { if err != nil {
t.Fatalf("Fetch() error = %v", err) t.Fatalf("Fetch() error = %v", err)
} }
@@ -1,98 +0,0 @@
package imageprocessor
import (
"bytes"
"encoding/binary"
"errors"
"os"
"testing"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// TestImageProcessor_RefusesAVIFAboveMaxPixels verifies that an AVIF output
// of more than MaxAVIFOutputPixels is refused with ErrAVIFOutputTooLarge
// before anything is decoded, whatever size and fit give it, the source's own
// size included, and that an AVIF output of at most that many pixels, or one
// in another format, is not. One input is a 64x32 PNG cut off in its pixel
// data. The other is testdata/red.avif, a 4x4 AVIF, with the size in its
// header changed to 4097x4096. libvips reads each input's size from its
// header but fails when it decodes its pixel data, so a request that gets
// that failure was not refused, and a refusal came before any decoding.
func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) {
t.Parallel()
whole := createTestPNG(t, 64, 32)
cutOff := whole[:len(whole)/2]
largeAVIF, err := os.ReadFile("testdata/red.avif")
if err != nil {
t.Fatalf("failed to read test AVIF: %v", err)
}
// The ispe box gives the width and then the height, 4 bytes each, after
// the box's type and its 4 bytes of version and flags
ispe := bytes.Index(largeAVIF, []byte("ispe"))
binary.BigEndian.PutUint32(largeAVIF[ispe+8:], 4097)
binary.BigEndian.PutUint32(largeAVIF[ispe+12:], 4096)
tests := []struct {
name string
input []byte
sourceFormat imagerequest.Format
format imagerequest.Format
width, height int
fit imagerequest.Fit
refused bool
}{
{"AVIF at the limit", cutOff, imagerequest.FormatPNG,
imagerequest.FormatAVIF, 4096, 4096, imagerequest.FitCover, false},
{"AVIF above the limit", cutOff, imagerequest.FormatPNG,
imagerequest.FormatAVIF, 4096, 4097, imagerequest.FitCover, true},
{"WebP above the AVIF limit", cutOff, imagerequest.FormatPNG,
imagerequest.FormatWebP, 4096, 4097, imagerequest.FitCover, false},
// FitInside does not enlarge, so the output stays 64x32
{"AVIF inside a larger size", cutOff, imagerequest.FormatPNG,
imagerequest.FormatAVIF, 8192, 8192, imagerequest.FitInside, false},
// FitOutside scales 64x32 by 128 to cover 8192x1: 8192x4096
{"AVIF outside a smaller size", cutOff, imagerequest.FormatPNG,
imagerequest.FormatAVIF, 8192, 1, imagerequest.FitOutside, true},
// With no size, the output keeps the source's 4097x4096
{"AVIF of the source's size", largeAVIF, imagerequest.FormatAVIF,
imagerequest.FormatAVIF, 0, 0, imagerequest.FitCover, true},
{"orig of an AVIF source", largeAVIF, imagerequest.FormatAVIF,
imagerequest.FormatOriginal, 0, 0, imagerequest.FitCover, true},
{"JPEG of the source's size", largeAVIF, imagerequest.FormatAVIF,
imagerequest.FormatJPEG, 0, 0, imagerequest.FitCover, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
_, err := New(Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(), bytes.NewReader(tt.input), tt.sourceFormat,
&imagerequest.Transform{
Size: imagerequest.Size{Width: tt.width, Height: tt.height},
Format: tt.format,
FitMode: tt.fit,
})
t.Logf("Process() error = %v", err)
if tt.refused {
if !errors.Is(err, ErrAVIFOutputTooLarge) {
t.Errorf("Process() error = %v, want %v", err, ErrAVIFOutputTooLarge)
}
return
}
if err == nil || errors.Is(err, ErrAVIFOutputTooLarge) {
t.Errorf("Process() error = %v, want the failure to decode "+
"the input", err)
}
})
}
}
@@ -4,8 +4,6 @@ import (
"bytes" "bytes"
"errors" "errors"
"testing" "testing"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestImageProcessor_RefusesComputedSideAboveMaximum verifies that a size // TestImageProcessor_RefusesComputedSideAboveMaximum verifies that a size
@@ -19,17 +17,13 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
inputWidth, inputHeight int inputWidth, inputHeight int
size imagerequest.Size size Size
wantErr error wantErr error
}{ }{
{"width at the maximum", 4096, 1, {"width at the maximum", 4096, 1, Size{Height: 2}, nil},
imagerequest.Size{Height: 2}, nil}, {"width above the maximum", 4096, 1, Size{Height: 3}, ErrOutputTooLarge},
{"width above the maximum", 4096, 1, {"height at the maximum", 1, 4096, Size{Width: 2}, nil},
imagerequest.Size{Height: 3}, ErrOutputTooLarge}, {"height above the maximum", 1, 4096, Size{Width: 3}, ErrOutputTooLarge},
{"height at the maximum", 1, 4096,
imagerequest.Size{Width: 2}, nil},
{"height above the maximum", 1, 4096,
imagerequest.Size{Width: 3}, ErrOutputTooLarge},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -38,16 +32,9 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) {
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
result, err := New(Params{ result, err := New(Params{}).Process(t.Context(),
MaxInputBytes: 50 << 20, bytes.NewReader(input),
MaxConcurrentProcessing: 1, &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitCover})
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatJPEG,
&imagerequest.Transform{
Size: tt.size,
Format: imagerequest.FormatJPEG,
FitMode: imagerequest.FitCover,
})
if result != nil { if result != nil {
t.Logf("Process() made %dx%d", result.Width, result.Height) t.Logf("Process() made %dx%d", result.Width, result.Height)
@@ -63,79 +50,3 @@ func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) {
}) })
} }
} }
// TestImageProcessor_ComputedSideIsAtLeastOne verifies that, for a size with
// one side 0, the side computed from the input's proportions is 1 where it
// would round down to 0, and that Process makes that size in every fit mode
// but FitOutside, which scales the input to cover the size and so can make
// it longer. Each case checks the size targetDimensions gives before it
// calls Process: with that side at 0, an 8192x1 input at 1300x0 makes
// libvips ask for about 39 GB with FitCover.
func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) {
t.Parallel()
tests := []struct {
name string
inputWidth, inputHeight int
size imagerequest.Size
wantWidth, wantHeight int
}{
{"8192x1 at 100x0", 8192, 1, imagerequest.Size{Width: 100}, 100, 1},
{"8192x1 at 1300x0", 8192, 1, imagerequest.Size{Width: 1300}, 1300, 1},
{"8192x2 at 100x0", 8192, 2, imagerequest.Size{Width: 100}, 100, 1},
{"1x8192 at 0x100", 1, 8192, imagerequest.Size{Height: 100}, 1, 100},
{"1x8192 at 0x1300", 1, 8192, imagerequest.Size{Height: 1300}, 1, 1300},
{"2x8192 at 0x100", 2, 8192, imagerequest.Size{Height: 100}, 1, 100},
}
fits := []imagerequest.Fit{
imagerequest.FitCover, imagerequest.FitContain, imagerequest.FitFill,
imagerequest.FitInside, imagerequest.FitOutside,
}
for _, tt := range tests {
for _, fit := range fits {
t.Run(tt.name+" "+string(fit), func(t *testing.T) {
t.Parallel()
width, height, err := targetDimensions(tt.size,
tt.inputWidth, tt.inputHeight)
if err != nil || width != tt.wantWidth || height != tt.wantHeight {
t.Fatalf("targetDimensions() = %dx%d, %v, want %dx%d",
width, height, err, tt.wantWidth, tt.wantHeight)
}
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
result, err := New(Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatJPEG,
&imagerequest.Transform{
Size: tt.size,
Format: imagerequest.FormatJPEG,
FitMode: fit,
})
if err != nil {
t.Fatalf("Process() error = %v", err)
}
defer func() { _ = result.Content.Close() }()
t.Logf("Process() made %dx%d", result.Width, result.Height)
ok := result.Width == tt.wantWidth && result.Height == tt.wantHeight
if fit == imagerequest.FitOutside {
ok = result.Width >= tt.wantWidth && result.Height >= tt.wantHeight
}
if !ok {
t.Errorf("Process() made %dx%d, want %dx%d (at least, for %s)",
result.Width, result.Height, tt.wantWidth, tt.wantHeight,
imagerequest.FitOutside)
}
})
}
}
}
@@ -1,109 +0,0 @@
package imageprocessor
import (
"bytes"
"context"
"errors"
"io"
"testing"
"time"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// readerAfterDeadline is a Process input whose first Read waits until done
// is closed, so the deadline passes after Process has taken its slot and
// before it encodes.
type readerAfterDeadline struct {
data *bytes.Reader
done <-chan struct{}
waited bool
}
func (r *readerAfterDeadline) Read(p []byte) (int, error) {
if !r.waited {
r.waited = true
<-r.done
}
return r.data.Read(p)
}
// inputNotToBeRead is a Process input that fails the test when it is read.
type inputNotToBeRead struct {
t *testing.T
}
func (r inputNotToBeRead) Read([]byte) (int, error) {
r.t.Error("Process read its input although its deadline had passed")
return 0, io.EOF
}
// TestProcessStopsWhenDeadlinePassed checks that Process returns
// context.DeadlineExceeded, and holds no slot afterwards, when its context's
// deadline has passed before it starts, then without reading its input, or
// while it reads its input. govips gives pixa no way to stop libvips once an
// encode has begun, so Process must not begin one for a request that has
// ended.
func TestProcessStopsWhenDeadlinePassed(t *testing.T) {
t.Parallel()
input := createTestJPEG(t, 50, 50)
t.Run("before Process starts", func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithDeadline(t.Context(),
time.Now().Add(-time.Second))
defer cancel()
// With a slot free, the select in acquireSlot takes the slot or sees
// the ended context at random, so Process is called many times
for range 100 {
checkProcessStopsAtDeadline(ctx, t, inputNotToBeRead{t: t})
}
})
t.Run("while Process reads its input", func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel()
checkProcessStopsAtDeadline(ctx, t, &readerAfterDeadline{
data: bytes.NewReader(input), done: ctx.Done(),
})
})
}
// checkProcessStopsAtDeadline runs Process on input with ctx, whose deadline
// has passed by the time Process could encode, and fails the test unless
// Process returns context.DeadlineExceeded and gives its slot back.
func checkProcessStopsAtDeadline(
ctx context.Context, t *testing.T, input io.Reader,
) {
t.Helper()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1})
result, err := proc.Process(ctx, input, imagerequest.FormatJPEG,
smallJPEGTransform())
if result != nil {
t.Logf("Process() made a %dx%d image after its deadline passed",
result.Width, result.Height)
_ = result.Content.Close()
}
t.Logf("Process() error = %v", err)
if !errors.Is(err, context.DeadlineExceeded) {
t.Errorf("Process() error = %v, want context.DeadlineExceeded", err)
}
if held := len(proc.processingSemaphore); held != 0 {
t.Errorf("%d slots still held after Process() returned", held)
}
}
@@ -4,8 +4,6 @@ import (
"bytes" "bytes"
"errors" "errors"
"testing" "testing"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestImageProcessor_EmptyFormatRefused verifies that a request with no format // TestImageProcessor_EmptyFormatRefused verifies that a request with no format
@@ -14,12 +12,8 @@ import (
func TestImageProcessor_EmptyFormatRefused(t *testing.T) { func TestImageProcessor_EmptyFormatRefused(t *testing.T) {
t.Parallel() t.Parallel()
_, err := New(Params{ _, err := New(Params{}).Process(
MaxInputBytes: 50 << 20, t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), &Request{},
MaxConcurrentProcessing: 1,
}).Process(
t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)),
imagerequest.FormatJPEG, &imagerequest.Transform{},
) )
if !errors.Is(err, ErrUnsupportedOutputFormat) { if !errors.Is(err, ErrUnsupportedOutputFormat) {
t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat) t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat)
@@ -8,21 +8,14 @@ import (
"testing" "testing"
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// processedSize runs input, an image in sourceFormat, through Process and // processedSize runs input through Process and returns the output's size in
// returns the output's size in bytes. // bytes.
func processedSize( func processedSize(t *testing.T, input []byte, req *Request) int64 {
t *testing.T, input []byte, sourceFormat imagerequest.Format,
transform *imagerequest.Transform,
) int64 {
t.Helper() t.Helper()
result, err := New(Params{ result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), req)
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(), bytes.NewReader(input), sourceFormat, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
} }
@@ -75,8 +68,7 @@ func TestImageProcessor_PNGIsCompressed(t *testing.T) {
} }
} }
size := processedSize(t, encodePNG(t, flat), imagerequest.FormatPNG, size := processedSize(t, encodePNG(t, flat), &Request{Format: FormatPNG})
&imagerequest.Transform{Format: imagerequest.FormatPNG})
const rawBytes = width * height * 3 const rawBytes = width * height * 3
if size > rawBytes/10 { if size > rawBytes/10 {
@@ -85,28 +77,27 @@ func TestImageProcessor_PNGIsCompressed(t *testing.T) {
} }
} }
// TestImageProcessor_WebPAtEffort2 verifies that WebP is saved at effort 2, // TestImageProcessor_WebPAtDefaultEffort verifies that WebP is saved at
// not libvips' default of 4: the output is the size of the same image saved // libvips' default effort, 4: the output is the size of the same image saved
// at effort 2. // at that effort.
func TestImageProcessor_WebPAtEffort2(t *testing.T) { func TestImageProcessor_WebPAtDefaultEffort(t *testing.T) {
t.Parallel() t.Parallel()
input := createTestJPEG(t, 200, 150) input := createTestJPEG(t, 200, 150)
size := processedSize(t, input, imagerequest.FormatJPEG, size := processedSize(t, input, &Request{Format: FormatWebP, Quality: 85})
&imagerequest.Transform{Format: imagerequest.FormatWebP, Quality: 85})
want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{ want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{
StripMetadata: true, StripMetadata: true,
Quality: 85, Quality: 85,
ReductionEffort: 2, ReductionEffort: 4,
}) })
if err != nil { if err != nil {
t.Fatalf("ExportWebp() error = %v", err) t.Fatalf("ExportWebp() error = %v", err)
} }
if size != int64(len(want)) { if size != int64(len(want)) {
t.Errorf("WebP output is %d bytes, want %d, the size at effort 2", t.Errorf("WebP output is %d bytes, want %d, the size at effort 4",
size, len(want)) size, len(want))
} }
} }
@@ -135,8 +126,7 @@ func TestImageProcessor_AVIFAtEffort1(t *testing.T) {
input := encodePNG(t, source) input := encodePNG(t, source)
size := processedSize(t, input, imagerequest.FormatPNG, size := processedSize(t, input, &Request{Format: FormatAVIF, Quality: 85})
&imagerequest.Transform{Format: imagerequest.FormatAVIF, Quality: 85})
want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{ want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{
StripMetadata: true, StripMetadata: true,
@@ -153,30 +143,3 @@ func TestImageProcessor_AVIFAtEffort1(t *testing.T) {
"and 8 bits", size, len(want)) "and 8 bits", size, len(want))
} }
} }
// TestImageProcessor_JPEGXLAtEffort4 verifies that JPEG XL is saved at
// effort 4, not libvips' default of 7: the output is the size of the same
// image saved at effort 4, with the distance exportJXL gives quality 85.
func TestImageProcessor_JPEGXLAtEffort4(t *testing.T) {
t.Parallel()
input := createTestJPEG(t, 200, 150)
size := processedSize(t, input, imagerequest.FormatJPEG,
&imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: 85})
params := vips.NewJxlExportParams()
params.Quality = 85
params.Distance = jxlDistance(85)
params.Effort = 4
want, _, err := decode(t, input).ExportJxl(params)
if err != nil {
t.Fatalf("ExportJxl() error = %v", err)
}
if size != int64(len(want)) {
t.Errorf("JPEG XL output is %d bytes, want %d, the size at effort 4",
size, len(want))
}
}
@@ -1,287 +0,0 @@
package imageprocessor
import (
"bytes"
"image"
"image/color"
"image/png"
"math"
"testing"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// TestImageProcessor_FitCoverServesThinSource verifies that FitCover serves
// a thin source at the requested size. An 8192x1 source scaled to cover
// 8192x8192 is 8192 times as long as the output on its long side. The centre
// of an 8192x1 source with the proportions of 1x8192 is 1/8192 of a pixel
// wide. Each case is also turned on its side.
func TestImageProcessor_FitCoverServesThinSource(t *testing.T) {
t.Parallel()
tests := []struct {
name string
inputWidth, inputHeight int
size imagerequest.Size
}{
{"8192x1 at 8192x8192", 8192, 1, imagerequest.Size{Width: 8192, Height: 8192}},
{"8192x1 at 1x8192", 8192, 1, imagerequest.Size{Width: 1, Height: 8192}},
{"1x8192 at 8192x8192", 1, 8192, imagerequest.Size{Width: 8192, Height: 8192}},
{"1x8192 at 8192x1", 1, 8192, imagerequest.Size{Width: 8192, Height: 1}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
result, err := New(Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatJPEG,
&imagerequest.Transform{
Size: tt.size,
Format: imagerequest.FormatJPEG,
FitMode: imagerequest.FitCover,
})
if err != nil {
t.Fatalf("Process() error = %v", err)
}
defer func() { _ = result.Content.Close() }()
t.Logf("Process() made %dx%d", result.Width, result.Height)
if result.Width != tt.size.Width || result.Height != tt.size.Height {
t.Errorf("Process() made %dx%d, want %dx%d",
result.Width, result.Height, tt.size.Width, tt.size.Height)
}
})
}
}
// createBandsPNG creates a PNG of three equal bands, red, green and blue,
// side by side when it is wider than tall and top to bottom otherwise.
func createBandsPNG(t *testing.T, width, height int) []byte {
t.Helper()
bands := []color.RGBA{
{R: 255, A: 255},
{G: 255, A: 255},
{B: 255, A: 255},
}
img := image.NewRGBA(image.Rect(0, 0, width, height))
for y := range height {
for x := range width {
band := 3 * y / height
if width > height {
band = 3 * x / width
}
img.Set(x, y, bands[band])
}
}
return encodePNG(t, img)
}
// TestImageProcessor_FitCoverCropsTheCentre verifies that FitCover keeps the
// centre of the source: a 300x100 source of red, green and blue bands side
// by side, or a 100x300 one with them top to bottom, comes out at 50x50 as
// its green band.
func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) {
t.Parallel()
tests := []struct {
name string
inputWidth, inputHeight int
}{
{"bands side by side", 300, 100},
{"bands top to bottom", 100, 300},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
input := createBandsPNG(t, tt.inputWidth, tt.inputHeight)
result, err := New(Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatPNG,
&imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatPNG,
FitMode: imagerequest.FitCover,
})
if err != nil {
t.Fatalf("Process() error = %v", err)
}
defer func() { _ = result.Content.Close() }()
output, err := png.Decode(result.Content)
if err != nil {
t.Fatalf("failed to decode output: %v", err)
}
bounds := output.Bounds()
t.Logf("Process() made %dx%d", bounds.Dx(), bounds.Dy())
if bounds.Dx() != 50 || bounds.Dy() != 50 {
t.Fatalf("Process() made %dx%d, want 50x50",
bounds.Dx(), bounds.Dy())
}
// Resampling may blend a little of the bands beside the
// green one into the output's edges.
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
r, g, b, _ := output.At(x, y).RGBA()
if r>>8 > 64 || g>>8 < 192 || b>>8 > 64 {
t.Fatalf("output pixel %d,%d is %d,%d,%d, want green",
x, y, r>>8, g>>8, b>>8)
}
}
}
})
}
}
// The greys of createSquarePNG, and the grey halfway between them.
const (
backgroundGrey = 192
squareGrey = 64
halfwayGrey = (backgroundGrey + squareGrey) / 2
)
// createSquarePNG creates a grey PNG side pixels square with a darker grey
// square squareSide pixels wide in its centre.
func createSquarePNG(t *testing.T, side, squareSide int) []byte {
t.Helper()
first := (side - squareSide) / 2
last := first + squareSide - 1
img := image.NewGray(image.Rect(0, 0, side, side))
for y := range side {
for x := range side {
grey := uint8(backgroundGrey)
if x >= first && x <= last && y >= first && y <= last {
grey = squareGrey
}
img.SetGray(x, y, color.Gray{Y: grey})
}
}
return encodePNG(t, img)
}
// squareLength returns how long the square is along greys, the greys of a
// row or column of the output that crosses it: the distance between the
// first and the last point where greys passes halfwayGrey, or 0 when there
// is no such point. Resampling blurs the square's edges over several
// pixels, and each point is placed between the two pixels on either side
// of it in proportion to their greys, so the length is measured to a
// fraction of a pixel.
func squareLength(greys []float64) float64 {
first, last := -1.0, -1.0
for i := 1; i < len(greys); i++ {
before, after := greys[i-1], greys[i]
if (before > halfwayGrey) == (after > halfwayGrey) {
continue
}
point := float64(i-1) + (before-halfwayGrey)/(before-after)
if first < 0 {
first = point
}
last = point
}
return last - first
}
// TestImageProcessor_FitCoverKeepsProportions verifies that FitCover scales
// both sides of the source by the same amount: a square in the centre of
// the source comes out square. Its width and height are measured along the
// output's middle row and column.
func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) {
t.Parallel()
tests := []struct {
name string
side, squareSide int
size imagerequest.Size
}{
{"16x16 at 100x60", 16, 8, imagerequest.Size{Width: 100, Height: 60}},
{"16x16 at 60x100", 16, 8, imagerequest.Size{Width: 60, Height: 100}},
{"10x10 at 100x35", 10, 2, imagerequest.Size{Width: 100, Height: 35}},
{"10x10 at 35x100", 10, 2, imagerequest.Size{Width: 35, Height: 100}},
{"64x64 at 300x100", 64, 16, imagerequest.Size{Width: 300, Height: 100}},
{"64x64 at 100x300", 64, 16, imagerequest.Size{Width: 100, Height: 300}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
input := createSquarePNG(t, tt.side, tt.squareSide)
result, err := New(Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatPNG,
&imagerequest.Transform{
Size: tt.size,
Format: imagerequest.FormatPNG,
FitMode: imagerequest.FitCover,
})
if err != nil {
t.Fatalf("Process() error = %v", err)
}
defer func() { _ = result.Content.Close() }()
output, err := png.Decode(result.Content)
if err != nil {
t.Fatalf("failed to decode output: %v", err)
}
bounds := output.Bounds()
if bounds.Dx() != tt.size.Width || bounds.Dy() != tt.size.Height {
t.Fatalf("Process() made %dx%d, want %dx%d",
bounds.Dx(), bounds.Dy(), tt.size.Width, tt.size.Height)
}
row := make([]float64, bounds.Dx())
for x := range row {
r, _, _, _ := output.At(x, bounds.Dy()/2).RGBA()
row[x] = float64(r >> 8)
}
column := make([]float64, bounds.Dy())
for y := range column {
r, _, _, _ := output.At(bounds.Dx()/2, y).RGBA()
column[y] = float64(r >> 8)
}
width, height := squareLength(row), squareLength(column)
t.Logf("square is %.2f wide and %.2f high", width, height)
if width == 0 || math.Abs(width-height) > 0.5 {
t.Errorf("square is %.2f wide and %.2f high, want it square",
width, height)
}
})
}
}
@@ -4,8 +4,6 @@ import (
"bytes" "bytes"
"errors" "errors"
"testing" "testing"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestImageProcessor_RefusesFitOutsideAboveMaximum verifies that FitOutside, // TestImageProcessor_RefusesFitOutsideAboveMaximum verifies that FitOutside,
@@ -19,20 +17,15 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
inputWidth, inputHeight int inputWidth, inputHeight int
size imagerequest.Size size Size
// wantWidth and wantHeight are the output's, 0 for a refusal // wantWidth and wantHeight are the output's, 0 for a refusal
wantWidth, wantHeight int wantWidth, wantHeight int
}{ }{
{"width above the maximum", 3000, 600, {"width above the maximum", 3000, 600, Size{Width: 1, Height: 8192}, 0, 0},
imagerequest.Size{Width: 1, Height: 8192}, 0, 0}, {"height above the maximum", 600, 3000, Size{Width: 8192, Height: 1}, 0, 0},
{"height above the maximum", 600, 3000, {"width at the maximum", 1024, 256, Size{Width: 1, Height: 2048}, 8192, 2048},
imagerequest.Size{Width: 8192, Height: 1}, 0, 0}, {"height at the maximum", 256, 1024, Size{Width: 2048, Height: 1}, 2048, 8192},
{"width at the maximum", 1024, 256, {"within the maximum", 3000, 600, Size{Width: 1, Height: 1200}, 6000, 1200},
imagerequest.Size{Width: 1, Height: 2048}, 8192, 2048},
{"height at the maximum", 256, 1024,
imagerequest.Size{Width: 2048, Height: 1}, 2048, 8192},
{"within the maximum", 3000, 600,
imagerequest.Size{Width: 1, Height: 1200}, 6000, 1200},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -41,16 +34,9 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) {
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight) input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
result, err := New(Params{ result, err := New(Params{}).Process(t.Context(),
MaxInputBytes: 50 << 20, bytes.NewReader(input),
MaxConcurrentProcessing: 1, &Request{Size: tt.size, Format: FormatJPEG, FitMode: FitOutside})
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatJPEG,
&imagerequest.Transform{
Size: tt.size,
Format: imagerequest.FormatJPEG,
FitMode: imagerequest.FitOutside,
})
if result != nil { if result != nil {
t.Logf("Process() made %dx%d", result.Width, result.Height) t.Logf("Process() made %dx%d", result.Width, result.Height)
} }
@@ -74,52 +60,3 @@ func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) {
}) })
} }
} }
// TestImageProcessor_FitOutsideSideIsAtLeastOne verifies that FitOutside
// keeps each side at 1 or more when it works both sides out again from the
// larger scale. A side of 49 scaled to 1 is 49 times 1/49, which in floating
// point is just under 1 and so rounded down to 0, a width libvips refuses.
func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) {
t.Parallel()
tests := []struct {
name string
inputWidth, inputHeight int
size imagerequest.Size
wantWidth, wantHeight int
}{
{"49x8192 at 0x100", 49, 8192, imagerequest.Size{Height: 100}, 1, 167},
{"49x49 at 1x1", 49, 49, imagerequest.Size{Width: 1, Height: 1}, 1, 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
result, err := New(Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 1,
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatJPEG,
&imagerequest.Transform{
Size: tt.size,
Format: imagerequest.FormatJPEG,
FitMode: imagerequest.FitOutside,
})
if err != nil {
t.Fatalf("Process() error = %v", err)
}
defer func() { _ = result.Content.Close() }()
t.Logf("Process() made %dx%d", result.Width, result.Height)
if result.Width != tt.wantWidth || result.Height != tt.wantHeight {
t.Errorf("Process() made %dx%d, want %dx%d",
result.Width, result.Height, tt.wantWidth, tt.wantHeight)
}
})
}
}
+295 -300
View File
@@ -7,11 +7,11 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"runtime"
"sync" "sync"
"time" "time"
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// vipsOnce ensures vips is initialized exactly once. // vipsOnce ensures vips is initialized exactly once.
@@ -20,10 +20,11 @@ import (
var vipsOnce sync.Once var vipsOnce sync.Once
// initVips initializes libvips with quiet logging, one worker thread per // initVips initializes libvips with quiet logging, one worker thread per
// image and no operation cache. Each request decodes different source // image and no operation cache. Process already works on one image per CPU
// bytes, so the operation cache would rarely be hit and would hold memory // by default, so more threads per image would only compete for the CPUs.
// outside MaxConcurrentProcessing; repeated requests are served from // Each request decodes different source bytes, so the operation cache
// pixa's disk cache instead. // would rarely be hit and would hold memory outside MaxConcurrentProcessing;
// repeated requests are served from pixa's disk cache instead.
func initVips() { func initVips() {
vipsOnce.Do(func() { vipsOnce.Do(func() {
vips.LoggingSettings(nil, vips.LogLevelError) vips.LoggingSettings(nil, vips.LogLevelError)
@@ -54,6 +55,49 @@ func CheckJPEGXLSupport() error {
return nil return nil
} }
// Format represents supported output image formats.
type Format string
// Supported image output formats.
const (
FormatOriginal Format = "orig"
FormatJPEG Format = "jpeg"
FormatPNG Format = "png"
FormatWebP Format = "webp"
FormatAVIF Format = "avif"
FormatJXL Format = "jxl"
FormatGIF Format = "gif"
)
// FitMode represents how to fit an image into requested dimensions.
type FitMode string
// Supported image fit modes.
const (
FitCover FitMode = "cover"
FitContain FitMode = "contain"
FitFill FitMode = "fill"
FitInside FitMode = "inside"
FitOutside FitMode = "outside"
)
// ErrInvalidFitMode is returned when an invalid fit mode is provided.
var ErrInvalidFitMode = errors.New("invalid fit mode")
// Size represents requested image dimensions.
type Size struct {
Width int
Height int
}
// Request holds the parameters for image processing.
type Request struct {
Size Size
Format Format
Quality int
FitMode FitMode
}
// Result contains the output of image processing. // Result contains the output of image processing.
type Result struct { type Result struct {
// Content is the processed image data. // Content is the processed image data.
@@ -70,12 +114,18 @@ type Result struct {
InputWidth int InputWidth int
// InputHeight is the original image height before processing. // InputHeight is the original image height before processing.
InputHeight int InputHeight int
// InputFormat is the detected input format (e.g., "jpeg", "png").
InputFormat string
} }
// MaxInputDimension is the maximum allowed width or height for input images. // MaxInputDimension is the maximum allowed width or height for input images.
// Images larger than this are rejected to prevent DoS via decompression bombs. // Images larger than this are rejected to prevent DoS via decompression bombs.
const MaxInputDimension = 8192 const MaxInputDimension = 8192
// DefaultMaxInputBytes is the default maximum input size in bytes (50 MiB).
// This matches the default upstream fetcher limit.
const DefaultMaxInputBytes = 50 << 20
// ErrInputTooLarge is returned when input image dimensions exceed MaxInputDimension. // ErrInputTooLarge is returned when input image dimensions exceed MaxInputDimension.
var ErrInputTooLarge = errors.New("input image dimensions exceed maximum") var ErrInputTooLarge = errors.New("input image dimensions exceed maximum")
@@ -84,8 +134,8 @@ var ErrInputTooLarge = errors.New("input image dimensions exceed maximum")
var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size") var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size")
// MaxOutputDimension is the maximum width or height of an output image, and // MaxOutputDimension is the maximum width or height of an output image, and
// so of a requested size (imagerequest.MaxDimension). // so of a requested size (imgcache.MaxDimension).
const MaxOutputDimension = imagerequest.MaxDimension const MaxOutputDimension = 8192
// ErrOutputTooLarge is returned, before resizing, when the output would have a // ErrOutputTooLarge is returned, before resizing, when the output would have a
// side above MaxOutputDimension: the side of a size with one side 0, computed // side above MaxOutputDimension: the side of a size with one side 0, computed
@@ -93,28 +143,14 @@ const MaxOutputDimension = imagerequest.MaxDimension
// requested size. // requested size.
var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum") var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum")
// MaxAVIFOutputPixels is the most pixels an AVIF output may have, so that an
// encode takes at most 40 seconds of the default downstream_timeout of 60
// seconds; AVIF is already at the lowest effort govips can set (avifEffort).
// With one thread, as pixad runs libvips, saving 4096x4096, 8192x2048 or
// 2048x8192 random pixels with an alpha channel takes up to about 34
// seconds, the slowest at any quality and at 8 or 16 bits per sample, and
// 5120x4096 at 8 bits already takes about 36.
const MaxAVIFOutputPixels = 4096 * 4096
// ErrAVIFOutputTooLarge is returned, before anything is decoded, when an AVIF
// output would have more than MaxAVIFOutputPixels pixels.
var ErrAVIFOutputTooLarge = errors.New("AVIF output has too many pixels")
// ErrUnsupportedOutputFormat is returned when the requested output format is // ErrUnsupportedOutputFormat is returned when the requested output format is
// not supported. // not supported.
var ErrUnsupportedOutputFormat = errors.New("unsupported output format") var ErrUnsupportedOutputFormat = errors.New("unsupported output format")
// ErrUnsupportedInputFormat is returned, before the input is decoded, when // ErrUnsupportedInputFormat is returned, before the input is decoded, when
// libvips reads the input's first bytes as another format than its source // libvips reads the input's first bytes as another format than the source
// format, or as none. An SVG whose first bytes pass the first-bytes check for // formats, or as none: for example an SVG whose first bytes pass the
// AVIF is one example. A source format other than the six pixa reads is also // first-bytes check for AVIF.
// refused with it.
var ErrUnsupportedInputFormat = errors.New("unsupported input format") var ErrUnsupportedInputFormat = errors.New("unsupported input format")
// ErrTooManyImages is returned when MaxConcurrentProcessing images are being // ErrTooManyImages is returned when MaxConcurrentProcessing images are being
@@ -136,40 +172,47 @@ type ImageProcessor struct {
processingWaitTimeout time.Duration processingWaitTimeout time.Duration
} }
// Params holds configuration for creating an ImageProcessor. Both fields // Params holds configuration for creating an ImageProcessor.
// must be at least 1. // Zero values use sensible defaults (MaxInputBytes defaults to DefaultMaxInputBytes).
type Params struct { type Params struct {
// MaxInputBytes is the maximum allowed input size in bytes. // MaxInputBytes is the maximum allowed input size in bytes.
// If <= 0, DefaultMaxInputBytes is used.
MaxInputBytes int64 MaxInputBytes int64
// MaxConcurrentProcessing is the most images processed at once. // MaxConcurrentProcessing is the most images processed at once.
// If <= 0, the number of CPUs Go uses (runtime.GOMAXPROCS(0)) is used.
MaxConcurrentProcessing int MaxConcurrentProcessing int
} }
// New creates a new image processor with the given parameters. // New creates a new image processor with the given parameters.
// A zero-value Params{} uses sensible defaults.
func New(params Params) *ImageProcessor { func New(params Params) *ImageProcessor {
initVips() initVips()
maxInputBytes := params.MaxInputBytes
if maxInputBytes <= 0 {
maxInputBytes = DefaultMaxInputBytes
}
maxConcurrentProcessing := params.MaxConcurrentProcessing
if maxConcurrentProcessing <= 0 {
maxConcurrentProcessing = runtime.GOMAXPROCS(0)
}
return &ImageProcessor{ return &ImageProcessor{
maxInputBytes: params.MaxInputBytes, maxInputBytes: maxInputBytes,
processingSemaphore: make(chan struct{}, params.MaxConcurrentProcessing), processingSemaphore: make(chan struct{}, maxConcurrentProcessing),
processingWaitTimeout: ProcessingWaitTimeout, processingWaitTimeout: ProcessingWaitTimeout,
} }
} }
// Process transforms the image read from input according to transform. // Process transforms an image according to the request. When
// sourceFormat is the input's format, which magic detected from its first // MaxConcurrentProcessing images are already being processed, it waits up
// bytes. When MaxConcurrentProcessing images are already being processed, // to ProcessingWaitTimeout for one to finish, then fails with
// it waits up to ProcessingWaitTimeout for one to finish, then fails with // ErrTooManyImages.
// ErrTooManyImages. When ctx has ended once Process has a slot, or before it
// encodes, it returns ctx's error and gives the slot back. govips gives pixa
// no way to stop libvips once an encode has begun, so an encode keeps its
// slot until it finishes. An AVIF output of more than MaxAVIFOutputPixels is
// refused with ErrAVIFOutputTooLarge before anything is decoded.
func (p *ImageProcessor) Process( func (p *ImageProcessor) Process(
ctx context.Context, ctx context.Context,
input io.Reader, input io.Reader,
sourceFormat imagerequest.Format, req *Request,
transform *imagerequest.Transform,
) (*Result, error) { ) (*Result, error) {
release, err := p.acquireSlot(ctx) release, err := p.acquireSlot(ctx)
if err != nil { if err != nil {
@@ -178,14 +221,27 @@ func (p *ImageProcessor) Process(
defer release() defer release()
data, err := p.readInput(input) // Read input with a size limit to prevent unbounded memory consumption.
// We read at most maxInputBytes+1 so we can detect if the input exceeds
// the limit without consuming additional memory.
limited := io.LimitReader(input, p.maxInputBytes+1)
data, err := io.ReadAll(limited)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("failed to read input: %w", err)
} }
// Input libvips would not load as sourceFormat is refused before it is if int64(len(data)) > p.maxInputBytes {
// decoded return nil, ErrInputDataTooLarge
err = checkInputFormat(data, sourceFormat) }
// govips picks the libvips loader from the input's first bytes with
// DetermineImageType. Input whose first bytes it reads as no source
// format, such as SVG, which librsvg would render, is refused before
// libvips reads it.
inputFormat := p.detectFormat(vips.DetermineImageType(data))
sourceFormat, err := p.formatFromString(inputFormat)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -214,36 +270,27 @@ func (p *ImageProcessor) Process(
} }
// Determine target dimensions; too large ones are refused before resizing // Determine target dimensions; too large ones are refused before resizing
targetWidth, targetHeight, err := targetDimensions(transform.Size, targetWidth, targetHeight, err := targetDimensions(req.Size, origWidth, origHeight)
origWidth, origHeight)
if err != nil {
return nil, err
}
// orig is the source's own format; encode refuses an empty format
outputFormat := transform.Format
if outputFormat == imagerequest.FormatOriginal {
outputFormat = sourceFormat
}
// Determine output dimensions; too large ones are refused before
// resizing, which with FitCover already decodes the input
outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight,
targetWidth, targetHeight, transform.FitMode, outputFormat)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Resize if needed // Resize if needed
if targetWidth != origWidth || targetHeight != origHeight { if targetWidth != origWidth || targetHeight != origHeight {
err := p.resize(img, outputWidth, outputHeight, transform.FitMode) err := p.resize(img, targetWidth, targetHeight, req.FitMode)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to resize: %w", err) return nil, fmt.Errorf("failed to resize: %w", err)
} }
} }
// orig is the source's own format; encode refuses an empty format
outputFormat := req.Format
if outputFormat == FormatOriginal {
outputFormat = sourceFormat
}
// Encode to target format // Encode to target format
output, err := p.encode(ctx, img, outputFormat, transform.Quality) output, err := p.encode(img, outputFormat, req.Quality)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to encode: %w", err) return nil, fmt.Errorf("failed to encode: %w", err)
} }
@@ -251,23 +298,20 @@ func (p *ImageProcessor) Process(
return &Result{ return &Result{
Content: io.NopCloser(bytes.NewReader(output)), Content: io.NopCloser(bytes.NewReader(output)),
ContentLength: int64(len(output)), ContentLength: int64(len(output)),
ContentType: outputFormat.MIMEType(), ContentType: FormatToMIME(outputFormat),
Width: img.Width(), Width: img.Width(),
Height: img.Height(), Height: img.Height(),
InputWidth: origWidth, InputWidth: origWidth,
InputHeight: origHeight, InputHeight: origHeight,
InputFormat: inputFormat,
}, nil }, nil
} }
// targetDimensions calculates the output dimensions for a requested size, // targetDimensions calculates the output dimensions for a requested size,
// scaling proportionally when only one dimension is given and keeping the // scaling proportionally when only one dimension is given and keeping the
// original dimensions when both are zero. A side scaled proportionally is // original dimensions when both are zero. A side scaled proportionally can
// rounded down but kept at 1 or more, as for a thin image it would round // come out above MaxOutputDimension, which is ErrOutputTooLarge.
// down to 0, a side of no pixels. It can come out above MaxOutputDimension, func targetDimensions(size Size, origWidth, origHeight int) (int, int, error) {
// which is ErrOutputTooLarge.
func targetDimensions(
size imagerequest.Size, origWidth, origHeight int,
) (int, int, error) {
width, height := size.Width, size.Height width, height := size.Width, size.Height
switch { switch {
@@ -276,10 +320,10 @@ func targetDimensions(
return origWidth, origHeight, nil return origWidth, origHeight, nil
case width == 0: case width == 0:
// Only height specified: calculate width proportionally // Only height specified: calculate width proportionally
width = max(1, origWidth*height/origHeight) width = origWidth * height / origHeight
case height == 0: case height == 0:
// Only width specified: calculate height proportionally // Only width specified: calculate height proportionally
height = max(1, origHeight*width/origWidth) height = origHeight * width / origWidth
} }
if width > MaxOutputDimension || height > MaxOutputDimension { if width > MaxOutputDimension || height > MaxOutputDimension {
@@ -290,6 +334,62 @@ func targetDimensions(
return width, height, nil return width, height, nil
} }
// MIME types for the supported image formats.
const (
mimeJPEG = "image/jpeg"
mimePNG = "image/png"
mimeGIF = "image/gif"
mimeWebP = "image/webp"
mimeAVIF = "image/avif"
mimeJXL = "image/jxl"
)
// SupportedInputFormats returns MIME types this processor can read.
func (p *ImageProcessor) SupportedInputFormats() []string {
return []string{
mimeJPEG,
mimePNG,
mimeGIF,
mimeWebP,
mimeAVIF,
mimeJXL,
}
}
// SupportedOutputFormats returns formats this processor can write.
func (p *ImageProcessor) SupportedOutputFormats() []Format {
return []Format{
FormatJPEG,
FormatPNG,
FormatGIF,
FormatWebP,
FormatAVIF,
FormatJXL,
}
}
// FormatToMIME converts a Format to its MIME type string.
func FormatToMIME(format Format) string {
switch format {
case FormatJPEG:
return mimeJPEG
case FormatPNG:
return mimePNG
case FormatWebP:
return mimeWebP
case FormatGIF:
return mimeGIF
case FormatAVIF:
return mimeAVIF
case FormatJXL:
return mimeJXL
case FormatOriginal:
return "application/octet-stream"
default:
return "application/octet-stream"
}
}
// WaitForProcessing waits until no image is being processed, or until ctx // WaitForProcessing waits until no image is being processed, or until ctx
// ends, and returns how many images were still being processed then. It // ends, and returns how many images were still being processed then. It
// waits by taking each slot in processingSemaphore as it frees up until it // waits by taking each slot in processingSemaphore as it frees up until it
@@ -317,230 +417,120 @@ func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int {
// acquireSlot takes a slot in processingSemaphore, waiting at most // acquireSlot takes a slot in processingSemaphore, waiting at most
// processingWaitTimeout for one to free up, and returns the func that gives // processingWaitTimeout for one to free up, and returns the func that gives
// it back. When ctx has ended by the time it has a slot, it gives the slot // it back. A free slot is taken even when ctx has ended; only the wait for
// back and returns ctx's error. // one stops when ctx ends, as the rest of Process does not check ctx.
func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) { func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
release := func() { <-p.processingSemaphore }
select { select {
case p.processingSemaphore <- struct{}{}: case p.processingSemaphore <- struct{}{}:
return release, nil
default:
}
select {
case p.processingSemaphore <- struct{}{}:
return release, nil
case <-time.After(p.processingWaitTimeout): case <-time.After(p.processingWaitTimeout):
return nil, ErrTooManyImages return nil, ErrTooManyImages
case <-ctx.Done(): case <-ctx.Done():
return nil, ctx.Err() return nil, ctx.Err()
} }
release := func() { <-p.processingSemaphore }
// select takes the slot even when ctx has ended at the same moment
err := ctx.Err()
if err != nil {
release()
return nil, err
} }
return release, nil // detectFormat returns the format string for a govips image type.
} func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string {
switch imageType {
// readInput reads input with a size limit to prevent unbounded memory case vips.ImageTypeJPEG:
// consumption, refusing more than maxInputBytes with ErrInputDataTooLarge. It return "jpeg"
// reads at most maxInputBytes+1 so it can detect if the input exceeds the case vips.ImageTypePNG:
// limit without consuming additional memory. return "png"
func (p *ImageProcessor) readInput(input io.Reader) ([]byte, error) { case vips.ImageTypeGIF:
data, err := io.ReadAll(io.LimitReader(input, p.maxInputBytes+1)) return "gif"
if err != nil { case vips.ImageTypeWEBP:
return nil, fmt.Errorf("failed to read input: %w", err) return "webp"
} case vips.ImageTypeAVIF, vips.ImageTypeHEIF:
return string(FormatAVIF)
if int64(len(data)) > p.maxInputBytes { case vips.ImageTypeJXL:
return nil, ErrInputDataTooLarge return string(FormatJXL)
} case vips.ImageTypeUnknown, vips.ImageTypeMagick, vips.ImageTypePDF,
vips.ImageTypeSVG, vips.ImageTypeTIFF, vips.ImageTypeBMP,
return data, nil vips.ImageTypeJP2K:
} return "unknown"
// checkInputFormat returns ErrUnsupportedInputFormat unless libvips would
// load data with the loader of sourceFormat. govips picks the libvips loader
// from the input's first bytes with DetermineImageType. It refuses an SVG
// whose first bytes pass the first-bytes check for AVIF, which librsvg would
// otherwise render.
func checkInputFormat(data []byte, sourceFormat imagerequest.Format) error {
// A format with no loader, such as orig or auto, has the type govips also
// gives input it cannot read, which the check below would let through.
sourceType := vipsImageType(sourceFormat)
if sourceType == vips.ImageTypeUnknown {
return fmt.Errorf("%w: %q is not a source format",
ErrUnsupportedInputFormat, sourceFormat)
}
if vips.DetermineImageType(data) != sourceType {
return fmt.Errorf("%w: libvips does not read the input as %s",
ErrUnsupportedInputFormat, sourceFormat)
}
return nil
}
// vipsImageType returns the govips image type whose libvips loader reads
// format. orig and auto stand for another format, so they have
// vips.ImageTypeUnknown, which govips has no loader for.
func vipsImageType(format imagerequest.Format) vips.ImageType {
switch format {
case imagerequest.FormatJPEG:
return vips.ImageTypeJPEG
case imagerequest.FormatPNG:
return vips.ImageTypePNG
case imagerequest.FormatGIF:
return vips.ImageTypeGIF
case imagerequest.FormatWebP:
return vips.ImageTypeWEBP
case imagerequest.FormatAVIF:
return vips.ImageTypeAVIF
case imagerequest.FormatJXL:
return vips.ImageTypeJXL
case imagerequest.FormatOriginal, imagerequest.FormatAuto:
return vips.ImageTypeUnknown
default: default:
return vips.ImageTypeUnknown return "unknown"
} }
} }
// outputDimensions returns the dimensions resize gives an image of imgWidth x // resize resizes the image according to the fit mode. With FitOutside one
// imgHeight for the target dimensions and fit mode, without resizing it, and // side can come out larger than requested; when it would be above
// refuses an output too large for format. With FitOutside one side can come // MaxOutputDimension, resize returns ErrOutputTooLarge without resizing.
// out larger than the target; when it would be above MaxOutputDimension,
// outputDimensions returns ErrOutputTooLarge. An AVIF output of more than
// MaxAVIFOutputPixels is ErrAVIFOutputTooLarge.
func outputDimensions(
imgWidth, imgHeight, width, height int,
fit imagerequest.Fit, format imagerequest.Format,
) (int, int, error) {
scaleW := float64(width) / float64(imgWidth)
scaleH := float64(height) / float64(imgHeight)
outputWidth, outputHeight := width, height
switch fit {
case imagerequest.FitCover, imagerequest.FitFill, "":
// Cover crops, and fill stretches, to exactly the target dimensions
case imagerequest.FitContain:
// Fit within the target dimensions, maintaining aspect ratio. Scaled
// by the smaller scale, the short side of a thin image can round
// down to 0, which libvips refuses as a width, so each side is at
// least 1.
scale := min(scaleW, scaleH)
outputWidth = max(1, int(float64(imgWidth)*scale))
outputHeight = max(1, int(float64(imgHeight)*scale))
case imagerequest.FitInside:
// Same as contain, but only shrink
scale := min(scaleW, scaleH, 1)
outputWidth = max(1, int(float64(imgWidth)*scale))
outputHeight = max(1, int(float64(imgHeight)*scale))
case imagerequest.FitOutside:
// The smallest dimension fits, the other may exceed the target. A
// side scaled to 1 can come out just under 1 in floating point, as
// 49 times 1/49 does, and round down to 0, which libvips refuses as
// a width, so each side is at least 1.
scale := max(scaleW, scaleH)
outputWidth = max(1, int(float64(imgWidth)*scale))
outputHeight = max(1, int(float64(imgHeight)*scale))
default:
return 0, 0, fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, fit)
}
// The side FitOutside scales past the target is held to the maximum too
if max(outputWidth, outputHeight) > MaxOutputDimension {
return 0, 0, fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge,
outputWidth, outputHeight, MaxOutputDimension)
}
if format == imagerequest.FormatAVIF &&
outputWidth*outputHeight > MaxAVIFOutputPixels {
return 0, 0, fmt.Errorf("%w: %dx%d is above %d pixels",
ErrAVIFOutputTooLarge, outputWidth, outputHeight, MaxAVIFOutputPixels)
}
return outputWidth, outputHeight, nil
}
// resize resizes the image to width x height, the dimensions
// outputDimensions gives for the fit mode.
func (p *ImageProcessor) resize( func (p *ImageProcessor) resize(
img *vips.ImageRef, width, height int, fit imagerequest.Fit, img *vips.ImageRef, width, height int, fit FitMode,
) error { ) error {
switch fit { switch fit {
case imagerequest.FitCover, "": case FitCover, "":
// Crop the centre and resize to fill exact dimensions (default) // Resize and crop to fill exact dimensions (default)
return cropCentreAndResize(img, width, height) return img.Thumbnail(width, height, vips.InterestingCentre)
case imagerequest.FitFill: case FitContain:
// Resize to fit within dimensions, maintaining aspect ratio
imgW, imgH := img.Width(), img.Height()
scaleW := float64(width) / float64(imgW)
scaleH := float64(height) / float64(imgH)
scale := min(scaleW, scaleH)
newW := int(float64(imgW) * scale)
newH := int(float64(imgH) * scale)
return img.Thumbnail(newW, newH, vips.InterestingNone)
case FitFill:
// Resize to exact dimensions (may distort) // Resize to exact dimensions (may distort)
return img.ThumbnailWithSize(width, height, vips.InterestingNone, vips.SizeForce) return img.ThumbnailWithSize(width, height, vips.InterestingNone, vips.SizeForce)
case imagerequest.FitInside: case FitInside:
// outputDimensions keeps the dimensions of an image that already fits // Same as contain, but only shrink
if img.Width() == width && img.Height() == height { if img.Width() <= width && img.Height() <= height {
return nil return nil // Already fits
} }
return img.Thumbnail(width, height, vips.InterestingNone) imgW, imgH := img.Width(), img.Height()
scaleW := float64(width) / float64(imgW)
scaleH := float64(height) / float64(imgH)
scale := min(scaleW, scaleH)
newW := int(float64(imgW) * scale)
newH := int(float64(imgH) * scale)
case imagerequest.FitContain, imagerequest.FitOutside: return img.Thumbnail(newW, newH, vips.InterestingNone)
return img.Thumbnail(width, height, vips.InterestingNone)
case FitOutside:
// Resize so smallest dimension fits, may exceed target on other dimension
imgW, imgH := img.Width(), img.Height()
scaleW := float64(width) / float64(imgW)
scaleH := float64(height) / float64(imgH)
scale := max(scaleW, scaleH)
newW := int(float64(imgW) * scale)
newH := int(float64(imgH) * scale)
// The dimension that exceeds the target is held to the maximum too
if newW > MaxOutputDimension || newH > MaxOutputDimension {
return fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge,
newW, newH, MaxOutputDimension)
}
return img.Thumbnail(newW, newH, vips.InterestingNone)
default: default:
return fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, fit) return fmt.Errorf("%w: %s", ErrInvalidFitMode, fit)
} }
} }
// cropCentreAndResize scales img by the same amount on both sides, the
// least that makes it cover width x height, and cuts the centre width x
// height out of it, as libvips' thumbnail does with InterestingCentre.
// thumbnail alone scales the whole image and copies the result into memory
// before it cuts, and for a thin image that is far larger than the image or
// the output: 8192x64 at 8192x8192 is 1048576x8192, about 24 GiB. So the
// centre is cut from img first, widened to whole pixels of img, which adds
// less than one pixel of img on each side, and thumbnail scales and cuts
// only that.
func cropCentreAndResize(img *vips.ImageRef, width, height int) error {
imgW, imgH := img.Width(), img.Height()
// An image wider than the requested proportions keeps its height and
// loses width down to imgH*width/height, any other keeps its width and
// loses height down to imgW*height/width. Half of what is lost comes
// off each side, rounded down to whole pixels, so the part kept holds
// all of the centre.
left, top := 0, 0
if imgW*height > imgH*width {
left = (imgW*height - imgH*width) / (2 * height) //nolint:mnd // each side
} else {
top = (imgH*width - imgW*height) / (2 * width) //nolint:mnd // each side
}
err := img.ExtractArea(left, top, imgW-2*left, imgH-2*top)
if err != nil {
return err
}
return img.Thumbnail(width, height, vips.InterestingCentre)
}
const defaultQuality = 85 const defaultQuality = 85
// encode encodes an image to the specified format. govips gives pixa no way // encode encodes an image to the specified format.
// to stop libvips once an encode has begun, so when ctx has ended encode
// returns ctx's error without beginning one.
func (p *ImageProcessor) encode( func (p *ImageProcessor) encode(
ctx context.Context, img *vips.ImageRef, format imagerequest.Format, img *vips.ImageRef, format Format, quality int,
quality int,
) ([]byte, error) { ) ([]byte, error) {
err := ctx.Err()
if err != nil {
return nil, err
}
if quality <= 0 { if quality <= 0 {
quality = defaultQuality quality = defaultQuality
} }
@@ -550,32 +540,32 @@ func (p *ImageProcessor) encode(
// libvips' built-in profile; govips' own sRGB path variable is set on // libvips' built-in profile; govips' own sRGB path variable is set on
// first use but read without a lock, so concurrent requests race on it. // first use but read without a lock, so concurrent requests race on it.
if img.HasICCProfile() { if img.HasICCProfile() {
err = img.TransformICCProfileWithFallback("srgb", "srgb") err := img.TransformICCProfileWithFallback("srgb", "srgb")
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to convert to sRGB: %w", err) return nil, fmt.Errorf("failed to convert to sRGB: %w", err)
} }
} }
switch format { switch format {
case imagerequest.FormatJPEG: case FormatJPEG:
return exportJPEG(img, quality) return exportJPEG(img, quality)
case imagerequest.FormatPNG: case FormatPNG:
return exportPNG(img) return exportPNG(img)
case imagerequest.FormatGIF: case FormatGIF:
return exportGIF(img) return exportGIF(img)
case imagerequest.FormatWebP: case FormatWebP:
return exportWebP(img, quality) return exportWebP(img, quality)
case imagerequest.FormatAVIF: case FormatAVIF:
return exportAVIF(img, quality) return exportAVIF(img, quality)
case imagerequest.FormatJXL: case FormatJXL:
return exportJXL(img, quality) return exportJXL(img, quality)
case imagerequest.FormatOriginal, imagerequest.FormatAuto: case FormatOriginal:
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format) return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
default: default:
@@ -625,16 +615,11 @@ func exportGIF(img *vips.ImageRef) ([]byte, error) {
return output, err return output, err
} }
// webpEffort is the WebP effort, from 0 (fastest) to 6. It is below libvips' // webpEffort is libvips' default WebP effort, from 0 (fastest) to 6.
// default of 4 so that an encode takes at most 40 seconds of the default const webpEffort = 4
// downstream_timeout of 60 seconds. With one thread, as pixad runs libvips,
// saving an 8192x8192 image of random pixels with an alpha channel takes up
// to about 23 seconds at 2, and up to about 16 without one, at any quality;
// with an alpha channel, 3 takes about 50 seconds and 4 about 61.
const webpEffort = 2
// exportWebP encodes img as lossy WebP at quality and webpEffort, without // exportWebP encodes img as lossy WebP at quality and libvips' default
// metadata. // effort, without metadata.
func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) { func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) {
output, _, err := img.ExportWebp(&vips.WebpExportParams{ output, _, err := img.ExportWebp(&vips.WebpExportParams{
StripMetadata: true, StripMetadata: true,
@@ -646,13 +631,11 @@ func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) {
} }
// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest // avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest
// govips can set. With one thread, as pixad runs libvips, 1 takes about 52 // govips can set. With one thread, as pixad runs libvips, 1 takes about 51
// seconds to save an 8192x8192 RGB image of random pixels at quality 85, // seconds to save an 8192x8192 image of random pixels, the worst case,
// and about 96 with an alpha channel, against the default // against the default downstream_timeout of 60 seconds. On an image of
// downstream_timeout of 60 seconds, so an AVIF output is held to // milder noise, which 1 saves in about 12 seconds, 2 takes nearly a minute
// MaxAVIFOutputPixels. On an image of milder noise, which 1 saves in about // and libvips' default, 4, takes minutes.
// 12 seconds, 2 takes nearly a minute and libvips' default, 4, takes
// minutes.
const avifEffort = 1 const avifEffort = 1
// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image. // avifBitdepth is the AVIF bit depth, 8 bits per sample for every image.
@@ -679,18 +662,10 @@ func exportAVIF(img *vips.ImageRef, quality int) ([]byte, error) {
// JPEG that names none. // JPEG that names none.
const jxlResolution = 72 / 25.4 const jxlResolution = 72 / 25.4
// jxlEffort is the JPEG XL effort, from 1 (fastest) to 9. It is below // exportJXL encodes img as JPEG XL at quality, with libvips' default effort
// libvips' default of 7 so that an encode takes at most 40 seconds of the // and without metadata. govips sends libvips a distance, the JPEG XL
// default downstream_timeout of 60 seconds. With one thread, as pixad runs // encoder's own measure of quality, along with the quality, and libvips then
// libvips, saving an 8192x8192 image of random pixels with an alpha channel // uses the distance alone, so the quality is also given as a distance.
// takes up to about 27 seconds at 4, and up to about 13 without one, at any
// quality; with an alpha channel, 5 takes about 43 seconds and 7 about 81.
const jxlEffort = 4
// exportJXL encodes img as JPEG XL at quality and jxlEffort, without
// metadata. govips sends libvips a distance, the JPEG XL encoder's own
// measure of quality, along with the quality, and libvips then uses the
// distance alone, so the quality is also given as a distance.
func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) { func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) {
// libvips converts a CMYK image to sRGB before it saves WebP, AVIF or // libvips converts a CMYK image to sRGB before it saves WebP, AVIF or
// PNG, but cannot save one as JPEG XL. encode has already converted // PNG, but cannot save one as JPEG XL. encode has already converted
@@ -729,7 +704,6 @@ func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) {
params := vips.NewJxlExportParams() params := vips.NewJxlExportParams()
params.Quality = quality params.Quality = quality
params.Distance = jxlDistance(quality) params.Distance = jxlDistance(quality)
params.Effort = jxlEffort
output, _, err := toSave.ExportJxl(params) output, _, err := toSave.ExportJxl(params)
if err != nil { if err != nil {
@@ -754,3 +728,24 @@ func jxlDistance(quality int) float64 {
return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0 return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0
} }
// formatFromString converts a format string from detectFormat to Format. A
// format detectFormat could not name is ErrUnsupportedInputFormat.
func (p *ImageProcessor) formatFromString(format string) (Format, error) {
switch format {
case "jpeg":
return FormatJPEG, nil
case "png":
return FormatPNG, nil
case "gif":
return FormatGIF, nil
case "webp":
return FormatWebP, nil
case string(FormatAVIF):
return FormatAVIF, nil
case string(FormatJXL):
return FormatJXL, nil
default:
return "", fmt.Errorf("%w: %s", ErrUnsupportedInputFormat, format)
}
}
@@ -15,7 +15,6 @@ import (
"testing" "testing"
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
@@ -88,29 +87,29 @@ func isAVIF(data []byte) bool {
brand := string(data[8:12]) brand := string(data[8:12])
return brand == string(imagerequest.FormatAVIF) || brand == "avis" return brand == string(FormatAVIF) || brand == "avis"
} }
// detectMIME is a minimal magic-byte detector for test assertions. // detectMIME is a minimal magic-byte detector for test assertions.
func detectMIME(data []byte) string { func detectMIME(data []byte) string {
if len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { if len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF {
return imagerequest.FormatJPEG.MIMEType() return mimeJPEG
} }
if len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n" { if len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n" {
return imagerequest.FormatPNG.MIMEType() return mimePNG
} }
if len(data) >= 4 && string(data[:4]) == "GIF8" { if len(data) >= 4 && string(data[:4]) == "GIF8" {
return imagerequest.FormatGIF.MIMEType() return mimeGIF
} }
if len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP" { if len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP" {
return imagerequest.FormatWebP.MIMEType() return mimeWebP
} }
if isAVIF(data) { if isAVIF(data) {
return imagerequest.FormatAVIF.MIMEType() return mimeAVIF
} }
return "" return ""
@@ -119,20 +118,19 @@ func detectMIME(data []byte) string {
func TestImageProcessor_ResizeJPEG(t *testing.T) { func TestImageProcessor_ResizeJPEG(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
input := createTestJPEG(t, 800, 600) input := createTestJPEG(t, 800, 600)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 400, Height: 300}, Size: Size{Width: 400, Height: 300},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
} }
@@ -158,7 +156,7 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) {
} }
mime := detectMIME(data) mime := detectMIME(data)
if mime != imagerequest.FormatJPEG.MIMEType() { if mime != mimeJPEG {
t.Errorf("Output format = %v, want image/jpeg", mime) t.Errorf("Output format = %v, want image/jpeg", mime)
} }
} }
@@ -166,19 +164,18 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) {
func TestImageProcessor_ConvertToPNG(t *testing.T) { func TestImageProcessor_ConvertToPNG(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
input := createTestJPEG(t, 200, 150) input := createTestJPEG(t, 200, 150)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 200, Height: 150}, Size: Size{Width: 200, Height: 150},
Format: imagerequest.FormatPNG, Format: FormatPNG,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
} }
@@ -191,7 +188,7 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) {
} }
mime := detectMIME(data) mime := detectMIME(data)
if mime != imagerequest.FormatPNG.MIMEType() { if mime != mimePNG {
t.Errorf("Output format = %v, want image/png", mime) t.Errorf("Output format = %v, want image/png", mime)
} }
} }
@@ -199,24 +196,23 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) {
// processAndCheckSize processes a test JPEG of the given input dimensions // processAndCheckSize processes a test JPEG of the given input dimensions
// with the requested size and asserts the resulting dimensions. // with the requested size and asserts the resulting dimensions.
func processAndCheckSize( func processAndCheckSize(
t *testing.T, inputW, inputH int, size imagerequest.Size, wantW, wantH int, t *testing.T, inputW, inputH int, size Size, wantW, wantH int,
) { ) {
t.Helper() t.Helper()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
input := createTestJPEG(t, inputW, inputH) input := createTestJPEG(t, inputW, inputH)
transform := &imagerequest.Transform{ req := &Request{
Size: size, Size: size,
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
} }
@@ -236,28 +232,27 @@ func TestImageProcessor_OriginalSize(t *testing.T) {
t.Parallel() t.Parallel()
// Width and height 0: keep original size // Width and height 0: keep original size
processAndCheckSize(t, 640, 480, imagerequest.Size{Width: 0, Height: 0}, 640, 480) processAndCheckSize(t, 640, 480, Size{Width: 0, Height: 0}, 640, 480)
} }
func TestImageProcessor_FitContain(t *testing.T) { func TestImageProcessor_FitContain(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
// 800x400 image (2:1 aspect) into 400x400 box with contain // 800x400 image (2:1 aspect) into 400x400 box with contain
// Should result in 400x200 (maintaining aspect ratio) // Should result in 400x200 (maintaining aspect ratio)
input := createTestJPEG(t, 800, 400) input := createTestJPEG(t, 800, 400)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 400, Height: 400}, Size: Size{Width: 400, Height: 400},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitContain, FitMode: FitContain,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
} }
@@ -275,7 +270,7 @@ func TestImageProcessor_ProportionalScale_WidthOnly(t *testing.T) {
// 800x600 image, request width=400 height=0 // 800x600 image, request width=400 height=0
// Should scale proportionally to 400x300 // Should scale proportionally to 400x300
processAndCheckSize(t, 800, 600, imagerequest.Size{Width: 400, Height: 0}, 400, 300) processAndCheckSize(t, 800, 600, Size{Width: 400, Height: 0}, 400, 300)
} }
func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) { func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) {
@@ -283,25 +278,24 @@ func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) {
// 800x600 image, request width=0 height=300 // 800x600 image, request width=0 height=300
// Should scale proportionally to 400x300 // Should scale proportionally to 400x300
processAndCheckSize(t, 800, 600, imagerequest.Size{Width: 0, Height: 300}, 400, 300) processAndCheckSize(t, 800, 600, Size{Width: 0, Height: 300}, 400, 300)
} }
func TestImageProcessor_ProcessPNG(t *testing.T) { func TestImageProcessor_ProcessPNG(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
input := createTestPNG(t, 400, 300) input := createTestPNG(t, 400, 300)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 200, Height: 150}, Size: Size{Width: 200, Height: 150},
Format: imagerequest.FormatPNG, Format: FormatPNG,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatPNG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
} }
@@ -317,6 +311,22 @@ func TestImageProcessor_ProcessPNG(t *testing.T) {
} }
} }
func TestImageProcessor_SupportedFormats(t *testing.T) {
t.Parallel()
proc := New(Params{})
inputFormats := proc.SupportedInputFormats()
if len(inputFormats) == 0 {
t.Error("SupportedInputFormats() returned empty slice")
}
outputFormats := proc.SupportedOutputFormats()
if len(outputFormats) == 0 {
t.Error("SupportedOutputFormats() returned empty slice")
}
}
func TestImageProcessor_RejectsOversizedInput(t *testing.T) { func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
t.Parallel() t.Parallel()
@@ -335,19 +345,18 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
input := createTestJPEG(t, tt.width, tt.height) input := createTestJPEG(t, tt.width, tt.height)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 100, Height: 100}, Size: Size{Width: 100, Height: 100},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
_, err := proc.Process(ctx, bytes.NewReader(input), _, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err == nil { if err == nil {
t.Error("Process() should reject oversized input images") t.Error("Process() should reject oversized input images")
} }
@@ -362,21 +371,20 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) { func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
// Create an image at exactly MaxInputDimension - should be accepted // Create an image at exactly MaxInputDimension - should be accepted
input := createTestJPEG(t, MaxInputDimension, 100) input := createTestJPEG(t, MaxInputDimension, 100)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 100, Height: 100}, Size: Size{Width: 100, Height: 100},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf( t.Fatalf(
"Process() should accept images at MaxInputDimension, got error: %v", "Process() should accept images at MaxInputDimension, got error: %v",
@@ -389,25 +397,22 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) {
// encodeAndCheck processes a 200x150 test JPEG into a 100x75 output of the // encodeAndCheck processes a 200x150 test JPEG into a 100x75 output of the
// given format and asserts the output MIME type and dimensions. // given format and asserts the output MIME type and dimensions.
func encodeAndCheck( func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) {
t *testing.T, format imagerequest.Format, quality int, wantMIME string,
) {
t.Helper() t.Helper()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
input := createTestJPEG(t, 200, 150) input := createTestJPEG(t, 200, 150)
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 100, Height: 75}, Size: Size{Width: 100, Height: 75},
Format: format, Format: format,
Quality: quality, Quality: quality,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v, want nil", err) t.Fatalf("Process() error = %v, want nil", err)
} }
@@ -438,13 +443,13 @@ func encodeAndCheck(
func TestImageProcessor_EncodeWebP(t *testing.T) { func TestImageProcessor_EncodeWebP(t *testing.T) {
t.Parallel() t.Parallel()
encodeAndCheck(t, imagerequest.FormatWebP, 80, imagerequest.FormatWebP.MIMEType()) encodeAndCheck(t, FormatWebP, 80, mimeWebP)
} }
func TestImageProcessor_DecodeAVIF(t *testing.T) { func TestImageProcessor_DecodeAVIF(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
ctx := context.Background() ctx := context.Background()
// Load test AVIF file // Load test AVIF file
@@ -454,15 +459,14 @@ func TestImageProcessor_DecodeAVIF(t *testing.T) {
} }
// Request resize and convert to JPEG // Request resize and convert to JPEG
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 2, Height: 2}, Size: Size{Width: 2, Height: 2},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatAVIF, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v, want nil (AVIF decoding should work)", err) t.Fatalf("Process() error = %v, want nil (AVIF decoding should work)", err)
} }
@@ -476,7 +480,7 @@ func TestImageProcessor_DecodeAVIF(t *testing.T) {
} }
mime := detectMIME(data) mime := detectMIME(data)
if mime != imagerequest.FormatJPEG.MIMEType() { if mime != mimeJPEG {
t.Errorf("Output format = %v, want image/jpeg", mime) t.Errorf("Output format = %v, want image/jpeg", mime)
} }
} }
@@ -487,7 +491,7 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) {
// Create a processor with a very small byte limit // Create a processor with a very small byte limit
const limit = 1024 const limit = 1024
proc := New(Params{MaxInputBytes: limit, MaxConcurrentProcessing: 1}) proc := New(Params{MaxInputBytes: limit})
ctx := context.Background() ctx := context.Background()
// Create a valid JPEG that exceeds the byte limit // Create a valid JPEG that exceeds the byte limit
@@ -496,15 +500,14 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) {
t.Fatalf("test JPEG must exceed %d bytes, got %d", limit, len(input)) t.Fatalf("test JPEG must exceed %d bytes, got %d", limit, len(input))
} }
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 100, Height: 75}, Size: Size{Width: 100, Height: 75},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
_, err := proc.Process(ctx, bytes.NewReader(input), _, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err == nil { if err == nil {
t.Fatal("Process() should reject input exceeding maxInputBytes") t.Fatal("Process() should reject input exceeding maxInputBytes")
} }
@@ -521,18 +524,17 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) {
input := createTestJPEG(t, 10, 10) input := createTestJPEG(t, 10, 10)
limit := int64(len(input)) * 10 // 10× headroom limit := int64(len(input)) * 10 // 10× headroom
proc := New(Params{MaxInputBytes: limit, MaxConcurrentProcessing: 1}) proc := New(Params{MaxInputBytes: limit})
ctx := context.Background() ctx := context.Background()
transform := &imagerequest.Transform{ req := &Request{
Size: imagerequest.Size{Width: 10, Height: 10}, Size: Size{Width: 10, Height: 10},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
result, err := proc.Process(ctx, bytes.NewReader(input), result, err := proc.Process(ctx, bytes.NewReader(input), req)
imagerequest.FormatJPEG, transform)
if err != nil { if err != nil {
t.Fatalf("Process() error = %v, want nil", err) t.Fatalf("Process() error = %v, want nil", err)
} }
@@ -540,25 +542,35 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) {
defer func() { _ = result.Content.Close() }() defer func() { _ = result.Content.Close() }()
} }
func TestImageProcessor_DefaultMaxInputBytes(t *testing.T) {
t.Parallel()
// Passing 0 should use the default
proc := New(Params{})
if proc.maxInputBytes != DefaultMaxInputBytes {
t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes)
}
// Passing negative should also use the default
proc = New(Params{MaxInputBytes: -1})
if proc.maxInputBytes != DefaultMaxInputBytes {
t.Errorf("maxInputBytes = %d, want %d", proc.maxInputBytes, DefaultMaxInputBytes)
}
}
func TestImageProcessor_EncodeAVIF(t *testing.T) { func TestImageProcessor_EncodeAVIF(t *testing.T) {
t.Parallel() t.Parallel()
encodeAndCheck(t, imagerequest.FormatAVIF, 85, imagerequest.FormatAVIF.MIMEType()) encodeAndCheck(t, FormatAVIF, 85, mimeAVIF)
} }
// processAndDecode runs input, a JPEG, through Process and decodes the output // processAndDecode runs input through Process and decodes the output with
// with vips, so a test can inspect the image a client would receive. // vips, so a test can inspect the image a client would receive.
func processAndDecode( func processAndDecode(t *testing.T, input []byte, req *Request) *vips.ImageRef {
t *testing.T, input []byte, transform *imagerequest.Transform,
) *vips.ImageRef {
t.Helper() t.Helper()
result, err := New(Params{ result, err := New(Params{}).Process(
MaxInputBytes: 50 << 20, context.Background(), bytes.NewReader(input), req,
MaxConcurrentProcessing: 1,
}).Process(
context.Background(), bytes.NewReader(input), imagerequest.FormatJPEG,
transform,
) )
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
@@ -602,17 +614,15 @@ func TestImageProcessor_StripsEXIF(t *testing.T) {
t.Fatal("testdata/gps-exif.jpg has no GPS latitude") t.Fatal("testdata/gps-exif.jpg has no GPS latitude")
} }
formats := []imagerequest.Format{ formats := []Format{
imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatWebP, FormatJPEG, FormatPNG, FormatWebP, FormatAVIF, FormatGIF, FormatOriginal,
imagerequest.FormatAVIF, imagerequest.FormatGIF, imagerequest.FormatOriginal,
} }
for _, format := range formats { for _, format := range formats {
t.Run(string(format), func(t *testing.T) { t.Run(string(format), func(t *testing.T) {
t.Parallel() t.Parallel()
output := processAndDecode(t, input, output := processAndDecode(t, input, &Request{Format: format})
&imagerequest.Transform{Format: format})
if output.HasExif() { if output.HasExif() {
t.Errorf("output has EXIF: %v", output.GetExif()) t.Errorf("output has EXIF: %v", output.GetExif())
@@ -634,21 +644,21 @@ func TestImageProcessor_AppliesEXIFOrientation(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
size imagerequest.Size size Size
wantW int wantW int
wantH int wantH int
}{ }{
{name: "original size", size: imagerequest.Size{}, wantW: 8, wantH: 16}, {name: "original size", size: Size{}, wantW: 8, wantH: 16},
{name: "width only", size: imagerequest.Size{Width: 4}, wantW: 4, wantH: 8}, {name: "width only", size: Size{Width: 4}, wantW: 4, wantH: 8},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
t.Parallel() t.Parallel()
output := processAndDecode(t, input, &imagerequest.Transform{ output := processAndDecode(t, input, &Request{
Size: tt.size, Size: tt.size,
Format: imagerequest.FormatPNG, Format: FormatPNG,
}) })
if output.Width() != tt.wantW || output.Height() != tt.wantH { if output.Width() != tt.wantW || output.Height() != tt.wantH {
@@ -684,8 +694,7 @@ func TestImageProcessor_ConvertsWideGamutToSRGB(t *testing.T) {
t.Fatalf("failed to read test JPEG: %v", err) t.Fatalf("failed to read test JPEG: %v", err)
} }
output := processAndDecode(t, input, output := processAndDecode(t, input, &Request{Format: FormatPNG})
&imagerequest.Transform{Format: imagerequest.FormatPNG})
if output.HasICCProfile() { if output.HasICCProfile() {
t.Error("output has an ICC profile") t.Error("output has an ICC profile")
@@ -8,7 +8,6 @@ import (
"testing" "testing"
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestImageProcessor_EncodeJPEGXL converts a JPEG to a smaller JPEG XL and // TestImageProcessor_EncodeJPEGXL converts a JPEG to a smaller JPEG XL and
@@ -16,17 +15,10 @@ import (
func TestImageProcessor_EncodeJPEGXL(t *testing.T) { func TestImageProcessor_EncodeJPEGXL(t *testing.T) {
t.Parallel() t.Parallel()
transform := &imagerequest.Transform{ req := &Request{Size: Size{Width: 100, Height: 75}, Format: FormatJXL}
Size: imagerequest.Size{Width: 100, Height: 75},
Format: imagerequest.FormatJXL,
}
result, err := New(Params{ result, err := New(Params{}).Process(
MaxInputBytes: 50 << 20, t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), req,
MaxConcurrentProcessing: 1,
}).Process(
t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)),
imagerequest.FormatJPEG, transform,
) )
if err != nil { if err != nil {
t.Fatalf("Process() error = %v", err) t.Fatalf("Process() error = %v", err)
@@ -68,12 +60,8 @@ func TestImageProcessor_JPEGXLQuality(t *testing.T) {
outputBytes := make(map[int]int64) outputBytes := make(map[int]int64)
for _, quality := range []int{30, 90} { for _, quality := range []int{30, 90} {
result, err := New(Params{ result, err := New(Params{}).Process(t.Context(),
MaxInputBytes: 50 << 20, bytes.NewReader(input), &Request{Format: FormatJXL, Quality: quality})
MaxConcurrentProcessing: 1,
}).Process(t.Context(),
bytes.NewReader(input), imagerequest.FormatJPEG,
&imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: quality})
if err != nil { if err != nil {
t.Fatalf("Process() at quality %d error = %v", quality, err) t.Fatalf("Process() at quality %d error = %v", quality, err)
} }
@@ -101,8 +89,7 @@ func TestImageProcessor_JPEGXLDropsSourceEXIF(t *testing.T) {
t.Fatalf("failed to read test JPEG: %v", err) t.Fatalf("failed to read test JPEG: %v", err)
} }
exif := processAndDecode(t, input, exif := processAndDecode(t, input, &Request{Format: FormatJXL}).GetExif()
&imagerequest.Transform{Format: imagerequest.FormatJXL}).GetExif()
for _, field := range []string{ for _, field := range []string{
"exif-ifd0-Make", "exif-ifd0-Model", "exif-ifd2-BodySerialNumber", "exif-ifd0-Make", "exif-ifd0-Model", "exif-ifd2-BodySerialNumber",
@@ -128,8 +115,7 @@ func TestImageProcessor_JPEGXLDropsSourceResolution(t *testing.T) {
t.Fatalf("failed to read test JPEG: %v", err) t.Fatalf("failed to read test JPEG: %v", err)
} }
output := processAndDecode(t, input, output := processAndDecode(t, input, &Request{Format: FormatJXL})
&imagerequest.Transform{Format: imagerequest.FormatJXL})
// libvips gives the resolution in pixels per millimetre. // libvips gives the resolution in pixels per millimetre.
xDPI := math.Round(output.ResX() * 25.4) xDPI := math.Round(output.ResX() * 25.4)
@@ -154,8 +140,7 @@ func TestImageProcessor_JPEGXLAppliesEXIFOrientation(t *testing.T) {
t.Fatalf("failed to read test JPEG: %v", err) t.Fatalf("failed to read test JPEG: %v", err)
} }
output := processAndDecode(t, input, output := processAndDecode(t, input, &Request{Format: FormatJXL})
&imagerequest.Transform{Format: imagerequest.FormatJXL})
if output.Width() != 8 || output.Height() != 16 { if output.Width() != 8 || output.Height() != 16 {
t.Fatalf("output is %dx%d, want 8x16", output.Width(), output.Height()) t.Fatalf("output is %dx%d, want 8x16", output.Width(), output.Height())
@@ -191,8 +176,7 @@ func TestImageProcessor_JPEGXLConvertsWideGamutToSRGB(t *testing.T) {
t.Fatalf("failed to read test JPEG: %v", err) t.Fatalf("failed to read test JPEG: %v", err)
} }
output := processAndDecode(t, input, output := processAndDecode(t, input, &Request{Format: FormatJXL})
&imagerequest.Transform{Format: imagerequest.FormatJXL})
pixel, err := output.GetPoint(4, 4) pixel, err := output.GetPoint(4, 4)
if err != nil { if err != nil {
@@ -219,8 +203,7 @@ func TestImageProcessor_JPEGXLFromCMYK(t *testing.T) {
t.Fatalf("failed to read test JPEG: %v", err) t.Fatalf("failed to read test JPEG: %v", err)
} }
output := processAndDecode(t, input, output := processAndDecode(t, input, &Request{Format: FormatJXL})
&imagerequest.Transform{Format: imagerequest.FormatJXL})
if output.Bands() != 3 { if output.Bands() != 3 {
t.Fatalf("output has %d bands, want 3", output.Bands()) t.Fatalf("output has %d bands, want 3", output.Bands())
@@ -5,13 +5,12 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"runtime"
"strings" "strings"
"sync" "sync"
"testing" "testing"
"testing/iotest" "testing/iotest"
"time" "time"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// errTestReadFailed is the error the unreadable test input returns. // errTestReadFailed is the error the unreadable test input returns.
@@ -77,13 +76,13 @@ func (r *gatedReader) Read(p []byte) (int, error) {
return n, err return n, err
} }
// smallJPEGTransform asks for a 5x5 JPEG. // smallJPEGRequest asks for a 5x5 JPEG.
func smallJPEGTransform() *imagerequest.Transform { func smallJPEGRequest() *Request {
return &imagerequest.Transform{ return &Request{
Size: imagerequest.Size{Width: 5, Height: 5}, Size: Size{Width: 5, Height: 5},
Format: imagerequest.FormatJPEG, Format: FormatJPEG,
Quality: 85, Quality: 85,
FitMode: imagerequest.FitCover, FitMode: FitCover,
} }
} }
@@ -93,8 +92,7 @@ func processInBackground(
proc *ImageProcessor, reader *gatedReader, results chan<- error, proc *ImageProcessor, reader *gatedReader, results chan<- error,
) { ) {
go func() { go func() {
result, err := proc.Process(context.Background(), reader, result, err := proc.Process(context.Background(), reader, smallJPEGRequest())
imagerequest.FormatJPEG, smallJPEGTransform())
if err == nil { if err == nil {
_ = result.Content.Close() _ = result.Content.Close()
} }
@@ -117,10 +115,18 @@ func waitForEntries(t *testing.T, entered <-chan struct{}, count int) {
} }
} }
func TestNewMakesMaxConcurrentProcessingSlots(t *testing.T) { func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 3}) for _, limit := range []int{0, -1} {
proc := New(Params{MaxConcurrentProcessing: limit})
if got := cap(proc.processingSemaphore); got != runtime.GOMAXPROCS(0) {
t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, one per CPU",
limit, got, runtime.GOMAXPROCS(0))
}
}
proc := New(Params{MaxConcurrentProcessing: 3})
if got := cap(proc.processingSemaphore); got != 3 { if got := cap(proc.processingSemaphore); got != 3 {
t.Errorf("MaxConcurrentProcessing 3: %d slots, want 3", got) t.Errorf("MaxConcurrentProcessing 3: %d slots, want 3", got)
} }
@@ -139,7 +145,7 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) {
calls = 6 calls = 6
) )
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: limit}) proc := New(Params{MaxConcurrentProcessing: limit})
input := createTestJPEG(t, 50, 50) input := createTestJPEG(t, 50, 50)
counter := &readingCounter{} counter := &readingCounter{}
@@ -186,7 +192,7 @@ func TestProcessNeverExceedsMaxConcurrentProcessing(t *testing.T) {
func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) { func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{MaxConcurrentProcessing: 1})
proc.processingWaitTimeout = 100 * time.Millisecond proc.processingWaitTimeout = 100 * time.Millisecond
input := createTestJPEG(t, 10, 10) input := createTestJPEG(t, 10, 10)
@@ -207,7 +213,7 @@ func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) {
start := time.Now() start := time.Now()
_, err := proc.Process(context.Background(), bytes.NewReader(input), _, err := proc.Process(context.Background(), bytes.NewReader(input),
imagerequest.FormatJPEG, smallJPEGTransform()) smallJPEGRequest())
if !errors.Is(err, ErrTooManyImages) { if !errors.Is(err, ErrTooManyImages) {
t.Fatalf("Process() error = %v, want ErrTooManyImages", err) t.Fatalf("Process() error = %v, want ErrTooManyImages", err)
} }
@@ -233,37 +239,37 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
valid := createTestJPEG(t, 10, 10) valid := createTestJPEG(t, 10, 10)
unsupported := smallJPEGTransform() unsupported := smallJPEGRequest()
unsupported.Format = "bmp" unsupported.Format = "bmp"
cases := []struct { cases := []struct {
name string name string
input io.Reader input io.Reader
transform *imagerequest.Transform req *Request
// want is the error Process must return; nil means any error. // want is the error Process must return; nil means any error.
want error want error
}{ }{
{ {
name: "input cannot be read", name: "input cannot be read",
input: iotest.ErrReader(errTestReadFailed), input: iotest.ErrReader(errTestReadFailed),
transform: smallJPEGTransform(), req: smallJPEGRequest(),
want: errTestReadFailed, want: errTestReadFailed,
}, },
{ {
name: "input over the byte limit", name: "input over the byte limit",
input: bytes.NewReader(createTestJPEG(t, 800, 600)), input: bytes.NewReader(createTestJPEG(t, 800, 600)),
transform: smallJPEGTransform(), req: smallJPEGRequest(),
want: ErrInputDataTooLarge, want: ErrInputDataTooLarge,
}, },
{ {
name: "input not an image", name: "input not an image",
input: strings.NewReader("not an image"), input: strings.NewReader("not an image"),
transform: smallJPEGTransform(), req: smallJPEGRequest(),
}, },
{ {
name: "output format not supported", name: "output format not supported",
input: bytes.NewReader(valid), input: bytes.NewReader(valid),
transform: unsupported, req: unsupported,
want: ErrUnsupportedOutputFormat, want: ErrUnsupportedOutputFormat,
}, },
} }
@@ -275,8 +281,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1}) proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1})
proc.processingWaitTimeout = 100 * time.Millisecond proc.processingWaitTimeout = 100 * time.Millisecond
_, err := proc.Process(context.Background(), tc.input, _, err := proc.Process(context.Background(), tc.input, tc.req)
imagerequest.FormatJPEG, tc.transform)
if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) {
t.Fatalf("Process() error = %v, want %v", err, tc.want) t.Fatalf("Process() error = %v, want %v", err, tc.want)
} }
@@ -286,7 +291,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
} }
result, err := proc.Process(context.Background(), bytes.NewReader(valid), result, err := proc.Process(context.Background(), bytes.NewReader(valid),
imagerequest.FormatJPEG, smallJPEGTransform()) smallJPEGRequest())
if err != nil { if err != nil {
t.Fatalf("Process() after the error = %v, want nil", err) t.Fatalf("Process() after the error = %v, want nil", err)
} }
@@ -303,7 +308,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
func TestWaitForProcessing(t *testing.T) { func TestWaitForProcessing(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 2}) proc := New(Params{MaxConcurrentProcessing: 2})
gate := make(chan struct{}) gate := make(chan struct{})
entered := make(chan struct{}, 1) entered := make(chan struct{}, 1)
@@ -6,13 +6,11 @@ import (
"testing" "testing"
"github.com/davidbyttow/govips/v2/vips" "github.com/davidbyttow/govips/v2/vips"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF // TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF
// input is refused with ErrUnsupportedInputFormat before it is decoded, and // input, which is not a source format, is refused with
// not saved as JPEG. TIFF is not a source format, and the test passes the // ErrUnsupportedInputFormat before it is decoded, and not saved as JPEG.
// input to Process as a JPEG.
func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) { func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) {
t.Parallel() t.Parallel()
@@ -28,12 +26,8 @@ func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) {
t.Fatalf("failed to save test TIFF: %v", err) t.Fatalf("failed to save test TIFF: %v", err)
} }
result, err := New(Params{ result, err := New(Params{}).Process(
MaxInputBytes: 50 << 20, t.Context(), bytes.NewReader(tiff), &Request{Format: FormatOriginal},
MaxConcurrentProcessing: 1,
}).Process(
t.Context(), bytes.NewReader(tiff), imagerequest.FormatJPEG,
&imagerequest.Transform{Format: imagerequest.FormatOriginal},
) )
if result != nil { if result != nil {
t.Logf("Process() returned %s", result.ContentType) t.Logf("Process() returned %s", result.ContentType)
@@ -2,11 +2,8 @@ package imageprocessor
import ( import (
"bytes" "bytes"
"errors"
"io" "io"
"testing" "testing"
"sneak.berlin/go/pixa/internal/imagerequest"
) )
// TestImageProcessor_LoadsEachSourceFormat verifies that an image in each of // TestImageProcessor_LoadsEachSourceFormat verifies that an image in each of
@@ -15,15 +12,13 @@ import (
func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) { func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) {
t.Parallel() t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1}) proc := New(Params{})
for _, format := range []imagerequest.Format{ for _, format := range []Format{
imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF, FormatJPEG, FormatPNG, FormatGIF, FormatWebP, FormatAVIF, FormatJXL,
imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL,
} { } {
saved, err := proc.Process(t.Context(), saved, err := proc.Process(t.Context(),
bytes.NewReader(createTestPNG(t, 20, 20)), imagerequest.FormatPNG, bytes.NewReader(createTestPNG(t, 20, 20)), &Request{Format: format})
&imagerequest.Transform{Format: format})
if err != nil { if err != nil {
t.Fatalf("saving the test image as %s: %v", format, err) t.Fatalf("saving the test image as %s: %v", format, err)
} }
@@ -33,8 +28,8 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) {
t.Fatalf("reading the %s image: %v", format, err) t.Fatalf("reading the %s image: %v", format, err)
} }
result, err := proc.Process(t.Context(), bytes.NewReader(source), format, result, err := proc.Process(t.Context(), bytes.NewReader(source),
&imagerequest.Transform{Format: imagerequest.FormatOriginal}) &Request{Format: FormatOriginal})
if err != nil { if err != nil {
t.Errorf("orig of the %s image: %v", format, err) t.Errorf("orig of the %s image: %v", format, err)
@@ -43,33 +38,9 @@ func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) {
t.Logf("orig of the %s image: %s", format, result.ContentType) t.Logf("orig of the %s image: %s", format, result.ContentType)
if result.ContentType != format.MIMEType() { if result.ContentType != FormatToMIME(format) {
t.Errorf("orig of the %s image is %s, want %s", t.Errorf("orig of the %s image is %s, want %s",
format, result.ContentType, format.MIMEType()) format, result.ContentType, FormatToMIME(format))
}
}
}
// TestImageProcessor_NotASourceFormatRefused verifies that a source format
// other than the six, such as orig, auto or none, is refused with
// ErrUnsupportedInputFormat. The input is text, which libvips reads as no
// format.
func TestImageProcessor_NotASourceFormatRefused(t *testing.T) {
t.Parallel()
proc := New(Params{MaxInputBytes: 50 << 20, MaxConcurrentProcessing: 1})
for _, sourceFormat := range []imagerequest.Format{
imagerequest.FormatOriginal, imagerequest.FormatAuto, "",
} {
result, err := proc.Process(t.Context(),
bytes.NewReader([]byte("this is text, not an image")), sourceFormat,
&imagerequest.Transform{Format: imagerequest.FormatJPEG})
t.Logf("source format %q: error = %v", sourceFormat, err)
if result != nil || !errors.Is(err, ErrUnsupportedInputFormat) {
t.Errorf("Process() with source format %q = %v, error %v; want %v",
sourceFormat, result, err, ErrUnsupportedInputFormat)
} }
} }
} }
@@ -1,100 +0,0 @@
package imageproxy
import (
"bytes"
"os"
"testing"
"time"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
)
// TestProxy_Get_MakesAgainVariantOfNoOutputFormat replaces a cached
// variant's file with text, and checks that the next request is a miss
// answered with the variant made again, and the one after a hit.
func TestProxy_Get_MakesAgainVariantOfNoOutputFormat(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
req := photoVariant(fixtures, 85, imagerequest.FitCover)
want, _ := getImage(t, svc, req)
path := variantFilePath(svc, imagerequest.CacheKey(req))
err := os.WriteFile(path, []byte("this is text, not an image"),
imgcache.StorageFilePerm)
if err != nil {
t.Fatalf("replacing the variant file: %v", err)
}
got, status := getImage(t, svc, req)
t.Logf("after the replacement: %d bytes, %s", len(got), status)
if status != CacheMiss || !bytes.Equal(got, want) {
t.Errorf("after the replacement: Get() = %d bytes, %s; want the %d bytes "+
"made again, %s", len(got), status, len(want), CacheMiss)
}
got, status = getImage(t, svc, req)
if status != CacheHit || !bytes.Equal(got, want) {
t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s",
len(got), status, len(want), CacheHit)
}
}
// TestProxy_Get_HitWaitsForNoDatabaseWrite holds the database's one
// connection while a cached variant is requested. The hit must be answered
// while the connection is still held, and its access time must reach the
// database once it is free.
func TestProxy_Get_HitWaitsForNoDatabaseWrite(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
req := photoVariant(fixtures, 85, imagerequest.FitCover)
key := imagerequest.CacheKey(req)
getImage(t, svc, req)
// Each connection to an in-memory SQLite database opens a new, empty one,
// so the goroutine that writes the access times must share the
// connection that holds the schema.
svc.db.SetMaxOpenConns(1)
svc.cache.StartPendingAccessTimeWrites()
defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }()
anHourAgo := time.Now().Add(-time.Hour)
setVariantLastAccessed(t, svc.db, key, anHourAgo)
releaseDatabase := holdDatabase(t, svc.db)
defer releaseDatabase()
results := startGet(t.Context(), svc, req)
select {
case got := <-results:
if got.err != nil {
t.Fatalf("Get() of the cached variant error = %v", got.err)
}
case <-time.After(pendingAccessTimesWait):
t.Fatal("the cache hit waited for the database")
}
releaseDatabase()
deadline := time.Now().Add(pendingAccessTimesWait)
for countRows(t, svc.db, `
SELECT COUNT(*) FROM variant_content
WHERE cache_key = ? AND last_accessed_at > ?
`, string(key), anHourAgo.UTC().Format(sqliteTimestampFormat)) == 0 {
if time.Now().After(deadline) {
t.Fatal("the hit's access time was not written")
}
time.Sleep(10 * time.Millisecond)
}
}
@@ -1,77 +0,0 @@
package imageproxy
import (
"errors"
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
"sneak.berlin/go/pixa/internal/imgcache"
)
// TestNew_RegistersItsCountersOnItsRegistry checks that New registers the
// proxy's counters on the registry it is given, under the names README.md
// gives them. The cache registers its own counters on another registry.
func TestNew_RegistersItsCountersOnItsRegistry(t *testing.T) {
t.Parallel()
cache, err := imgcache.NewCache(setupTestDB(t),
imgcache.Config{StateDir: t.TempDir(), MaxBytes: 1 << 30},
prometheus.NewRegistry())
if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err)
}
registry := prometheus.NewRegistry()
_, err = New(&Config{Cache: cache}, registry)
if err != nil {
t.Fatalf("New() error = %v", err)
}
names := []string{
"pixa_upstream_fetches_total",
"pixa_upstream_fetch_bytes_total",
"pixa_transforms_total",
}
count, err := testutil.GatherAndCount(registry, names...)
if err != nil {
t.Fatalf("testutil.GatherAndCount() error = %v", err)
}
if count != len(names) {
t.Errorf("%d of the counters %v are on the registry, want %d",
count, names, len(names))
}
}
// TestNew_FailsWhenItsCountersCannotBeRegistered makes a second proxy on
// the registry of a first. Its counters have the same names, so the
// registry refuses them, and New must return that error.
func TestNew_FailsWhenItsCountersCannotBeRegistered(t *testing.T) {
t.Parallel()
cache, err := imgcache.NewCache(setupTestDB(t),
imgcache.Config{StateDir: t.TempDir(), MaxBytes: 1 << 30},
prometheus.NewRegistry())
if err != nil {
t.Fatalf("imgcache.NewCache() error = %v", err)
}
registry := prometheus.NewRegistry()
_, err = New(&Config{Cache: cache}, registry)
if err != nil {
t.Fatalf("New() of the first proxy error = %v", err)
}
_, err = New(&Config{Cache: cache}, registry)
t.Logf("New() of the second proxy error = %v", err)
var alreadyRegistered prometheus.AlreadyRegisteredError
if !errors.As(err, &alreadyRegistered) {
t.Errorf("New() of the second proxy error = %v, want %T",
err, alreadyRegistered)
}
}
-592
View File
@@ -1,592 +0,0 @@
// Package imageproxy fetches, transforms, caches and serves images. Its
// Proxy serves an image from the cache when it can, and otherwise makes it
// from the cached or fetched source image.
package imageproxy
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log/slog"
"runtime/debug"
"time"
"github.com/dustin/go-humanize"
"github.com/getsentry/sentry-go"
"github.com/go-chi/chi/v5/middleware"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/sync/singleflight"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
"sneak.berlin/go/pixa/internal/magic"
)
// Proxy orchestrates the cache, fetcher, and processor.
type Proxy struct {
cache *imgcache.Cache
fetcher httpfetcher.Fetcher
processor *imageprocessor.ImageProcessor
log *slog.Logger
maxResponseSize int64
// variantsInProgress lets the requests that miss the same variant at the
// same time share one fetch and one transcode.
variantsInProgress singleflight.Group
// The counts of upstream fetches and transforms, made and registered by
// New. They are kept in memory, so adding to one waits for nothing.
upstreamFetches prometheus.Counter
upstreamFetchBytes prometheus.Counter
transforms prometheus.Counter
}
// Config holds the proxy's configuration.
type Config struct {
// Cache is the cache instance
Cache *imgcache.Cache
// Fetcher fetches source images from upstream hosts
Fetcher httpfetcher.Fetcher
// Processor converts and resizes the images
Processor *imageprocessor.ImageProcessor
// MaxResponseSize is the largest source accepted, in bytes; a cached
// source larger than this is fetched again
MaxResponseSize int64
// Logger for logging
Logger *slog.Logger
}
// errCacheRequired is returned when the proxy is built without a cache.
var errCacheRequired = errors.New("cache is required")
// New creates a new image proxy and registers its counters on
// metricsRegistry. pixad passes Prometheus' default registry, the one
// /metrics serves. It fails when metricsRegistry already holds counters of
// the same names.
func New(
cfg *Config, metricsRegistry prometheus.Registerer,
) (*Proxy, error) {
if cfg.Cache == nil {
return nil, errCacheRequired
}
log := cfg.Logger
if log == nil {
log = slog.Default()
}
// JPEG XL is the default output format, so pixad does not start
// without it.
err := imageprocessor.CheckJPEGXLSupport()
if err != nil {
return nil, err
}
p := &Proxy{
cache: cfg.Cache,
fetcher: cfg.Fetcher,
processor: cfg.Processor,
log: log,
maxResponseSize: cfg.MaxResponseSize,
upstreamFetches: prometheus.NewCounter(prometheus.CounterOpts{
Name: "pixa_upstream_fetches_total",
Help: "Fetches of a source image from upstream that read any bytes.",
}),
upstreamFetchBytes: prometheus.NewCounter(prometheus.CounterOpts{
Name: "pixa_upstream_fetch_bytes_total",
Help: "Bytes read from upstream by fetches of source images.",
}),
transforms: prometheus.NewCounter(prometheus.CounterOpts{
Name: "pixa_transforms_total",
Help: "Images the image processor transformed.",
}),
}
for _, counter := range []prometheus.Counter{
p.upstreamFetches, p.upstreamFetchBytes, p.transforms,
} {
err = metricsRegistry.Register(counter)
if err != nil {
return nil, fmt.Errorf("failed to register the proxy's counters: %w", err)
}
}
return p, nil
}
// ErrNegativeCached is returned when a URL is in the negative cache (recently failed).
var ErrNegativeCached = errors.New("request is in negative cache (recently failed)")
// ErrProcessingTimedOut is returned when processing an image stopped because
// the downstream_timeout of the request doing it ended, to that request and to
// every request that waited for its image.
var ErrProcessingTimedOut = errors.New(
"image processing stopped at the deadline of the request doing it")
// ImageResponse represents a processed image ready to serve
type ImageResponse struct {
// Content is the image data reader
Content io.ReadCloser
// ContentLength is the size in bytes (-1 if unknown)
ContentLength int64
// ContentType is the MIME type of the response
ContentType string
// ETag is the entity tag for caching
ETag string
// CacheStatus indicates HIT or MISS
CacheStatus CacheStatus
// FetchedBytes is the number of bytes fetched from upstream (0 if cache hit)
FetchedBytes int64
}
// CacheStatus indicates whether the response was served from cache.
type CacheStatus string
// Cache status values for response headers.
const (
CacheHit CacheStatus = "HIT"
CacheMiss CacheStatus = "MISS"
)
// Get retrieves a processed image, fetching and processing if necessary.
// While its source URL is in the negative cache, a cached variant is still
// served and a variant not in the cache is still made from a cached source;
// only a request that needs the source fetched is refused.
//
// A cache hit opens the variant file and makes no database call: the cache
// hands its access time to the goroutine that writes access times, without
// waiting for it, and its hit is counted in memory. A variant file that
// cannot be served, such as one whose first bytes are not those of an output
// format, is a miss, and the variant is made again.
func (p *Proxy) Get(
ctx context.Context, req *imagerequest.Request,
) (*ImageResponse, error) {
cacheKey := imagerequest.CacheKey(req)
reader, size, contentType, err := p.cache.Variant(cacheKey)
if err == nil {
p.cache.IncrementStats(true)
return &ImageResponse{
Content: reader,
ContentLength: size,
ContentType: contentType,
CacheStatus: CacheHit,
ETag: formatETag(cacheKey),
}, nil
}
if !errors.Is(err, imgcache.ErrNotFound) {
p.log.Warn("cached variant not served, making it again",
"cache_key", cacheKey, "error", err)
}
// Cache miss - get the variant, processed once for all the requests that
// miss it at the same time, then count this request's miss, also when it
// failed or the request context has ended meanwhile
response, err := p.processOrWait(ctx, req)
p.cache.IncrementStats(false)
if err != nil {
return nil, err
}
response.CacheStatus = CacheMiss
return response, nil
}
// WaitForProcessing waits until no image is being processed, or until ctx
// ends, and returns how many images were still being processed then.
func (p *Proxy) WaitForProcessing(ctx context.Context) int {
return p.processor.WaitForProcessing(ctx)
}
// errPanicked is returned when processing a variant panicked.
var errPanicked = errors.New("panic while processing image")
// processOrWait returns the variant req asks for. The first of the requests
// that miss a variant at the same time processes it, and singleflight hands
// its result, or its error, to the others: they fetch nothing, read no source
// and take no upstream connection or processing slot. The processing ignores
// the first request's cancellation, so the others are still served if that
// client goes away, but keeps its deadline.
func (p *Proxy) processOrWait(
ctx context.Context, req *imagerequest.Request,
) (*ImageResponse, error) {
// A request that has already ended starts no processing
if ctx.Err() != nil {
return nil, ctx.Err()
}
cacheKey := imagerequest.CacheKey(req)
// Closed when this request's own function runs, which singleflight does
// only when no other request is processing the variant
processing := make(chan struct{})
results := p.variantsInProgress.DoChan(string(cacheKey),
func() (_ any, err error) {
close(processing)
// singleflight would raise a panic again in a goroutine of its
// own, where no handler recovers it, and stop pixad. It is
// reported through the Sentry hub that the Sentry middleware puts
// on the request's context when sentry_dsn is set.
defer func() {
recovered := recover()
if recovered != nil {
p.log.Error("panic while processing image",
"host", req.Source.Host, "path", req.Source.Path,
"panic", recovered, "stack", string(debug.Stack()))
if hub := sentry.GetHubFromContext(ctx); hub != nil {
hub.RecoverWithContext(ctx, recovered)
}
err = fmt.Errorf("%w: %v", errPanicked, recovered)
}
}()
processingCtx, cancel := withoutCancelKeepingDeadline(ctx)
defer cancel()
return p.processFromSourceOrFetch(processingCtx, req, cacheKey)
})
var result singleflight.Result
select {
case result = <-results:
case <-ctx.Done():
select {
case <-processing:
// This request is processing the variant: it waits for the
// result, as every request did before misses were shared
result = <-results
default:
// Another request is processing the variant, or this request's
// function has not started yet; the processing goes on without it
return nil, ctx.Err()
}
}
if result.Err != nil {
return nil, result.Err
}
variant, _ := result.Val.(*processedVariant)
return &ImageResponse{
Content: io.NopCloser(bytes.NewReader(variant.data)),
ContentLength: int64(len(variant.data)),
ContentType: variant.contentType,
FetchedBytes: variant.fetchedBytes,
ETag: formatETag(cacheKey),
}, nil
}
// withoutCancelKeepingDeadline returns ctx without its cancellation but with
// its deadline, if it has one, and the func that releases the returned
// context.
func withoutCancelKeepingDeadline(
ctx context.Context,
) (context.Context, context.CancelFunc) {
deadline, hasDeadline := ctx.Deadline()
if !hasDeadline {
return context.WithoutCancel(ctx), func() {}
}
return context.WithDeadline(context.WithoutCancel(ctx), deadline)
}
// loadCachedSource opens src's cached source, reading only the first bytes,
// which give its format, and returns it with its size and its format. It
// returns nil if the source is not in the cache, cannot be read, is in no
// source format or exceeds maxResponseSize.
func (p *Proxy) loadCachedSource(
ctx context.Context, src imagerequest.Source,
) (io.ReadCloser, int64, imagerequest.Format) {
reader, size, format, err := p.cache.Source(ctx, src)
if errors.Is(err, imgcache.ErrNotFound) {
return nil, 0, ""
}
if err != nil {
p.log.Warn("failed to load cached source, fetching", "error", err)
return nil, 0, ""
}
p.log.Debug("using cached source", "source_key", src.Key())
if size > p.maxResponseSize {
_ = reader.Close()
p.log.Warn("cached source exceeds max response size, discarding",
"source_key", src.Key(),
"max_bytes", p.maxResponseSize,
)
return nil, 0, ""
}
return reader, size, format
}
// processFromSourceOrFetch processes an image, using cached source content
// if available.
func (p *Proxy) processFromSourceOrFetch(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
) (*processedVariant, error) {
source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source)
// Fetch from upstream if we don't have source data or it's empty
if source == nil {
return p.fetchAndProcess(ctx, req, cacheKey)
}
defer func() { _ = source.Close() }()
// Process using cached source; nothing was fetched from upstream. The
// image processor reads the source only once it has a processing slot,
// so a request waiting for one holds none of it in memory.
return p.processAndStore(ctx, req, cacheKey, source, sourceFormat, sourceSize)
}
// fetchAndProcess fetches from upstream, processes, and caches the result.
// It counts the fetch with the bytes read from upstream, including when
// reading the response or a later step fails. It fetches nothing while the
// source URL is in the negative cache.
func (p *Proxy) fetchAndProcess(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
) (*processedVariant, error) {
// Check negative cache - skip fetching for recently-failed URLs
negHit, err := p.cache.Failure(ctx, req.Source)
if err != nil {
p.log.Warn("negative cache check failed", "error", err)
}
if negHit {
p.log.Debug("negative cache hit",
"host", req.Source.Host,
"path", req.Source.Path,
)
return nil, fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamError, ErrNegativeCached)
}
// Fetch from upstream
p.log.Debug("fetching from upstream", "host", req.Source.Host,
"path", req.Source.Path, "query", req.Source.Query)
fetchResult, err := p.fetcher.Fetch(ctx, req.Source)
if err != nil {
// Store negative cache for certain errors
if isNegativeCacheable(err) {
p.storeNegative(ctx, req)
}
return nil, fmt.Errorf("upstream fetch failed: %w", err)
}
// Closing the body frees the upstream connection. It is closed only
// after processing, so the fetcher's connection limit also bounds the
// fetched sources held in memory while their requests wait for a
// processing slot.
defer func() { _ = fetchResult.Content.Close() }()
// Read and validate the source content
sourceData, err := io.ReadAll(fetchResult.Content)
fetchBytes := int64(len(sourceData))
// Counted also when the request context has ended meanwhile
p.incrementUpstreamFetch(fetchBytes)
if err != nil {
return nil, fmt.Errorf("failed to read upstream response: %w", err)
}
// Calculate download bitrate
var downloadRate string
if fetchResult.FetchDurationMs > 0 {
seconds := float64(fetchResult.FetchDurationMs) / 1000.0 //nolint:mnd // ms to seconds
bitsPerSecond := float64(fetchBytes*8) / seconds //nolint:mnd // bytes to bits
downloadRate = humanize.SI(bitsPerSecond, "bps")
}
// Log upstream fetch details
p.log.Info("upstream fetched",
"request_id", middleware.GetReqID(ctx),
"host", req.Source.Host,
"path", req.Source.Path,
"bytes", fetchBytes,
"fetch_ms", fetchResult.FetchDurationMs,
"rate", downloadRate,
"remote_addr", fetchResult.RemoteAddr,
"http", fetchResult.HTTPVersion,
"tls", fetchResult.TLSVersion,
"cipher", fetchResult.TLSCipherSuite,
)
// The source's format, detected from its first bytes, which must match
// its content type. A body that does not is the upstream's error, so it
// gets a negative cache entry, as an upstream error status does.
sourceFormat, err := magic.ValidateMagicBytes(sourceData,
fetchResult.ContentType)
if err != nil {
p.log.Warn("source refused", "request_id", middleware.GetReqID(ctx),
"host", req.Source.Host, "path", req.Source.Path,
"content_type", fetchResult.ContentType, "error", err)
p.storeNegative(ctx, req)
return nil, fmt.Errorf("content validation failed: %w", err)
}
// Store source content
_, err = p.cache.PutSource(ctx, req.Source, bytes.NewReader(sourceData))
if err != nil {
p.log.Warn("failed to store source content", "error", err)
// Continue even if caching fails
}
return p.processAndStore(
ctx, req, cacheKey, bytes.NewReader(sourceData), sourceFormat, fetchBytes,
)
}
// incrementUpstreamFetch counts one upstream fetch that read fetchBytes bytes.
// A fetch that read no bytes is not counted.
func (p *Proxy) incrementUpstreamFetch(fetchBytes int64) {
if fetchBytes <= 0 {
return
}
p.upstreamFetches.Inc()
p.upstreamFetchBytes.Add(float64(fetchBytes))
}
// storeNegative stores a negative cache entry for req's source, whose fetch
// failed, so the source is not fetched again until the entry expires. A
// failure to store the entry is only logged.
func (p *Proxy) storeNegative(ctx context.Context, req *imagerequest.Request) {
err := p.cache.PutFailure(ctx, req.Source)
if err != nil {
p.log.Warn("failed to store negative cache entry",
"host", req.Source.Host, "path", req.Source.Path, "error", err)
}
}
// processedVariant is a variant as processAndStore made it. Each request that
// shared its processing serves it through a reader of its own.
type processedVariant struct {
data []byte
contentType string
fetchedBytes int64
}
// processAndStore processes the image read from source, which is in
// sourceFormat, and stores the result.
func (p *Proxy) processAndStore(
ctx context.Context,
req *imagerequest.Request,
cacheKey imagerequest.VariantKey,
source io.Reader,
sourceFormat imagerequest.Format,
fetchBytes int64,
) (*processedVariant, error) {
// Process the image
processStart := time.Now()
// ctx has the deadline of the request doing the processing, which can
// end while a request waiting for its image has time left
processResult, err := p.processor.Process(ctx, source, sourceFormat,
&req.Transform)
if errors.Is(err, context.DeadlineExceeded) {
return nil, fmt.Errorf("%w: %w", ErrProcessingTimedOut, err)
}
if err != nil {
return nil, fmt.Errorf("image processing failed: %w", err)
}
processDuration := time.Since(processStart)
// Counted also when the request context has ended meanwhile
p.transforms.Inc()
// Read processed content
processedData, err := io.ReadAll(processResult.Content)
_ = processResult.Content.Close()
if err != nil {
return nil, fmt.Errorf("failed to read processed content: %w", err)
}
// Log conversion details
outputSize := int64(len(processedData))
var sizePercent float64
if fetchBytes > 0 {
//nolint:mnd // percentage calculation
sizePercent = float64(outputSize) / float64(fetchBytes) * 100.0
}
p.log.Info("image converted",
"request_id", middleware.GetReqID(ctx),
"host", req.Source.Host,
"path", req.Source.Path,
"src_format", sourceFormat,
"dst_format", req.Transform.Format,
"src_bytes", fetchBytes,
"dst_bytes", outputSize,
"src_dimensions", fmt.Sprintf("%dx%d",
processResult.InputWidth, processResult.InputHeight),
"dst_dimensions", fmt.Sprintf("%dx%d",
processResult.Width, processResult.Height),
"size_ratio", fmt.Sprintf("%.1f%%", sizePercent),
"convert_ms", processDuration.Milliseconds(),
"quality", req.Transform.Quality,
"fit", req.Transform.FitMode,
)
// Store variant to cache
err = p.cache.PutVariant(ctx, cacheKey, bytes.NewReader(processedData))
if err != nil {
p.log.Warn("failed to store variant", "error", err)
// Continue even if caching fails
}
return &processedVariant{
data: processedData,
contentType: processResult.ContentType,
fetchedBytes: fetchBytes,
}, nil
}
// isNegativeCacheable returns true if the error should be cached.
func isNegativeCacheable(err error) bool {
return errors.Is(err, httpfetcher.ErrUpstreamError)
}
// etagHashLength is the number of hash characters to use for ETags.
const etagHashLength = 16
// formatETag formats a VariantKey as a quoted ETag value.
func formatETag(key imagerequest.VariantKey) string {
hash := string(key)
// Use first 16 characters of hash for a shorter but still unique ETag
if len(hash) > etagHashLength {
hash = hash[:etagHashLength]
}
return `"` + hash + `"`
}
@@ -1,575 +0,0 @@
package imageproxy
import (
"context"
"io"
"os"
"testing"
"time"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/magic"
)
// Test data literals used repeatedly in this file (goconst).
const (
testPathPhoto = "/images/photo.jpg"
testPathUpload = "/uploads/image.jpg"
)
func TestProxy_Get_GoodHost(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
resp, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
defer func() { _ = resp.Content.Close() }()
// Verify we got content
data, err := io.ReadAll(resp.Content)
if err != nil {
t.Fatalf("failed to read response: %v", err)
}
if len(data) == 0 {
t.Error("expected non-empty response")
}
if resp.ContentType != testContentTypeJPEG {
t.Errorf("ContentType = %q, want %q", resp.ContentType, testContentTypeJPEG)
}
}
// TestProxy_Get_OtherHost checks that the proxy serves a second host as it
// serves GoodHost.
func TestProxy_Get_OtherHost(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.OtherHost,
Path: testPathUpload,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
// Should fetch successfully
resp, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
defer func() { _ = resp.Content.Close() }()
data, err := io.ReadAll(resp.Content)
if err != nil {
t.Fatalf("failed to read response: %v", err)
}
if len(data) == 0 {
t.Error("expected non-empty response")
}
}
func TestProxy_Get_InvalidFile(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := photoVariant(fixtures, 85, imagerequest.FitCover)
req.Source.Path = "/images/fake.jpg"
req.Transform.Size = imagerequest.Size{Width: 50, Height: 50}
// Should fail because magic bytes don't match
_, err := svc.Get(ctx, req)
if err == nil {
t.Error("Get() expected error for invalid file")
}
}
func TestProxy_Get_NotFound(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := photoVariant(fixtures, 85, imagerequest.FitCover)
req.Source.Path = "/images/nonexistent.jpg"
req.Transform.Size = imagerequest.Size{Width: 50, Height: 50}
_, err := svc.Get(ctx, req)
if err == nil {
t.Error("Get() expected error for nonexistent file")
}
}
func TestProxy_Get_FormatConversion(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
tests := []struct {
name string
sourcePath string
outFormat imagerequest.Format
wantMIME string
}{
{
name: "JPEG to PNG",
sourcePath: testPathPhoto,
outFormat: imagerequest.FormatPNG,
wantMIME: "image/png",
},
{
name: "PNG to JPEG",
sourcePath: "/images/logo.png",
outFormat: imagerequest.FormatJPEG,
wantMIME: testContentTypeJPEG,
},
{
name: "GIF to PNG",
sourcePath: "/images/animation.gif",
outFormat: imagerequest.FormatPNG,
wantMIME: "image/png",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
req := &imagerequest.Request{
Source: imagerequest.Source{Host: fixtures.GoodHost, Path: tt.sourcePath},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: tt.outFormat, Quality: 85, FitMode: imagerequest.FitCover,
},
}
resp, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
defer func() { _ = resp.Content.Close() }()
if resp.ContentType != tt.wantMIME {
t.Errorf("ContentType = %q, want %q", resp.ContentType, tt.wantMIME)
}
// Verify magic bytes match the expected format
data, err := io.ReadAll(resp.Content)
if err != nil {
t.Fatalf("failed to read response: %v", err)
}
detectedFormat, err := magic.DetectFormat(data)
if err != nil {
t.Fatalf("failed to detect format: %v", err)
}
if detectedFormat.MIMEType() != tt.wantMIME {
t.Errorf("detected format = %q, want the format of %s",
detectedFormat, tt.wantMIME)
}
})
}
}
func TestProxy_Get_Caching(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
// First request - should be a cache miss
resp1, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() first request error = %v", err)
}
if resp1.CacheStatus != CacheMiss {
t.Errorf("first request CacheStatus = %q, want %q", resp1.CacheStatus, CacheMiss)
}
data1, err := io.ReadAll(resp1.Content)
if err != nil {
t.Fatalf("failed to read first response: %v", err)
}
_ = resp1.Content.Close()
// Second request - should be a cache hit
resp2, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() second request error = %v", err)
}
if resp2.CacheStatus != CacheHit {
t.Errorf("second request CacheStatus = %q, want %q", resp2.CacheStatus, CacheHit)
}
data2, err := io.ReadAll(resp2.Content)
if err != nil {
t.Fatalf("failed to read second response: %v", err)
}
_ = resp2.Content.Close()
// Content should be identical
if len(data1) != len(data2) {
t.Errorf("response sizes differ: %d vs %d", len(data1), len(data2))
}
}
// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and
// a variant of it and sets back two days every time pixa stored for them.
// For the source: last_accessed_at in source_content, and the access and
// modification times of its file. For the variant: last_accessed_at in
// variant_content, and the access and modification times of its file. It
// then checks that the variant is still served from the cache and that a new
// variant is made from the cached source without fetching it again.
//
// Left at the present: the files' status change and birth times, which a
// program cannot set, and the times of the directories holding the files,
// which other entries share.
func TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
variant := photoVariant(fixtures, 85, imagerequest.FitCover)
resp, err := svc.Get(t.Context(), variant)
if err != nil {
t.Fatalf("Get() first request error = %v", err)
}
_ = resp.Content.Close()
twoDaysAgo := time.Now().AddDate(0, 0, -2)
backdateSource(t, svc, variant, twoDaysAgo)
backdateVariant(t, svc, variant, twoDaysAgo)
resp, err = svc.Get(t.Context(), variant)
if err != nil {
t.Fatalf("Get() of the cached variant error = %v", err)
}
_ = resp.Content.Close()
if resp.CacheStatus != CacheHit {
t.Errorf("cached variant CacheStatus = %q, want %q",
resp.CacheStatus, CacheHit)
}
// Another quality is a new variant, made from the cached source
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover))
if err != nil {
t.Fatalf("Get() of a new variant error = %v", err)
}
_ = resp.Content.Close()
fetches := readCounters(t, svc).upstreamFetchCount
if fetches != 1 {
t.Errorf("upstream fetches = %d, want 1", fetches)
}
}
// backdateSource sets every time pixa stored for req's cached source to when,
// as listed on TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo.
func backdateSource(
t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
) {
t.Helper()
key := req.Source.Key()
setSourceLastAccessed(t, svc.db, key, when)
path := sourceFilePath(svc, key)
err := os.Chtimes(path, when, when)
if err != nil {
t.Fatalf("setting back the times of %s: %v", path, err)
}
}
// backdateVariant sets every time pixa stored for the cached variant req asks
// for to when, as listed on
// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo.
func backdateVariant(
t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
) {
t.Helper()
key := imagerequest.CacheKey(req)
path := variantFilePath(svc, key)
setVariantLastAccessed(t, svc.db, key, when)
err := os.Chtimes(path, when, when)
if err != nil {
t.Fatalf("setting back the times of %s: %v", path, err)
}
}
func TestProxy_Get_DifferentSizes(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
// Request same image at different sizes
sizes := []imagerequest.Size{
{Width: 25, Height: 25},
{Width: 50, Height: 50},
{Width: 75, Height: 75},
}
responses := make([][]byte, 0, len(sizes))
for _, size := range sizes {
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: size,
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
resp, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() error for size %v = %v", size, err)
}
data, err := io.ReadAll(resp.Content)
if err != nil {
t.Fatalf("failed to read response: %v", err)
}
_ = resp.Content.Close()
responses = append(responses, data)
}
// All responses should be different sizes (different cache entries)
for i := range len(responses) - 1 {
if len(responses[i]) == len(responses[i+1]) {
// Not necessarily an error, but worth noting
t.Logf("responses %d and %d have same size: %d bytes",
i, i+1, len(responses[i]))
}
}
}
func TestProxy_Get_ContextCancellation(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
_, err := svc.Get(ctx, req)
if err == nil {
t.Error("Get() expected error for cancelled context")
}
}
func TestProxy_Get_ReturnsETag(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
resp, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() error = %v", err)
}
defer func() { _ = resp.Content.Close() }()
// ETag should be set
if resp.ETag == "" {
t.Error("ETag should be set in response")
}
// ETag should be properly quoted
if len(resp.ETag) < 2 || resp.ETag[0] != '"' || resp.ETag[len(resp.ETag)-1] != '"' {
t.Errorf("ETag should be quoted, got %q", resp.ETag)
}
}
func TestProxy_Get_ETagConsistency(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
// First request
resp1, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() first request error = %v", err)
}
etag1 := resp1.ETag
_ = resp1.Content.Close()
// Second request (from cache)
resp2, err := svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() second request error = %v", err)
}
etag2 := resp2.ETag
_ = resp2.Content.Close()
// ETags should be identical for the same content
if etag1 != etag2 {
t.Errorf("ETags should match: %q != %q", etag1, etag2)
}
}
func TestProxy_Get_DifferentETagsForDifferentContent(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx := context.Background()
// Request same image at different sizes - should get different ETags
req1 := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 25, Height: 25},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
req2 := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
resp1, err := svc.Get(ctx, req1)
if err != nil {
t.Fatalf("Get() first request error = %v", err)
}
etag1 := resp1.ETag
_ = resp1.Content.Close()
resp2, err := svc.Get(ctx, req2)
if err != nil {
t.Fatalf("Get() second request error = %v", err)
}
etag2 := resp2.ETag
_ = resp2.Content.Close()
// ETags should be different for different content
if etag1 == etag2 {
t.Error("ETags should differ for different sizes")
}
}
@@ -1,96 +0,0 @@
package imageproxy
import (
"errors"
"os"
"testing"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// TestProxy_Get_ServesCachedVariantOfNegativeCachedURL caches a variant of
// the test photo, then keeps a failed fetch of the photo's URL in the negative
// cache, as a failed fetch for another variant does once the source has been
// evicted. The cached variant is still served, and a variant not in the cache
// is made from the cached source. Once the source is evicted, a variant not in
// the cache needs a fetch and is refused.
func TestProxy_Get_ServesCachedVariantOfNegativeCachedURL(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
resp, err := svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover))
if err != nil {
t.Fatalf("Get() first request error = %v", err)
}
_ = resp.Content.Close()
err = svc.cache.PutFailure(t.Context(),
photoVariant(fixtures, 85, imagerequest.FitCover).Source)
if err != nil {
t.Fatalf("PutFailure() error = %v", err)
}
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 85, imagerequest.FitCover))
if err != nil {
t.Fatalf("Get() of the cached variant error = %v", err)
}
_ = resp.Content.Close()
if resp.CacheStatus != CacheHit {
t.Errorf("cached variant CacheStatus = %q, want %q",
resp.CacheStatus, CacheHit)
}
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover))
t.Logf("Get() of a variant made from the cached source: error = %v", err)
if err != nil {
t.Fatalf("Get() of a variant made from the cached source error = %v", err)
}
_ = resp.Content.Close()
source := photoVariant(fixtures, 85, imagerequest.FitCover).Source
reader, _, _, err := svc.cache.Source(t.Context(), source)
if err != nil {
t.Fatalf("Source() error = %v, want the cached source", err)
}
_ = reader.Close()
// Evicts the source as the cache's eviction does: its row, then its file
_, err = svc.db.ExecContext(t.Context(),
`DELETE FROM source_content WHERE source_key = ?`, string(source.Key()))
if err != nil {
t.Fatalf("deleting the source's row: %v", err)
}
err = os.Remove(sourceFilePath(svc, source.Key()))
if err != nil {
t.Fatalf("removing the source's file: %v", err)
}
_, err = svc.Get(t.Context(), photoVariant(fixtures, 60, imagerequest.FitCover))
t.Logf("Get() of a variant whose source was evicted: error = %v", err)
if !errors.Is(err, ErrNegativeCached) {
t.Errorf("Get() of a variant whose source was evicted error = %v, want %v",
err, ErrNegativeCached)
}
}
func TestProxy_Get_ReturnsErrorForNegativeCachedURL(t *testing.T) {
t.Parallel()
// This test verifies that Proxy.Get() checks the negative cache
// We can't easily test the full pipeline without vips, but we can
// verify the error type
err := ErrNegativeCached
if !errors.Is(err, ErrNegativeCached) {
t.Error("ErrNegativeCached should be identifiable with errors.Is")
}
}
@@ -1,184 +0,0 @@
package imageproxy
import (
"bytes"
"io"
"os"
"testing"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
)
// getImage calls Get for req and returns the image it answered with and
// its cache status.
func getImage(
t *testing.T, svc *testProxy, req *imagerequest.Request,
) ([]byte, CacheStatus) {
t.Helper()
resp, err := svc.Get(t.Context(), req)
if err != nil {
t.Fatalf("Get() of quality %d error = %v", req.Transform.Quality, err)
}
defer func() { _ = resp.Content.Close() }()
body, err := io.ReadAll(resp.Content)
if err != nil {
t.Fatalf("reading the image of quality %d: %v", req.Transform.Quality, err)
}
return body, resp.CacheStatus
}
// fileSize returns the size of the file at path.
func fileSize(t *testing.T, path string) int64 {
t.Helper()
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat %s: %v", path, err)
}
return info.Size()
}
// cachedSourcePath returns the path of the cached source file req uses.
func cachedSourcePath(
t *testing.T, svc *testProxy, req *imagerequest.Request,
) string {
t.Helper()
reader, _, _, err := svc.cache.Source(t.Context(), req.Source)
if err != nil {
t.Fatalf("Source() error = %v; want the cached source", err)
}
_ = reader.Close()
return sourceFilePath(svc, req.Source.Key())
}
// TestProxy_Get_FetchesAgainForRemovedCachedSource removes the cached
// source's row and file, as the reconciliation pass does when the file is
// empty or cut short. internal/imgcache tests that the pass does so. The
// test then asks for two new sizes of the source. The first must fetch it
// again and cache it whole, so the second is made from the cache without a
// third fetch.
func TestProxy_Get_FetchesAgainForRemovedCachedSource(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
source := photoVariant(fixtures, 85, imagerequest.FitCover).Source
path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
wholeSize := fileSize(t, path)
// Removes the source as the reconciliation pass does: its row, then its file
_, err := svc.db.ExecContext(t.Context(),
`DELETE FROM source_content WHERE source_key = ?`, string(source.Key()))
if err != nil {
t.Fatalf("deleting the source's row: %v", err)
}
err = os.Remove(path)
if err != nil {
t.Fatalf("removing the source's file: %v", err)
}
getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover))
getImage(t, svc, photoVariant(fixtures, 30, imagerequest.FitCover))
fetches := readCounters(t, svc).upstreamFetchCount
t.Logf("upstream fetches = %d", fetches)
if fetches != 2 {
t.Errorf("upstream fetches = %d, want 2", fetches)
}
size := fileSize(t, path)
if size != wholeSize {
t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
}
}
// TestProxy_Get_FetchesAgainSourceOfNoSourceFormat replaces the cached
// source's file with text. The next request for a new size of the source
// must fetch it again and cache it whole.
func TestProxy_Get_FetchesAgainSourceOfNoSourceFormat(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
getImage(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
wholeSize := fileSize(t, path)
err := os.WriteFile(path, []byte("this is text, not an image"),
imgcache.StorageFilePerm)
if err != nil {
t.Fatalf("replacing the source's file: %v", err)
}
getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover))
fetches := readCounters(t, svc).upstreamFetchCount
t.Logf("upstream fetches = %d", fetches)
if fetches != 2 {
t.Errorf("upstream fetches = %d, want 2", fetches)
}
size := fileSize(t, path)
if size != wholeSize {
t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
}
}
// TestProxy_Get_ConvertsAgainForRemovedCachedVariant caches a variant of
// the photo and removes its row and file, as the reconciliation pass does
// when the file is empty or cut short. internal/imgcache tests that the
// pass does so. The test then checks that Get converts the image again, and
// that the next Get serves the whole variant from the cache.
func TestProxy_Get_ConvertsAgainForRemovedCachedVariant(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
req := photoVariant(fixtures, 85, imagerequest.FitCover)
want, _ := getImage(t, svc, req)
key := imagerequest.CacheKey(req)
// Removes the variant as the reconciliation pass does: its row, then its file
_, err := svc.db.ExecContext(t.Context(),
`DELETE FROM variant_content WHERE cache_key = ?`, string(key))
if err != nil {
t.Fatalf("deleting the variant's row: %v", err)
}
err = os.Remove(variantFilePath(svc, key))
if err != nil {
t.Fatalf("removing the variant's file: %v", err)
}
got, status := getImage(t, svc, req)
t.Logf("after the removal: %d bytes, %s", len(got), status)
if status != CacheMiss || !bytes.Equal(got, want) {
t.Errorf("after the removal: Get() = %d bytes, %s; want the %d bytes "+
"converted again, %s", len(got), status, len(want), CacheMiss)
}
got, status = getImage(t, svc, req)
t.Logf("next request: %d bytes, %s", len(got), status)
if status != CacheHit || !bytes.Equal(got, want) {
t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s",
len(got), status, len(want), CacheHit)
}
}
@@ -1,78 +0,0 @@
package imageproxy
import (
"context"
"database/sql"
"errors"
"testing"
"time"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// pendingAccessTimesWait is how long a test waits for the database's
// connection, for a cache hit to be answered, for an access time to reach
// the database, or for a request to return past its deadline.
const pendingAccessTimesWait = 5 * time.Second
// holdDatabase takes the one connection of db, so that every other query
// waits for it, and returns the func that frees it.
func holdDatabase(t *testing.T, db *sql.DB) func() {
t.Helper()
ctx, cancel := context.WithTimeout(t.Context(), pendingAccessTimesWait)
defer cancel()
conn, err := db.Conn(ctx)
if err != nil {
t.Fatalf("failed to take the database connection: %v", err)
}
return func() { _ = conn.Close() }
}
// TestProxy_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the
// database's one connection while a request whose fetch is held reaches its
// deadline. The request must return with the deadline's error while the
// connection is still held, as it waits for no database write past its
// deadline.
func TestProxy_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) {
t.Parallel()
svc, fixtures, fetcher := setupHeldFetchProxy(t)
svc.cache.StartPendingAccessTimeWrites()
defer func() { _ = svc.cache.StopPendingAccessTimeWrites(t.Context()) }()
// Room for the request to reach its fetch on a busy host
const timeout = 2 * time.Second
ctx, cancel := context.WithTimeout(t.Context(), timeout)
defer cancel()
deadline, _ := ctx.Deadline()
results := startGet(ctx, svc, photoVariant(fixtures, 85, imagerequest.FitCover))
// The request has made its database reads by the time it fetches.
select {
case <-fetcher.started:
case <-time.After(timeout):
t.Fatal("request did not reach its fetch by its deadline")
}
releaseDatabase := holdDatabase(t, svc.db)
defer releaseDatabase()
select {
case got := <-results:
t.Logf("Get() returned %v after its deadline, error = %v",
time.Since(deadline), got.err)
if !errors.Is(got.err, context.DeadlineExceeded) {
t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded)
}
case <-time.After(time.Until(deadline) + pendingAccessTimesWait):
t.Fatal("request did not return while the database was busy")
}
}
-245
View File
@@ -1,245 +0,0 @@
package imageproxy
import (
"bytes"
"context"
"image/color"
"io"
"io/fs"
"testing"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// TestProxy_Get_CountsStats walks Get through a miss that fetches the
// source, a hit, a miss that reuses the cached source, and two misses whose
// source cannot be used, checking every counter of the proxy and its cache
// after each.
func TestProxy_Get_CountsStats(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
// NewTestFS builds the same files the test proxy's fetcher serves.
testFS, _ := NewTestFS(t)
photo, err := fs.ReadFile(testFS, fixtures.GoodHostJPEG)
if err != nil {
t.Fatal(err)
}
fake, err := fs.ReadFile(testFS, fixtures.InvalidFile)
if err != nil {
t.Fatal(err)
}
photoBytes, fakeBytes := int64(len(photo)), int64(len(fake))
// want is hits, misses, upstream fetches, upstream bytes, transforms.
steps := []struct {
name string
path string
size int
wantErr bool
want counterValues
}{
{"miss that fetches the source", testPathPhoto, 50, false,
counterValues{0, 1, 1, photoBytes, 1}},
{"hit", testPathPhoto, 50, false,
counterValues{1, 1, 1, photoBytes, 1}},
{"miss that reuses the cached source", testPathPhoto, 25, false,
counterValues{1, 2, 1, photoBytes, 2}},
{"miss whose source fails the magic byte check", "/images/fake.jpg", 50, true,
counterValues{1, 3, 2, photoBytes + fakeBytes, 2}},
{"miss whose source is not found", "/images/nonexistent.jpg", 50, true,
counterValues{1, 4, 2, photoBytes + fakeBytes, 2}},
}
for _, step := range steps {
resp, err := svc.Get(t.Context(), &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: step.path,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: step.size, Height: step.size},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
})
if (err != nil) != step.wantErr {
t.Fatalf("%s: Get() error = %v, want error %t", step.name, err, step.wantErr)
}
if err == nil {
_ = resp.Content.Close()
}
got := readCounters(t, svc)
if got != step.want {
t.Fatalf("after the %s: counters = %+v, want %+v", step.name, got, step.want)
}
}
}
// fakeUpstream answers every fetch with itself as a JPEG body. The body
// serves data, then calls cancel, when set, and returns err; io.EOF ends
// the body normally.
type fakeUpstream struct {
data *bytes.Reader
cancel context.CancelFunc
err error
}
func (u *fakeUpstream) Fetch(
context.Context, imagerequest.Source,
) (*httpfetcher.FetchResult, error) {
return &httpfetcher.FetchResult{
Content: io.NopCloser(u),
ContentLength: -1,
ContentType: testContentTypeJPEG,
}, nil
}
func (u *fakeUpstream) Read(p []byte) (int, error) {
if u.data.Len() > 0 {
return u.data.Read(p)
}
if u.cancel != nil {
u.cancel()
}
return 0, u.err
}
// TestProxy_Get_CountsInterruptedMisses checks every counter of the proxy
// and its cache after a miss whose request context ends during or after the
// upstream fetch, and after a miss whose upstream body is over the size
// limit.
func TestProxy_Get_CountsInterruptedMisses(t *testing.T) {
t.Parallel()
photo := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255})
half := len(photo) / 2
// want is hits, misses, upstream fetches, upstream bytes, transforms.
tests := []struct {
name string
served int // bytes of the photo the upstream body serves
cancel bool // whether the body then ends the request context
readErr error // what the body then returns
wantErr bool
want counterValues
}{
{"request context ends during the fetch", half, true, context.Canceled, true,
counterValues{0, 1, 1, int64(half), 0}},
{"request context ends after the fetch", len(photo), true, io.EOF, false,
counterValues{0, 1, 1, int64(len(photo)), 1}},
{"upstream body over the size limit", half, false,
httpfetcher.ErrResponseTooLarge, true,
counterValues{0, 1, 1, int64(half), 0}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
upstream := &fakeUpstream{
data: bytes.NewReader(photo[:tc.served]),
err: tc.readErr,
}
if tc.cancel {
upstream.cancel = cancel
}
svc.fetcher = upstream
resp, err := svc.Get(ctx, &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
})
t.Logf("Get() error = %v", err)
if (err != nil) != tc.wantErr {
t.Fatalf("Get() error = %v, want error %t", err, tc.wantErr)
}
if err == nil {
_ = resp.Content.Close()
}
got := readCounters(t, svc)
if got != tc.want {
t.Errorf("counters = %+v, want %+v", got, tc.want)
}
})
}
}
// TestProxy_Get_CountsHitAfterRequestEnds checks every counter of the
// proxy and its cache after a hit served with a request context that has
// already ended: only the hit count moves.
func TestProxy_Get_CountsHitAfterRequestEnds(t *testing.T) {
t.Parallel()
svc, fixtures := setupTestProxy(t)
req := &imagerequest.Request{
Source: imagerequest.Source{
Host: fixtures.GoodHost,
Path: testPathPhoto,
},
Transform: imagerequest.Transform{
Size: imagerequest.Size{Width: 50, Height: 50},
Format: imagerequest.FormatJPEG,
Quality: 85,
FitMode: imagerequest.FitCover,
},
}
// A first request caches the variant.
resp, err := svc.Get(t.Context(), req)
if err != nil {
t.Fatalf("first Get() error = %v", err)
}
_ = resp.Content.Close()
want := readCounters(t, svc)
want.hitCount++
ctx, cancel := context.WithCancel(t.Context())
cancel()
resp, err = svc.Get(ctx, req)
if err != nil {
t.Fatalf("Get() with an ended request context: error = %v", err)
}
_ = resp.Content.Close()
if resp.CacheStatus != CacheHit {
t.Fatalf("CacheStatus = %v, want %v", resp.CacheStatus, CacheHit)
}
got := readCounters(t, svc)
if got != want {
t.Errorf("counters = %+v, want %+v", got, want)
}
}
@@ -1,355 +0,0 @@
package imageproxy
import (
"bytes"
"context"
"database/sql"
"image"
"image/color"
"image/gif"
"image/jpeg"
"image/png"
"io/fs"
"path/filepath"
"testing"
"testing/fstest"
"time"
"github.com/prometheus/client_golang/prometheus"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher"
"sneak.berlin/go/pixa/internal/imageprocessor"
"sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
)
// Shared test data literals, extracted as constants for goconst.
const (
testContentTypeJPEG = "image/jpeg"
)
// sqliteTimestampFormat matches the format SQLite's CURRENT_TIMESTAMP
// produces, so injected timestamps compare correctly against ones the
// cache writes.
const sqliteTimestampFormat = "2006-01-02 15:04:05"
// TestFixtures contains paths to test files in the mock filesystem.
type TestFixtures struct {
// Valid image files
GoodHostJPEG string // valid JPEG on GoodHost
GoodHostPNG string // valid PNG on GoodHost
GoodHostGIF string // valid GIF on GoodHost
OtherHostJPEG string // valid JPEG on OtherHost
OtherHostPNG string // valid PNG on OtherHost
// Invalid/edge case files
InvalidFile string // file with wrong magic bytes
EmptyFile string // zero-byte file
TextFile string // text file masquerading as image
// Hostnames
GoodHost string // a host the mock fetcher serves
OtherHost string // a second host the mock fetcher serves
}
// DefaultFixtures returns the standard test fixture paths.
func DefaultFixtures() *TestFixtures {
return &TestFixtures{
GoodHostJPEG: "goodhost.example.com/images/photo.jpg",
GoodHostPNG: "goodhost.example.com/images/logo.png",
GoodHostGIF: "goodhost.example.com/images/animation.gif",
OtherHostJPEG: "otherhost.example.com/uploads/image.jpg",
OtherHostPNG: "otherhost.example.com/uploads/icon.png",
InvalidFile: "goodhost.example.com/images/fake.jpg",
EmptyFile: "goodhost.example.com/images/empty.jpg",
TextFile: "goodhost.example.com/images/text.png",
GoodHost: "goodhost.example.com",
OtherHost: "otherhost.example.com",
}
}
// NewTestFS creates a mock filesystem with test images.
func NewTestFS(t *testing.T) (fs.FS, *TestFixtures) {
t.Helper()
fixtures := DefaultFixtures()
// Generate test images
jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255})
pngData := generateTestPNG(t, 100, 100, color.RGBA{0, 255, 0, 255})
gifData := generateTestGIF(t, 100, 100, color.RGBA{0, 0, 255, 255})
// Create the mock filesystem
mockFS := fstest.MapFS{
// Good host files
fixtures.GoodHostJPEG: &fstest.MapFile{Data: jpegData},
fixtures.GoodHostPNG: &fstest.MapFile{Data: pngData},
fixtures.GoodHostGIF: &fstest.MapFile{Data: gifData},
// Other host files
fixtures.OtherHostJPEG: &fstest.MapFile{Data: jpegData},
fixtures.OtherHostPNG: &fstest.MapFile{Data: pngData},
// Invalid files
fixtures.InvalidFile: &fstest.MapFile{Data: []byte("not a real image file content")},
fixtures.EmptyFile: &fstest.MapFile{Data: []byte{}},
fixtures.TextFile: &fstest.MapFile{Data: []byte("This is just text, not a PNG")},
}
return mockFS, fixtures
}
// generateTestJPEG creates a minimal valid JPEG image.
func generateTestJPEG(t *testing.T, width, height int, c color.Color) []byte {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
for y := range height {
for x := range width {
img.Set(x, y, c)
}
}
var buf bytes.Buffer
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85})
if err != nil {
t.Fatalf("failed to encode test JPEG: %v", err)
}
return buf.Bytes()
}
// generateTestPNG creates a minimal valid PNG image.
func generateTestPNG(t *testing.T, width, height int, c color.Color) []byte {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
for y := range height {
for x := range width {
img.Set(x, y, c)
}
}
var buf bytes.Buffer
err := png.Encode(&buf, img)
if err != nil {
t.Fatalf("failed to encode test PNG: %v", err)
}
return buf.Bytes()
}
// generateTestGIF creates a minimal valid GIF image.
func generateTestGIF(t *testing.T, width, height int, c color.Color) []byte {
t.Helper()
img := image.NewPaletted(
image.Rect(0, 0, width, height),
[]color.Color{c, color.White},
)
for y := range height {
for x := range width {
img.SetColorIndex(x, y, 0)
}
}
var buf bytes.Buffer
err := gif.Encode(&buf, img, nil)
if err != nil {
t.Fatalf("failed to encode test GIF: %v", err)
}
return buf.Bytes()
}
// testProxy is a Proxy with the database and the state directory of its
// cache, through which a test reads and changes the cache behind the proxy.
// registry is the metrics registry the proxy and its cache share.
type testProxy struct {
*Proxy
db *sql.DB
stateDir string
registry *prometheus.Registry
}
// setupTestProxy creates a Proxy with mock fetcher for testing.
func setupTestProxy(t *testing.T) (*testProxy, *TestFixtures) {
t.Helper()
mockFS, fixtures := NewTestFS(t)
// Create temp directory for cache
tmpDir := t.TempDir()
// Create in-memory database
db := setupTestDB(t)
registry := prometheus.NewRegistry()
cache, err := imgcache.NewCache(db, imgcache.Config{
StateDir: tmpDir,
NegativeTTL: 5 * time.Minute,
MaxBytes: 1 << 30,
}, registry)
if err != nil {
t.Fatalf("failed to create cache: %v", err)
}
proxy, err := New(&Config{
Cache: cache,
Fetcher: httpfetcher.NewMock(mockFS),
Processor: imageprocessor.New(imageprocessor.Params{
MaxInputBytes: 50 << 20,
MaxConcurrentProcessing: 2,
}),
MaxResponseSize: 50 << 20,
}, registry)
if err != nil {
t.Fatalf("failed to create proxy: %v", err)
}
return &testProxy{
Proxy: proxy,
db: db,
stateDir: tmpDir,
registry: registry,
}, fixtures
}
// counterValues holds the values of the counters of a proxy and its cache.
type counterValues struct {
hitCount int64
missCount int64
upstreamFetchCount int64
upstreamFetchBytes int64
transformCount int64
}
// readCounters reads the counters of svc and its cache from their metrics
// registry.
func readCounters(t *testing.T, svc *testProxy) counterValues {
t.Helper()
families, err := svc.registry.Gather()
if err != nil {
t.Fatalf("failed to gather the counters: %v", err)
}
values := make(map[string]int64, len(families))
for _, family := range families {
values[family.GetName()] = int64(family.GetMetric()[0].GetCounter().GetValue())
}
return counterValues{
hitCount: values["pixa_cache_hits_total"],
missCount: values["pixa_cache_misses_total"],
upstreamFetchCount: values["pixa_upstream_fetches_total"],
upstreamFetchBytes: values["pixa_upstream_fetch_bytes_total"],
transformCount: values["pixa_transforms_total"],
}
}
// sourceFilePath returns the path of the file of the source key in svc's
// cache, as README.md gives it: <state_dir>/cache/sources/<ab>/<cd>/<key>.
func sourceFilePath(svc *testProxy, key imagerequest.SourceKey) string {
name := string(key)
return filepath.Join(svc.stateDir, "cache", "sources", name[0:2], name[2:4], name)
}
// variantFilePath returns the path of the file of the variant key in svc's
// cache, as README.md gives it: <state_dir>/cache/variants/<ab>/<cd>/<key>.
func variantFilePath(svc *testProxy, key imagerequest.VariantKey) string {
name := string(key)
return filepath.Join(svc.stateDir, "cache", "variants", name[0:2], name[2:4], name)
}
// setVariantLastAccessed backdates the last access time of a tracked
// variant.
func setVariantLastAccessed(
t *testing.T, db *sql.DB, key imagerequest.VariantKey, when time.Time,
) {
t.Helper()
res, err := db.ExecContext(t.Context(),
`UPDATE variant_content SET last_accessed_at = ? WHERE cache_key = ?`,
when.UTC().Format(sqliteTimestampFormat), string(key),
)
if err != nil {
t.Fatalf("failed to set variant last_accessed_at: %v", err)
}
affected, err := res.RowsAffected()
if err != nil {
t.Fatalf("failed to read affected rows: %v", err)
}
if affected != 1 {
t.Fatalf("variant %s has no accounting row (affected=%d); "+
"stores must track variants in the database", key, affected)
}
}
// setSourceLastAccessed backdates the last access time of a tracked
// source content blob.
func setSourceLastAccessed(
t *testing.T, db *sql.DB, key imagerequest.SourceKey, when time.Time,
) {
t.Helper()
res, err := db.ExecContext(t.Context(),
`UPDATE source_content SET last_accessed_at = ? WHERE source_key = ?`,
when.UTC().Format(sqliteTimestampFormat), string(key),
)
if err != nil {
t.Fatalf("failed to set source last_accessed_at: %v", err)
}
affected, err := res.RowsAffected()
if err != nil {
t.Fatalf("failed to read affected rows: %v", err)
}
if affected != 1 {
t.Fatalf("source %s has no accounting row (affected=%d)", key, affected)
}
}
// countRows returns the number of rows the given query yields.
func countRows(t *testing.T, db *sql.DB, query string, args ...any) int {
t.Helper()
var n int
err := db.QueryRowContext(t.Context(), query, args...).Scan(&n)
if err != nil {
t.Fatalf("count query %q failed: %v", query, err)
}
return n
}
// setupTestDB creates an in-memory SQLite database for testing
// using the production schema.
func setupTestDB(t *testing.T) *sql.DB {
t.Helper()
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatalf("failed to open test db: %v", err)
}
// Use the real production schema via migrations
err = database.ApplyMigrations(context.Background(), db, nil)
if err != nil {
t.Fatalf("failed to apply migrations: %v", err)
}
return db
}
-163
View File
@@ -1,163 +0,0 @@
// Package imagerequest holds the image request type, its parts, its checks,
// the parsing of the /v1/image/ path, the variant key, the source key and the
// source's upstream URL. It also holds the image format type and the MIME type
// of each format. It imports no other package of this repo, so every package
// that handles an image request can import it.
package imagerequest
import (
"errors"
"net/url"
)
// ErrInvalidFitMode is returned when an invalid fit mode is provided.
var ErrInvalidFitMode = errors.New("invalid fit mode")
// Format is an image format. pixa reads and writes the formats from JPEG to
// GIF below. In a request, orig and auto stand for one of them.
type Format string
// Supported image formats.
const (
FormatOriginal Format = "orig"
FormatJPEG Format = "jpeg"
FormatPNG Format = "png"
FormatWebP Format = "webp"
FormatAVIF Format = "avif"
FormatJXL Format = "jxl"
FormatGIF Format = "gif"
// FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each
// request from its Accept header once the URL's signature or token has
// been checked; it is never processed or cached as itself.
FormatAuto Format = "auto"
)
// MIMEType returns the MIME type of the format, such as image/jpeg for
// FormatJPEG. orig and auto stand for another format, so they have none:
// MIMEType returns "" for them.
func (f Format) MIMEType() string {
switch f {
case FormatJPEG:
return "image/jpeg"
case FormatPNG:
return "image/png"
case FormatWebP:
return "image/webp"
case FormatAVIF:
return "image/avif"
case FormatJXL:
return "image/jxl"
case FormatGIF:
return "image/gif"
case FormatOriginal, FormatAuto:
return ""
default:
return ""
}
}
// Size represents requested image dimensions
type Size struct {
Width int
Height int
}
// OriginalSize returns true if this represents "keep original size"
func (s Size) OriginalSize() bool {
return s.Width == 0 && s.Height == 0
}
// Fit represents how to fit image into requested dimensions.
type Fit string
// Supported image fit modes.
const (
FitCover Fit = "cover"
FitContain Fit = "contain"
FitFill Fit = "fill"
FitInside Fit = "inside"
FitOutside Fit = "outside"
)
// ValidateFitMode checks if the given fit mode is valid.
// Returns ErrInvalidFitMode for unrecognized fit modes.
func ValidateFitMode(fit Fit) error {
switch fit {
case FitCover, FitContain, FitFill, FitInside, FitOutside, "":
return nil
default:
return ErrInvalidFitMode
}
}
// ValidateImageRequest checks a request's width and height with
// ValidateDimension and its fit mode with ValidateFitMode. Both the plain
// /v1/image/ route and the encrypted /v1/e/ route validate through this
// function so a request from either source enforces identical bounds.
func ValidateImageRequest(req *Request) error {
err := ValidateDimension("width", req.Transform.Size.Width)
if err != nil {
return err
}
err = ValidateDimension("height", req.Transform.Size.Height)
if err != nil {
return err
}
return ValidateFitMode(req.Transform.FitMode)
}
// Request represents a request for a processed image: the source to fetch
// and the transform to apply to it. It holds exactly what the variant key
// covers.
type Request struct {
// Source is what to fetch
Source Source
// Transform is what to do to it
Transform Transform
}
// Source is the image a request fetches from the origin.
type Source struct {
// Host is the origin host (e.g., "cdn.example.com")
Host string
// Path is the path on the origin with its percent-escapes decoded, as
// url.URL's Path holds it (e.g., "/photos/my cat.jpg")
Path string
// Query is the optional query string for the origin URL as the URL
// writes it, percent-escapes included, as url.URL's RawQuery holds it
Query string
}
// Transform is what a request does to its source; the image processor
// takes it alone.
type Transform struct {
// Size is the requested output dimensions
Size Size
// Format is the requested output format
Format Format
// Quality is the output quality (1-100) for lossy formats
Quality int
// FitMode is how to fit the image into requested dimensions
FitMode Fit
}
// URL returns the source's upstream URL, with the scheme http when allowHTTP
// is true and https otherwise. net/url writes its path percent-encoded again,
// so each source path is one upstream path. Its query is the source query as
// it is, with the characters escapeQuery encodes percent-encoded.
func (s Source) URL(allowHTTP bool) *url.URL {
scheme := "https"
if allowHTTP {
scheme = "http"
}
return &url.URL{
Scheme: scheme,
Host: s.Host,
Path: s.Path,
RawQuery: escapeQuery(s.Query),
}
}
@@ -1,30 +0,0 @@
package imagerequest
import "testing"
// TestFormat_MIMEType checks the MIME type of each format pixa reads and
// writes, and that orig and auto have none.
func TestFormat_MIMEType(t *testing.T) {
t.Parallel()
tests := []struct {
format Format
want string
}{
{FormatJPEG, "image/jpeg"},
{FormatPNG, "image/png"},
{FormatWebP, "image/webp"},
{FormatAVIF, "image/avif"},
{FormatJXL, "image/jxl"},
{FormatGIF, "image/gif"},
{FormatOriginal, ""},
{FormatAuto, ""},
}
for _, tt := range tests {
got := tt.format.MIMEType()
if got != tt.want {
t.Errorf("%s MIMEType() = %q, want %q", tt.format, got, tt.want)
}
}
}
-26
View File
@@ -1,26 +0,0 @@
package imagerequest
import (
"crypto/sha256"
"encoding/hex"
"fmt"
)
// SourceKey is a SHA256 hash identifying a cached source image (hex-encoded).
type SourceKey string
// Key returns the key the source is cached under, made as CacheKey makes a
// variant key.
// Format: sha256(host:path:query)
// The path and query go through escapePercentAndColon, so that a ":" in
// either cannot be taken for the one between them.
func (s Source) Key() SourceKey {
data := fmt.Sprintf("%s:%s:%s",
s.Host,
escapePercentAndColon(s.Path),
escapePercentAndColon(s.Query),
)
h := sha256.Sum256([]byte(data))
return SourceKey(hex.EncodeToString(h[:]))
}
@@ -1,40 +0,0 @@
package imagerequest
import "testing"
// TestSourceKey_DiffersForEachSource checks that each source gets a key of
// 64 hex characters, the same each time, and that no two of these sources
// share one, also when they differ only by a ":" or "%" in the path or query.
func TestSourceKey_DiffersForEachSource(t *testing.T) {
t.Parallel()
sources := []Source{
{Host: testHostCDN, Path: testPathCat},
{Host: testHostCDN, Path: testPathCat, Query: "v=2"},
{Host: "other.example.com", Path: testPathCat},
{Host: testHostCDN, Path: "/c:d"},
{Host: testHostCDN, Path: "/c", Query: "d:"},
{Host: testHostCDN, Path: "/c%3Ad"},
}
seen := make(map[SourceKey]Source)
for _, source := range sources {
key := source.Key()
t.Logf("Key() of %+v = %s", source, key)
if len(key) != 64 {
t.Errorf("Key() of %+v length = %d, want 64", source, len(key))
}
if again := source.Key(); again != key {
t.Errorf("Key() of %+v not deterministic: %s vs %s", source, key, again)
}
if earlier, found := seen[key]; found {
t.Errorf("Key() gave %+v and %+v the same key", earlier, source)
}
seen[key] = source
}
}

Some files were not shown because too many files have changed in this diff Show More