Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
410c167c70 | ||
|
|
bdde021b45 | ||
|
|
db91ab29e6 | ||
|
|
8597253ffd | ||
|
|
99e4aa3bb2 | ||
|
|
d2944aa891 | ||
|
|
54328377d0 | ||
|
|
5058fd532b | ||
|
|
15d95436a5 | ||
|
|
01823d27db | ||
|
|
a941a80bf9 | ||
|
|
55cf7f4fac | ||
|
|
c434581a54 | ||
|
|
f77faf13de | ||
|
|
ae7c3f226d | ||
|
|
2beba15ae7 | ||
|
|
23ec4026f6 | ||
|
|
ef828f71a5 | ||
|
|
f3231a3c5a | ||
|
|
cca2e3f926 | ||
|
|
708a9bec20 |
+70
-9
@@ -1,12 +1,73 @@
|
||||
# .git is sent without its config. Without a VERSION build argument the
|
||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||
# does not need .git/config; that file can hold a credential, such as a
|
||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
||||
# `certs/server.key` still ship while this file reads as solved. Only
|
||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
||||
# into .gitignore, where `**/` is wrong.
|
||||
#
|
||||
# Matching is case-sensitive, so secrets use character ranges rather
|
||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
||||
#
|
||||
# Extend with this repo's own host-built artifacts, written anchored:
|
||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
||||
# deletes the package directory from the context.
|
||||
|
||||
# Unlike the standard file, which leaves out all of .git, pixa sends
|
||||
# .git without its config. Without a VERSION build argument the stage
|
||||
# that compiles runs `git describe --tags --always` on .git, which does
|
||||
# not need .git/config; that file can hold a credential, such as a
|
||||
# password in a remote URL or the token the CI checkout step stores there.
|
||||
.git/config
|
||||
.gitignore
|
||||
.DS_Store
|
||||
.env*
|
||||
|
||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
||||
# Anchored because it occurs once where agents run at the repo root.
|
||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
||||
# `services/api/.claude/` and must add its own anchored entry.
|
||||
.claude
|
||||
node_modules
|
||||
bin/
|
||||
data/
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Re-include a committed template with a negation if the
|
||||
# build needs one: `!docs/example.env`.
|
||||
**/*.[eE][nN][vV]
|
||||
**/.[eE][nN][vV].*
|
||||
**/.[eE][nN][vV][rR][cC]
|
||||
|
||||
# Private keys and the bundles carrying them. Public certificates
|
||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
||||
**/*.[pP][eE][mM]
|
||||
**/*.[kK][eE][yY]
|
||||
**/*.[pP]12
|
||||
**/*.[pP][fF][xX]
|
||||
**/[iI][dD]_[rR][sS][aA]
|
||||
**/[iI][dD]_[dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
**/[iI][dD]_[eE][dD]25519
|
||||
|
||||
# Dependencies: restored inside the image, never copied in.
|
||||
**/node_modules
|
||||
|
||||
# OS metadata.
|
||||
**/.DS_Store
|
||||
**/Thumbs.db
|
||||
|
||||
# Editor state: never a build input, and it churns COPY.
|
||||
**/*.swp
|
||||
**/*.swo
|
||||
**/*~
|
||||
**/*.bak
|
||||
**/.idea
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# pixa's own entries. Nothing in the build reads .gitignore. On the
|
||||
# host, `make build` writes bin/pixad, and the example config keeps its
|
||||
# state directory in data/.
|
||||
.gitignore
|
||||
/bin
|
||||
/data
|
||||
|
||||
# Local config files, kept out of git because they can hold the signing key.
|
||||
**/[cC][oO][nN][fF][iI][gG].[yY][mM][lL]
|
||||
**/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL]
|
||||
**/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL]
|
||||
|
||||
@@ -6,5 +6,10 @@ jobs:
|
||||
steps:
|
||||
# actions/checkout v4.2.2, 2026-02-22
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
# The default clone is shallow and has no tags, so the
|
||||
# version the build takes from `git describe` would be a
|
||||
# bare commit; this fetches the whole history with its tags.
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- run: script/cibuild
|
||||
- run: script/docker-smoke
|
||||
|
||||
@@ -11,6 +11,12 @@ Thumbs.db
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
||||
# entry and must not be given a `**/` prefix on the way into one.
|
||||
.claude/
|
||||
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
@@ -31,5 +37,6 @@ node_modules/
|
||||
*.sqlite3
|
||||
|
||||
# Local dev configs
|
||||
config.yml
|
||||
config.yaml
|
||||
config.dev.yml
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
node_modules/
|
||||
yarn.lock
|
||||
|
||||
# A byte-for-byte copy of the one in sneak/prompts.
|
||||
REPO_POLICIES.md
|
||||
|
||||
vendor/
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"tabWidth": 4,
|
||||
"proseWrap": "always"
|
||||
}
|
||||
@@ -4,73 +4,68 @@ Last Updated 2026-01-08
|
||||
|
||||
These rules MUST be followed at all times, it is very important.
|
||||
|
||||
* Never use `git add -A` - add specific changes to a deliberate commit. A
|
||||
commit should contain one change. After each change, make a commit with a
|
||||
good one-line summary.
|
||||
- Never use `git add -A` - add specific changes to a deliberate commit. A commit
|
||||
should contain one change. After each change, make a commit with a good
|
||||
one-line summary.
|
||||
|
||||
* NEVER modify the linter config without asking first.
|
||||
- NEVER modify the linter config without asking first.
|
||||
|
||||
* NEVER modify tests to exclude special cases or otherwise get them to pass
|
||||
without asking first. In almost all cases, the code should be changed,
|
||||
NOT the tests. If you think the test needs to be changed, make your case
|
||||
for that and ask for permission to proceed, then stop. You need explicit
|
||||
user approval to modify existing tests. (You do not need user approval
|
||||
for writing NEW tests.)
|
||||
- NEVER modify tests to exclude special cases or otherwise get them to pass
|
||||
without asking first. In almost all cases, the code should be changed, NOT the
|
||||
tests. If you think the test needs to be changed, make your case for that and
|
||||
ask for permission to proceed, then stop. You need explicit user approval to
|
||||
modify existing tests. (You do not need user approval for writing NEW tests.)
|
||||
|
||||
* When linting, assume the linter config is CORRECT, and that each item
|
||||
output by the linter is something that legitimately needs fixing in the
|
||||
code.
|
||||
- When linting, assume the linter config is CORRECT, and that each item output
|
||||
by the linter is something that legitimately needs fixing in the code.
|
||||
|
||||
* When running tests, use `make test`.
|
||||
- When running tests, use `make test`.
|
||||
|
||||
* Before commits, run `make check`. This runs `make lint` and `make test`
|
||||
and `make check-fmt`. Any issues discovered MUST be resolved before
|
||||
committing unless explicitly told otherwise.
|
||||
- Before commits, run `make check`. This runs `make lint` and `make test` and
|
||||
`make check-fmt`. Any issues discovered MUST be resolved before committing
|
||||
unless explicitly told otherwise.
|
||||
|
||||
* When fixing a bug, write a failing test for the bug FIRST. Add
|
||||
appropriate logging to the test to ensure it is written correctly. Commit
|
||||
that. Then go about fixing the bug until the test passes (without
|
||||
modifying the test further). Then commit that.
|
||||
- When fixing a bug, write a failing test for the bug FIRST. Add appropriate
|
||||
logging to the test to ensure it is written correctly. Commit that. Then go
|
||||
about fixing the bug until the test passes (without modifying the test
|
||||
further). Then commit that.
|
||||
|
||||
* When adding a new feature, do the same - implement a test first (TDD). It
|
||||
doesn't have to be super complex. Commit the test, then commit the
|
||||
feature.
|
||||
- When adding a new feature, do the same - implement a test first (TDD). It
|
||||
doesn't have to be super complex. Commit the test, then commit the feature.
|
||||
|
||||
* When adding a new feature, use a feature branch. When the feature is
|
||||
completely finished and the code is up to standards (passes `make check`)
|
||||
then and only then can the feature branch be merged into `main` and the
|
||||
branch deleted.
|
||||
- When adding a new feature, use a feature branch. When the feature is
|
||||
completely finished and the code is up to standards (passes `make check`) then
|
||||
and only then can the feature branch be merged into `main` and the branch
|
||||
deleted.
|
||||
|
||||
* Write godoc documentation comments for all exported types and functions as
|
||||
you go along.
|
||||
- Write godoc documentation comments for all exported types and functions as you
|
||||
go along.
|
||||
|
||||
* ALWAYS be consistent in naming. If you name something one thing in one
|
||||
place, name it the EXACT SAME THING in another place.
|
||||
- ALWAYS be consistent in naming. If you name something one thing in one place,
|
||||
name it the EXACT SAME THING in another place.
|
||||
|
||||
* Be descriptive and specific in naming. `wl` is bad;
|
||||
`SourceHostWhitelist` is good. `ConnsPerHost` is bad;
|
||||
`MaxConnectionsPerHost` is good.
|
||||
- Be descriptive and specific in naming. `wl` is bad; `SourceHostWhitelist` is
|
||||
good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good.
|
||||
|
||||
* This is not prototype or teaching code - this is designed for production.
|
||||
Any security issues (such as denial of service) or other web
|
||||
vulnerabilities are P1 bugs and must be added to TODO.md at the top.
|
||||
- This is not prototype or teaching code - this is designed for production. Any
|
||||
security issues (such as denial of service) or other web vulnerabilities are
|
||||
P1 bugs and must be added to TODO.md at the top.
|
||||
|
||||
* As this is production code, no stubbing of implementations unless
|
||||
specifically instructed. We need working implementations.
|
||||
- As this is production code, no stubbing of implementations unless specifically
|
||||
instructed. We need working implementations.
|
||||
|
||||
* NEVER silently fall back to a different setting when a user's parameter
|
||||
explicitly specifies a value. If a user requests format=webp and WebP
|
||||
encoding is not supported, return an error - do NOT silently output PNG
|
||||
instead. If a user specifies fit=invalid and that fit mode doesn't exist,
|
||||
return an error - do NOT silently default to "cover". Silent fallbacks
|
||||
violate the principle of least surprise and mask bugs. The only acceptable
|
||||
defaults are for OMITTED parameters, never for INVALID explicit values.
|
||||
- NEVER silently fall back to a different setting when a user's parameter
|
||||
explicitly specifies a value. If a user requests format=webp and WebP encoding
|
||||
is not supported, return an error - do NOT silently output PNG instead. If a
|
||||
user specifies fit=invalid and that fit mode doesn't exist, return an error -
|
||||
do NOT silently default to "cover". Silent fallbacks violate the principle of
|
||||
least surprise and mask bugs. The only acceptable defaults are for OMITTED
|
||||
parameters, never for INVALID explicit values.
|
||||
|
||||
* Avoid vendoring deps unless specifically instructed to. NEVER commit
|
||||
the vendor directory, NEVER commit compiled binaries. If these
|
||||
directories or files exist, add them to .gitignore (and commit the
|
||||
.gitignore) if they are not already in there. Keep the entire git
|
||||
repository (with history) small - under 20MiB, unless you specifically
|
||||
must commit larger files (e.g. test fixture example media files). Only
|
||||
OUR source code and immediately supporting files (such as test examples)
|
||||
goes into the repo/history.
|
||||
- Avoid vendoring deps unless specifically instructed to. NEVER commit the
|
||||
vendor directory, NEVER commit compiled binaries. If these directories or
|
||||
files exist, add them to .gitignore (and commit the .gitignore) if they are
|
||||
not already in there. Keep the entire git repository (with history) small -
|
||||
under 20MiB, unless you specifically must commit larger files (e.g. test
|
||||
fixture example media files). Only OUR source code and immediately supporting
|
||||
files (such as test examples) goes into the repo/history.
|
||||
|
||||
+51
-35
@@ -1,55 +1,66 @@
|
||||
# Lint stage
|
||||
# Same image as Dockerfile.lint: change both pins together.
|
||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
|
||||
# Lint phase. script/lint builds it alone. The linter is run directly:
|
||||
# `make lint` and script/lint are themselves a docker build.
|
||||
# golangci/golangci-lint:v2.12.2, 2026-10-04
|
||||
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||
|
||||
# The linter compiles every package, and govips needs the libvips
|
||||
# headers for that. REPO_POLICIES.md has the lint phase install them
|
||||
# itself; this image is Debian, so with apt-get rather than apk.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. script/test builds it alone.
|
||||
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
||||
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# script/bootstrap installs the build dependencies and downloads the Go
|
||||
# modules. Only script/, go.mod and go.sum are copied first, so this
|
||||
# layer is reused until one of them changes.
|
||||
# script/bootstrap --cgo installs the build dependencies (a C compiler,
|
||||
# the libvips and libheif headers, and libvips' JPEG XL support, which
|
||||
# the tests need) and downloads the Go modules.
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN script/bootstrap
|
||||
RUN script/bootstrap --cgo
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Tells script/lint it is inside a container, so it runs the linter.
|
||||
ENV container=docker
|
||||
# Without -v first; on a failure, again with -v for the details, and
|
||||
# the step fails even if the second run passes. -parallel 4: by default
|
||||
# a package runs as many of its tests at once as the host has CPUs, and
|
||||
# on a busy host they then wait so long to be scheduled that a test
|
||||
# that times a request can see it return a second late.
|
||||
RUN go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; }
|
||||
|
||||
# Run formatting check and linter. script/cibuild and script/docker pass
|
||||
# a new CHECK_EPOCH on every run, and each check step names it in its
|
||||
# command, so a new value reruns the step instead of reusing a cached
|
||||
# success that checked nothing. A plain `docker build .` leaves it empty
|
||||
# and reuses the check steps only for an identical build context.
|
||||
ARG CHECK_EPOCH
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make lint
|
||||
# Build stage. Nothing is wanted from the two phases above: these copies
|
||||
# make BuildKit build them first, so this stage runs only when lint and
|
||||
# test passed.
|
||||
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
||||
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||
|
||||
# Build stage
|
||||
# golang:1.25.4-alpine, 2026-02-25
|
||||
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||
|
||||
# Depend on lint stage passing
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Build dependencies and Go modules, as in the lint stage
|
||||
# Build dependencies and Go modules, as in the test phase
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN script/bootstrap
|
||||
RUN script/bootstrap --cgo
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Run tests; a new CHECK_EPOCH reruns them, as in the lint stage.
|
||||
ARG CHECK_EPOCH
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
||||
|
||||
# VERSION is declared here, not earlier: a new value reruns only the
|
||||
# build, not script/bootstrap or the tests. Given none, the version is
|
||||
# build, not script/bootstrap. Given none, the version is
|
||||
# `git describe --tags --always` of the .git in the build context (git
|
||||
# comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH
|
||||
# after one, the short commit when no tag is reachable. A context that
|
||||
@@ -68,13 +79,18 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
||||
-ldflags "-s -w -X main.Version=${version}" \
|
||||
-o /pixad ./cmd/pixad
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-02-25
|
||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||
# Runtime stage, and the last one: a plain `docker build .` builds this
|
||||
# stage and what it depends on, and nothing else. It must use the Alpine
|
||||
# release the golang image above is based on, so that pixad runs against
|
||||
# the libvips and musl it was built with.
|
||||
# alpine:3.22, 2026-10-08
|
||||
FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8
|
||||
|
||||
# Install runtime dependencies only
|
||||
# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
|
||||
# support, without which pixad does not start.
|
||||
RUN apk add --no-cache \
|
||||
vips \
|
||||
vips-jxl \
|
||||
libheif \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
# Dockerfile.lint: the container script/lint builds to run golangci-lint,
|
||||
# which is never installed on the host. Pinned to the same image as the
|
||||
# Dockerfile lint stage: change both pins together, or the two run
|
||||
# different linter versions.
|
||||
#
|
||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# pixa is CGO/libvips: the type-aware linters compile every package, so
|
||||
# this image needs the same C libraries the build does. script/bootstrap
|
||||
# installs them and downloads the Go modules. Only script/, go.mod and
|
||||
# go.sum are copied first; they settle this layer's result, so it may
|
||||
# safely be reused between runs.
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN script/bootstrap
|
||||
|
||||
COPY . .
|
||||
|
||||
# Tells script/lint it is inside a container, so it runs the linter.
|
||||
ENV container=docker
|
||||
|
||||
# script/lint passes a different CACHEBUST on every run, and BuildKit
|
||||
# keys every RUN after this ARG on its value, so the lint step always
|
||||
# runs instead of returning a cached success that linted nothing.
|
||||
#
|
||||
# Go's and golangci-lint's caches (/root/.cache, hundreds of MB) go on a
|
||||
# tmpfs that is discarded after the step. Written into the layer, they
|
||||
# would pile up as build cache on every run, since no later run, with
|
||||
# its new CACHEBUST, can reuse that layer.
|
||||
ARG CACHEBUST
|
||||
RUN --mount=type=tmpfs,target=/root/.cache script/lint
|
||||
@@ -1,10 +1,10 @@
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest
|
||||
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
LDFLAGS := -X main.Version=$(VERSION)
|
||||
|
||||
# Use nix-shell to provide CGO dependencies unless they are already available
|
||||
# (e.g. inside a Docker build or an existing nix-shell).
|
||||
# (e.g. inside an existing nix-shell).
|
||||
HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null)
|
||||
ifdef HAS_PKGCONFIG
|
||||
NIX_RUN_PREFIX =
|
||||
@@ -32,11 +32,11 @@ fmt-check:
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
# Run linter
|
||||
# Run linter (the lint phase of the Dockerfile)
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
# Run tests (30-second timeout)
|
||||
# Run tests (the test phase of the Dockerfile)
|
||||
test:
|
||||
@script/test
|
||||
|
||||
@@ -59,20 +59,25 @@ docker:
|
||||
docker-smoke:
|
||||
@script/docker-smoke
|
||||
|
||||
# Build Docker image tagged pixad:$(VERSION) and pixad:latest
|
||||
docker-versioned:
|
||||
docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
|
||||
# Measure throughput, latency and peak memory with the default duration and
|
||||
# number of clients (needs Docker and Go; a benchmark, not part of check)
|
||||
loadtest:
|
||||
@script/loadtest
|
||||
|
||||
# Run tests in Docker (needed for CGO/libvips)
|
||||
# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION)
|
||||
docker-versioned:
|
||||
@script/docker
|
||||
docker tag pixa pixa:$(VERSION)
|
||||
|
||||
# Run tests in Docker, as `make test` does
|
||||
docker-test:
|
||||
docker build --target builder --build-arg VERSION=$(VERSION) -t pixad-builder .
|
||||
docker run --rm pixad-builder sh -c "CGO_ENABLED=1 GOTOOLCHAIN=auto go test -v ./..."
|
||||
@script/test
|
||||
|
||||
# Run local dev server in Docker
|
||||
devserver: docker-versioned devserver-stop
|
||||
docker run -d --name pixad-dev -p 8080:8080 \
|
||||
-v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
|
||||
pixad:latest
|
||||
pixa:latest
|
||||
@echo "pixad running at http://localhost:8080"
|
||||
|
||||
# Stop dev server
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
# pixa
|
||||
|
||||
pixa is a GPL-3.0-licensed Go web server by
|
||||
[@sneak](https://sneak.berlin) that proxies images from upstream
|
||||
sources, optionally resizing or transforming them, and serves the
|
||||
results. Both source and transformed images are cached to disk so that
|
||||
subsequent requests are served without origin fetches or additional
|
||||
pixa is a GPL-3.0-licensed Go web server by [@sneak](https://sneak.berlin) that
|
||||
proxies images from upstream sources, optionally resizing or transforming them,
|
||||
and serves the results. Both source and transformed images are cached to disk so
|
||||
that subsequent requests are served without origin fetches or additional
|
||||
processing.
|
||||
|
||||
## Getting Started
|
||||
@@ -27,12 +26,11 @@ make docker
|
||||
docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest
|
||||
```
|
||||
|
||||
A container takes its settings from environment variables (see
|
||||
Configuration below for the list). Only `PIXA_SIGNING_KEY` is required; if
|
||||
it is unset the container exits at startup naming the variable. Everything
|
||||
else has a built-in default. A config file mounted at `/etc/pixa/config.yml`
|
||||
is optional: it is read when present, and an environment variable wins over
|
||||
the same setting in it.
|
||||
A container takes its settings from environment variables (see Configuration
|
||||
below for the list). Only `PIXA_SIGNING_KEY` is required; if it is unset the
|
||||
container exits at startup naming the variable. Everything else has a built-in
|
||||
default. A config file mounted at `/etc/pixa/config.yml` is optional: it is read
|
||||
when present, and an environment variable wins over the same setting in it.
|
||||
|
||||
## Deployment
|
||||
|
||||
@@ -85,47 +83,54 @@ which answers 200 whenever pixa is running, in maintenance mode too (see
|
||||
`maintenance_mode`).
|
||||
|
||||
On SIGTERM or SIGINT pixa stops accepting connections, gives the requests in
|
||||
progress and the images being processed 5 seconds to finish, and exits: with 0,
|
||||
or with 1 when images were still being processed after those 5 seconds or
|
||||
another part of pixa failed to stop. A request not finished by then is cut off.
|
||||
`docker stop` waits 10 seconds before it kills the container.
|
||||
progress and the images being processed 5 seconds to finish, then writes to the
|
||||
database the counts of cache hits, misses, fetches and conversions that requests
|
||||
could not write by their deadline, and exits: with 0, or with 1 when images were
|
||||
still being processed after those 5 seconds, some of those counts could not be
|
||||
written, or another part of pixa failed to stop. A request not finished by then
|
||||
is cut off. `docker stop` waits 10 seconds before it kills the container.
|
||||
|
||||
Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
|
||||
it uses libvips through CGO; building it also needs their development files,
|
||||
`pkg-config` and a C compiler. `script/bootstrap` installs all of these with
|
||||
nix, apt, brew or apk.
|
||||
Outside Docker, pixa needs libvips (the image has 8.16) and libheif to run, as
|
||||
it uses libvips through CGO. pixad does not start unless libvips has its JPEG XL
|
||||
support, which on Alpine is the `vips-jxl` package and which the nix and brew
|
||||
packages of libvips include, as do the apt ones from Debian 12 and Ubuntu 24.04
|
||||
on. Building pixa also needs the development files of libvips and libheif,
|
||||
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these,
|
||||
as the `Dockerfile` does where it compiles pixa. Plain `script/bootstrap`, which
|
||||
`script/setup` and `script/cibuild` run, installs git, make and Go, and Node,
|
||||
Yarn and the prettier pinned in `yarn.lock` for formatting the markdown, but
|
||||
none of the C libraries: the checks compile pixa in Docker. Docker itself must
|
||||
already be installed.
|
||||
|
||||
## Running under upaas
|
||||
|
||||
What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
|
||||
|
||||
- **Port:** pixa listens on container port `8080`.
|
||||
- **Volume:** container path `/var/lib/pixa`, where pixa keeps its
|
||||
database and cache. Creating the host directory when it is missing is
|
||||
upaas's job, tracked in https://git.eeqj.de/sneak/upaas/issues/235.
|
||||
- **Volume:** container path `/var/lib/pixa`, where pixa keeps its database and
|
||||
cache. Creating the host directory when it is missing is upaas's job, tracked
|
||||
in https://git.eeqj.de/sneak/upaas/issues/235.
|
||||
- **Environment variables:**
|
||||
- `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs
|
||||
and login, 32+ characters, for example from
|
||||
`openssl rand -base64 32`
|
||||
- `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs and
|
||||
login, 32+ characters, for example from `openssl rand -base64 32`
|
||||
- `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature,
|
||||
comma-separated
|
||||
- `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it;
|
||||
default 75% of (free space + what the cache holds)
|
||||
- the rest are in the table under Configuration below
|
||||
- **Health check:** the image's `HEALTHCHECK` requests
|
||||
`/.well-known/healthcheck.json`. upaas reads the container's health 60
|
||||
seconds after a deploy and marks the deploy failed unless it is
|
||||
`healthy`. The probe uses the port from `PORT` (default `8080`), so a
|
||||
port changed only in a mounted config file is not seen by it: change
|
||||
the port with `PORT`.
|
||||
`/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds
|
||||
after a deploy and marks the deploy failed unless it is `healthy`. The probe
|
||||
uses the port from `PORT` (default `8080`), so a port changed only in a
|
||||
mounted config file is not seen by it: change the port with `PORT`.
|
||||
|
||||
## Rationale
|
||||
|
||||
Image-heavy web applications need a fast, caching reverse proxy that
|
||||
can resize and transcode images on the fly. pixa fills that role as a
|
||||
single, self-contained binary with no external runtime dependencies
|
||||
beyond libvips. It supports HMAC-SHA256 signed URLs with expiration to
|
||||
prevent abuse, and allowlisted source hosts for open access.
|
||||
Image-heavy web applications need a fast, caching reverse proxy that can resize
|
||||
and transcode images on the fly. pixa fills that role as a single,
|
||||
self-contained binary with no external runtime dependencies beyond libvips. It
|
||||
supports HMAC-SHA256 signed URLs with expiration to prevent abuse, and
|
||||
allowlisted source hosts for open access.
|
||||
|
||||
## Design
|
||||
|
||||
@@ -134,8 +139,8 @@ prevent abuse, and allowlisted source hosts for open access.
|
||||
- **Source content**:
|
||||
`<state_dir>/cache/sources/<ab>/<cd>/<sha256 of source content>`
|
||||
- **Source metadata**:
|
||||
`<state_dir>/cache/metadata/<hostname>/<sha256 of path and query>.json`
|
||||
(host, path and query, content hash, upstream status and headers, fetch time)
|
||||
`<state_dir>/cache/metadata/<hostname>/<sha256 of path and query>.json` (host,
|
||||
path and query, content hash, upstream status and headers, fetch time)
|
||||
- **Database**: `<state_dir>/state.sqlite3` (SQLite)
|
||||
- **Transformed images**:
|
||||
`<state_dir>/cache/variants/<ab>/<cd>/<sha256 of host, path, query, size, format, quality and fit>`,
|
||||
@@ -144,12 +149,13 @@ prevent abuse, and allowlisted source hosts for open access.
|
||||
`<ab>` and `<cd>` are the first and second pairs of characters of the file's
|
||||
name.
|
||||
|
||||
Multiple source paths may reference the same content blob; the
|
||||
database tracks references rather than using filesystem refcounting.
|
||||
Toward a target of 1-5k r/s, pixa keeps in memory the content types of
|
||||
the 10,000 transformed images most recently cached or served, so a
|
||||
cache hit on one of them reads only the image file from disk and not
|
||||
the metadata file stored beside it.
|
||||
Multiple source paths may reference the same content blob; the database tracks
|
||||
references rather than using filesystem refcounting.
|
||||
|
||||
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
||||
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
||||
transformed images most recently cached or served, so a cache hit on one of them
|
||||
reads only the image file from disk and not the metadata file stored beside it.
|
||||
|
||||
### Routes
|
||||
|
||||
@@ -159,8 +165,8 @@ answers any method as it answers `GET`. A browser's CORS preflight request
|
||||
(`OPTIONS` with `Origin` and `Access-Control-Request-Method` headers) to any
|
||||
path under `/v1/` answers 200, in maintenance mode too.
|
||||
|
||||
- `GET /` — the login page, or the URL generator page with a login session
|
||||
(see Encrypted URLs). Needs: nothing. Answers: 200.
|
||||
- `GET /` — the login page, or the URL generator page with a login session (see
|
||||
Encrypted URLs). Needs: nothing. Answers: 200.
|
||||
- `POST /` — log in with the signing key typed into the login page. Needs: the
|
||||
login page's form (below). Answers: 303 to `/` with a login session cookie
|
||||
that lasts 30 days for the right key; 200 with the login page and an error for
|
||||
@@ -171,33 +177,36 @@ path under `/v1/` answers 200, in maintenance mode too.
|
||||
with the page naming a field that is not valid; 500 when the URL cannot be
|
||||
made.
|
||||
- `GET /logout` — end the login session. Needs: nothing. Answers: 303 to `/`.
|
||||
- `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>` — an image, fetched,
|
||||
resized and converted (below). Needs: a signature, unless the host is
|
||||
allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches
|
||||
the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a
|
||||
missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when
|
||||
the request's `Referer` names a host in `referer_blocklist`, checked before
|
||||
the signature, the cache and the upstream fetch; 403 when the upstream host,
|
||||
or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`,
|
||||
or has an address in a blocked network (see `blocked_networks`); 502 when the
|
||||
upstream answered with an error status, and for 5 minutes after that for the
|
||||
same source URL; 503 when pixa is busy or in maintenance mode; 500 for any
|
||||
other failure.
|
||||
- `GET` or `HEAD` `/v1/image/<host>/<path>/<size>.<format>`, or
|
||||
`/v1/image/<host>/<path>/<size>` with no format — an image, fetched, resized
|
||||
and converted (below). Needs: a signature, unless the host is allowlisted (see
|
||||
Source Hosts). Answers: 200; 304 when `If-None-Match` matches the image's
|
||||
`ETag`; 400 for a URL or parameter that is not valid, or for the format `auto`
|
||||
an `Accept` header that is not valid; 406 for the format `auto` when `Accept`
|
||||
allows none of the formats it chooses from; 401 for a missing or wrong
|
||||
signature, a missing `exp` or an `exp` in the past; 403 when the request's
|
||||
`Referer` names a host in `referer_blocklist`, checked before the signature,
|
||||
the cache and the upstream fetch; 403 when the upstream host, or a host it
|
||||
redirects to, is `localhost`, ends in `.localhost` or `.local`, or has an
|
||||
address in a blocked network (see `blocked_networks`); 502 when the upstream
|
||||
answered with an error status, and for 5 minutes after that for the same
|
||||
source URL; 503 when pixa is busy or in maintenance mode; 500 for any other
|
||||
failure.
|
||||
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
||||
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
||||
decrypt, or that asks for a size or fit that is not valid; 410 once it has
|
||||
expired; 504 when the upstream has not sent its response headers within
|
||||
`upstream_fetch_timeout`, but 500 when that time runs out while the image
|
||||
itself is still arriving; 403, 502, 503 and 500 as for `/v1/image/`.
|
||||
itself is still arriving; 400 for an `Accept` header that is not valid, and
|
||||
406, 403, 502, 503 and 500, as for `/v1/image/`.
|
||||
- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs:
|
||||
nothing. Answers: 200.
|
||||
- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`,
|
||||
`uptime_seconds`, `uptime_human`, `version`, `appname` and
|
||||
`maintenance_mode`. Needs: nothing. Answers: 200, always.
|
||||
- `GET /static/<file>` — the stylesheet and script the login and generator
|
||||
pages load. Needs: nothing. Answers: 200, or 404 for a file that does not
|
||||
exist.
|
||||
`uptime_seconds`, `uptime_human`, `version`, `appname` and `maintenance_mode`.
|
||||
Needs: nothing. Answers: 200, always.
|
||||
- `GET /static/<file>` — the stylesheet and script the login and generator pages
|
||||
load. Needs: nothing. Answers: 200, or 404 for a file that does not exist.
|
||||
- `GET /metrics` — Prometheus metrics (see Architecture). Needs: HTTP basic
|
||||
authentication with `metrics.username` and `metrics.password`. Answers: 200;
|
||||
401 without them; 404 when they are not set, as the route then does not exist.
|
||||
@@ -221,62 +230,87 @@ HTTP is for development on the browser's own machine: the login session cookie
|
||||
is always marked `Secure`, and over plain HTTP a browser keeps such a cookie
|
||||
only for its own machine (`localhost`), if at all. A form is also refused with
|
||||
403 when the page's host is not the `Host` header pixa receives, so a reverse
|
||||
proxy in front of pixa must pass that header on unchanged. A form body over
|
||||
1 MiB is refused with 413. The image routes answer the errors listed for them
|
||||
with JSON holding `error`, `status` and `timestamp`.
|
||||
proxy in front of pixa must pass that header on unchanged. A form body over 1
|
||||
MiB is refused with 413. The image routes answer the errors listed for them with
|
||||
JSON holding `error`, `status` and `timestamp`.
|
||||
|
||||
An image URL has this form:
|
||||
An image URL has one of these forms, the second with no format:
|
||||
|
||||
```
|
||||
/v1/image/<host>/<path>/<size>.<format>?sig=<signature>&exp=<expiration>&q=<quality>&fit=<fit>
|
||||
/v1/image/<host>/<path>/<size>?sig=<signature>&exp=<expiration>&q=<quality>&fit=<fit>
|
||||
```
|
||||
|
||||
Images are only fetched from origins using TLS with valid certificates, unless
|
||||
`allow_http` is set: then pixa fetches every image over plain HTTP, which is for
|
||||
testing only.
|
||||
|
||||
A request whose query string cannot be decoded, or gives any parameter more
|
||||
than once, is refused with 400.
|
||||
A request whose query string cannot be decoded, or gives any parameter more than
|
||||
once, is refused with 400.
|
||||
|
||||
- `<format>`: one of `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`,
|
||||
`avif`, `gif`
|
||||
`avif`, `jxl` (JPEG XL), `gif`, or `auto` (below). A URL with no format (the
|
||||
second form, with no dot after the size) is served as JPEG XL, the default, as
|
||||
with `jxl`
|
||||
- `<size>`: `orig` or `<width>x<height>` (e.g. `800x600`)
|
||||
- `sig` and `exp`: the signature and its expiry, needed unless the host is
|
||||
allowlisted (see Signature Specification)
|
||||
- `q` and `fit`: the output quality and how the image is fitted to `<size>`,
|
||||
both optional (values under Signature Specification). Both are part of what
|
||||
is cached, so each value of either is a separate cached image.
|
||||
both optional (values under Signature Specification). Both are part of what is
|
||||
cached, so each value of either is a separate cached image.
|
||||
|
||||
The source image may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or SVG. The
|
||||
upstream's `Content-Type` must name its format, and the image's first bytes must
|
||||
match it.
|
||||
|
||||
With the format `auto`, pixa chooses the format for each request from its
|
||||
`Accept` header, in this order:
|
||||
|
||||
1. JPEG XL, when the header names `image/jxl`;
|
||||
2. AVIF, when it names `image/avif`;
|
||||
3. WebP, when it names `image/webp`;
|
||||
4. JPEG, when the first of `image/jpeg`, `image/*` and `*/*` that it names
|
||||
allows it, or when there is no `Accept` header or it is empty.
|
||||
|
||||
An entry with `q=0` refuses its format; other `q` values do not change the
|
||||
order. JPEG XL, AVIF and WebP must be named, as clients that cannot show them
|
||||
also send `image/*` and `*/*`. pixa never sends a format the client refused:
|
||||
when the header allows none of the four, the answer is 406, and a header that
|
||||
does not parse, or has a `q` that is not a number from 0 to 1, is refused
|
||||
with 400. The signature, or the token of an encrypted URL, covers `auto` itself,
|
||||
so one URL serves every client. Each format chosen is cached as a separate
|
||||
image, and every answer that depends on `Accept` (the image, a 304, and the 400
|
||||
and 406 above) carries `Vary: Accept`, so a shared cache keeps the formats apart
|
||||
too.
|
||||
|
||||
An image is served with `Cache-Control: public, max-age=<seconds>, immutable`.
|
||||
When the URL has an expiry (an `exp`, or the TTL of an encrypted URL),
|
||||
`max-age` is the whole seconds left until then, at most one year, so no browser
|
||||
or proxy cache keeps the image after pixa would refuse the URL. A URL with no
|
||||
expiry gets one year. `immutable` only stops a client revalidating while its
|
||||
copy is fresh.
|
||||
When the URL has an expiry (an `exp`, or the TTL of an encrypted URL), `max-age`
|
||||
is the whole seconds left until then, at most one year, so no browser or proxy
|
||||
cache keeps the image after pixa would refuse the URL. A URL with no expiry gets
|
||||
one year. `immutable` only stops a client revalidating while its copy is fresh.
|
||||
|
||||
When several requests for the same image, size, format, quality and fit miss
|
||||
the cache at once, they share one upstream fetch (or one read of the cached
|
||||
source) and one transcode: the first request does the work, and the others wait
|
||||
for its image or its error, holding no upstream connection or processing slot
|
||||
of their own. A waiting request stops waiting when its own client goes away.
|
||||
The work goes on for the others even if the first request's client goes away,
|
||||
until that request's `downstream_timeout` ends. The shared fetch sends the first
|
||||
request's ID upstream, and the lines logged for the fetch and the transcode
|
||||
carry that ID.
|
||||
When several requests for the same image, size, format, quality and fit miss the
|
||||
cache at once, they share one upstream fetch (or one read of the cached source)
|
||||
and one transcode: the first request does the work, and the others wait for its
|
||||
image or its error, holding no upstream connection or processing slot of their
|
||||
own. A waiting request stops waiting when its own client goes away. The work
|
||||
goes on for the others even if the first request's client goes away, until that
|
||||
request's `downstream_timeout` ends. The shared fetch sends the first request's
|
||||
ID upstream, and the lines logged for the fetch and the transcode carry that ID.
|
||||
|
||||
The login form (`POST /`) is limited to 5 attempts per minute per client
|
||||
address, counting an IPv6 client by its /64; an attempt over the limit is
|
||||
refused with 429 and a `Retry-After` header. Behind a reverse proxy the client
|
||||
address comes from `X-Forwarded-For` only when the address pixa sees for
|
||||
requests that come through the proxy is in `trusted_proxies`; otherwise all
|
||||
users behind the proxy are counted as one client. That address is not always
|
||||
the proxy's own: a proxy on the Docker host that connects to pixa over
|
||||
`127.0.0.1` is seen as the gateway of the container's Docker network, such as
|
||||
`172.17.0.1` on the default bridge, and one that connects through another of the
|
||||
host's addresses is seen with that address. To be sure, read it as `remoteIP` in
|
||||
pixa's request log while it is not in `trusted_proxies` (see `trusted_proxies`
|
||||
under Configuration). With the default `trusted_proxies` (the RFC 1918 ranges),
|
||||
a client with a private address can choose the address it is counted by through
|
||||
users behind the proxy are counted as one client. That address is not always the
|
||||
proxy's own: a proxy on the Docker host that connects to pixa over `127.0.0.1`
|
||||
is seen as the gateway of the container's Docker network, such as `172.17.0.1`
|
||||
on the default bridge, and one that connects through another of the host's
|
||||
addresses is seen with that address. To be sure, read it as `remoteIP` in pixa's
|
||||
request log while it is not in `trusted_proxies` (see `trusted_proxies` under
|
||||
Configuration). With the default `trusted_proxies` (the RFC 1918 ranges), a
|
||||
client with a private address can choose the address it is counted by through
|
||||
its own `X-Forwarded-For`, whether it connects directly or through the proxy,
|
||||
because its own address is trusted too. Setting `trusted_proxies` to only the
|
||||
address pixa sees for requests that come through the proxy closes this.
|
||||
@@ -293,26 +327,28 @@ nor change what it asks for.
|
||||
lasts 30 days, or until `/logout`.
|
||||
2. On the generator page, give the source image's URL, the width and height, the
|
||||
format, quality and fit, and how long the URL lasts, then submit the form
|
||||
(`POST /generate`). Width and height both empty or `0` keep the original
|
||||
size; if only one of them is empty or `0`, that side is scaled to keep the
|
||||
image's proportions.
|
||||
(`POST /generate`). The format is JPEG XL unless another is chosen; a form
|
||||
sent with an empty format, or none, also makes a JPEG XL URL. Width and
|
||||
height both empty or `0` keep the original size; if only one of them is empty
|
||||
or `0`, that side is scaled to keep the image's proportions.
|
||||
3. The page shows the URL, `https://<host>/v1/e/<token>/img.<format>`, and when
|
||||
it expires. `<host>` is the host the page was opened on, and the URL starts
|
||||
with `http` instead while `debug` is on. The name after the token is ignored
|
||||
and only gives the URL a file extension, `jpg` for `orig`.
|
||||
and only gives the URL a file extension, `jpg` for `orig` and `auto`, and
|
||||
`jxl` for a form with no format.
|
||||
|
||||
The token holds the source's host, path and query and the size, format,
|
||||
quality, fit and expiry, encrypted with a key derived from `signing_key`. The
|
||||
source URL's scheme is not kept: the image is fetched like any other (see
|
||||
Routes), and the blocked networks still apply.
|
||||
The token holds the source's host, path and query and the size, format, quality,
|
||||
fit and expiry, encrypted with a key derived from `signing_key`. The source
|
||||
URL's scheme is not kept: the image is fetched like any other (see Routes), and
|
||||
the blocked networks still apply.
|
||||
|
||||
How long the URL lasts is chosen on the page, from 1 minute to 1 year, or
|
||||
never. The expiry is fixed in the token when the URL is made and cannot be
|
||||
changed or revoked afterwards. Until then the image is served with a `max-age`
|
||||
that ends at the expiry (see Routes); after it the URL answers 410
|
||||
`URL has expired`. A URL made to last forever stops working only when
|
||||
`signing_key` changes: changing it makes every encrypted URL already handed out
|
||||
answer 400, and ends every login session.
|
||||
How long the URL lasts is chosen on the page, from 1 minute to 1 year, or never.
|
||||
The expiry is fixed in the token when the URL is made and cannot be changed or
|
||||
revoked afterwards. Until then the image is served with a `max-age` that ends at
|
||||
the expiry (see Routes); after it the URL answers 410 `URL has expired`. A URL
|
||||
made to last forever stops working only when `signing_key` changes: changing it
|
||||
makes every encrypted URL already handed out answer 400, and ends every login
|
||||
session.
|
||||
|
||||
### Image Metadata
|
||||
|
||||
@@ -328,19 +364,22 @@ turned off.
|
||||
- An image with an ICC profile is converted to sRGB first, since clients show an
|
||||
image with no profile as sRGB. Colours outside sRGB, such as the most
|
||||
saturated ones in a Display P3 photo, are clipped.
|
||||
- The one exception is JPEG XL with libvips 8.16 and later: libvips then writes
|
||||
an EXIF block of its own into the image, as govips cannot ask libvips to leave
|
||||
it out. It holds the image's size and otherwise fixed values, such as an
|
||||
orientation of 1 and a resolution of 72 dpi, and nothing from the source.
|
||||
|
||||
### Source Hosts
|
||||
|
||||
Source hosts may be allowlisted in the configuration. Non-allowlisted
|
||||
hosts require an HMAC-SHA256 signature.
|
||||
Source hosts may be allowlisted in the configuration. Non-allowlisted hosts
|
||||
require an HMAC-SHA256 signature.
|
||||
|
||||
#### Signature Specification
|
||||
|
||||
Signatures use HMAC-SHA256 and include an expiration timestamp to
|
||||
prevent replay attacks. Signatures are **exact match only**: every
|
||||
component (host, path, query, dimensions, format, expiration, quality,
|
||||
fit) must match exactly what was signed. No suffix matching, wildcard
|
||||
matching, or partial matching is supported.
|
||||
Signatures use HMAC-SHA256 and include an expiration timestamp to prevent replay
|
||||
attacks. Signatures are **exact match only**: every component (host, path,
|
||||
query, dimensions, format, expiration, quality, fit) must match exactly what was
|
||||
signed. No suffix matching, wildcard matching, or partial matching is supported.
|
||||
|
||||
**Signed data format** (colon-separated):
|
||||
|
||||
@@ -356,25 +395,27 @@ Where:
|
||||
- `width` — requested width in pixels, `0` for original
|
||||
- `height` — requested height in pixels, `0` for original
|
||||
- `format` — output format, one of those listed under Routes, with `original`
|
||||
signed as `orig` and `jpg` as `jpeg`
|
||||
- `expiration` — the URL's `exp` query parameter, the Unix timestamp when
|
||||
the signature expires; a request whose `exp` is not a whole number, an
|
||||
empty `exp=` included, is refused with 400
|
||||
- `quality` — the URL's `q` query parameter, a whole number from 1 to 100,
|
||||
or `85` when the URL has no `q`; a request whose `q` is anything else is
|
||||
refused with 400
|
||||
signed as `orig`, `jpg` as `jpeg`, and no format as `jxl`, so a URL with no
|
||||
format has the signature of the same URL ending in `.jxl`; `auto` is signed as
|
||||
`auto`, not as the format chosen for the request
|
||||
- `expiration` — the URL's `exp` query parameter, the Unix timestamp when the
|
||||
signature expires; a request whose `exp` is not a whole number, an empty
|
||||
`exp=` included, is refused with 400
|
||||
- `quality` — the URL's `q` query parameter, a whole number from 1 to 100, or
|
||||
`85` when the URL has no `q`; a request whose `q` is anything else is refused
|
||||
with 400
|
||||
- `fit` — the URL's `fit` query parameter (cover, contain, fill, inside,
|
||||
outside), or `cover` when the URL has no `fit`; a request whose `fit` is
|
||||
anything else, an empty `fit=` included, is refused with 400
|
||||
|
||||
The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet
|
||||
of RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa
|
||||
The URL's `sig` is the HMAC-SHA256 result in base64url (the URL-safe alphabet of
|
||||
RFC 4648) with the trailing `=` padding kept, 44 characters in all. pixa
|
||||
compares it exactly, so a signature encoded without padding, as Node's
|
||||
`base64url` and Go's `base64.RawURLEncoding` do, is refused with 401.
|
||||
|
||||
**Example:** with the signing key `example-signing-key-for-documentation`,
|
||||
resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP with
|
||||
expiration 1704067200, default quality and fit:
|
||||
resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP with expiration
|
||||
1704067200, default quality and fit:
|
||||
|
||||
1. Build input:
|
||||
`cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover`
|
||||
@@ -402,29 +443,29 @@ or a `*.` wildcard, aborts startup.
|
||||
|
||||
### Configuration
|
||||
|
||||
Every setting can be given as an environment variable, in a YAML config
|
||||
file (`--config`), or both. A variable present in the environment wins over
|
||||
the file, even when it is empty, and the file wins over the built-in
|
||||
default. The one exception is a variable named in the file's `env:` section:
|
||||
it is set while the file loads, so it overrides both the environment the
|
||||
process was started with and the file's own key. A variable's value is
|
||||
parsed as the same text in the file would be. The three lists take
|
||||
comma-separated entries, with the spaces around each trimmed; an empty
|
||||
variable is an empty list. A value that does not parse or is invalid aborts
|
||||
startup, naming the variable. A variable whose name starts with `PIXA_` but
|
||||
is not in the table below, such as a misspelled one or `PIXA_PORT`, aborts
|
||||
startup naming it, as an unknown config key does. The one other accepted
|
||||
name is `PIXA_CONFIG_PATH`, the config file's path (like `--config`). The
|
||||
variables set by the file's `env:` section are checked the same way.
|
||||
Every setting can be given as an environment variable, in a YAML config file
|
||||
(`--config`), or both. A variable present in the environment wins over the file,
|
||||
even when it is empty, and the file wins over the built-in default. The one
|
||||
exception is a variable named in the file's `env:` section: it is set while the
|
||||
file loads, so it overrides both the environment the process was started with
|
||||
and the file's own key. A variable's value is parsed as the same text in the
|
||||
file would be. The three lists take comma-separated entries, with the spaces
|
||||
around each trimmed; an empty variable is an empty list. A value that does not
|
||||
parse or is invalid aborts startup, naming the variable. A variable whose name
|
||||
starts with `PIXA_` but is not in the table below, such as a misspelled one or
|
||||
`PIXA_PORT`, aborts startup naming it, as an unknown config key does. The one
|
||||
other accepted name is `PIXA_CONFIG_PATH`, the config file's path (like
|
||||
`--config`). The variables set by the file's `env:` section are checked the same
|
||||
way.
|
||||
|
||||
pixa reads at most one config file: the one given with `--config` (or `-c`),
|
||||
otherwise the one `PIXA_CONFIG_PATH` names, otherwise the first of these that
|
||||
pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`,
|
||||
`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml`
|
||||
and `config.yaml` in the working directory. A named file that does not exist,
|
||||
cannot be read or does not parse aborts startup. Of the files pixa looks for on
|
||||
its own, only one that does not exist is passed over, without a message. One
|
||||
that pixa cannot read or parse aborts startup, naming the file. So does one in a
|
||||
`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml` and
|
||||
`config.yaml` in the working directory. A named file that does not exist, cannot
|
||||
be read or does not parse aborts startup. Of the files pixa looks for on its
|
||||
own, only one that does not exist is passed over, without a message. One that
|
||||
pixa cannot read or parse aborts startup, naming the file. So does one in a
|
||||
directory pixa may not enter, whether or not it is there, since pixa cannot
|
||||
tell. With no file, pixa uses the environment and the defaults.
|
||||
|
||||
@@ -459,11 +500,11 @@ Key settings in more detail:
|
||||
of the image routes, `/v1/image/` and `/v1/e/`, sent as the CORS
|
||||
`Access-Control-Allow-Origin` header; no other route sends it. `*`, the
|
||||
default, is any site; otherwise one `http` or `https` origin such as
|
||||
`https://example.com`, whose host is a lowercase host name (letters,
|
||||
digits, hyphens and dots, with a letter in its last part) or an IP address
|
||||
(IPv6 in brackets, in its shortest form), with an optional port 1-65535
|
||||
that has no leading zero and is not the scheme's default. Any other value,
|
||||
including another scheme such as a browser extension's, aborts startup
|
||||
`https://example.com`, whose host is a lowercase host name (letters, digits,
|
||||
hyphens and dots, with a letter in its last part) or an IP address (IPv6 in
|
||||
brackets, in its shortest form), with an optional port 1-65535 that has no
|
||||
leading zero and is not the scheme's default. Any other value, including
|
||||
another scheme such as a browser extension's, aborts startup
|
||||
- `allowlist_hosts` — list of allowed upstream hosts
|
||||
- `referer_blocklist` — list of hosts whose pages may not show pixa's images, to
|
||||
stop other sites hotlinking them. Entries are written and matched as for
|
||||
@@ -472,40 +513,37 @@ Key settings in more detail:
|
||||
`/v1/e/` whose `Referer` header names a listed host is refused with 403 before
|
||||
its signature or token is checked and before the cache or the upstream host is
|
||||
used, so it fetches nothing, and it is refused even when the image is cached.
|
||||
A request with no `Referer`, or one that does not parse as a URL
|
||||
with a host, is served, as many clients send none. So this is easily got
|
||||
around: a site whose pages send no `Referer` (for example with
|
||||
A request with no `Referer`, or one that does not parse as a URL with a host,
|
||||
is served, as many clients send none. So this is easily got around: a site
|
||||
whose pages send no `Referer` (for example with
|
||||
`Referrer-Policy: no-referrer`) is not stopped. It does not apply to the login
|
||||
and generator pages. Default: empty
|
||||
- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection,
|
||||
added to the always-enforced built-in ranges (loopback, private,
|
||||
link-local, CGNAT, benchmark, NAT64, and the like); an invalid CIDR
|
||||
aborts startup
|
||||
- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front
|
||||
of pixa. `X-Forwarded-For` is believed only when the direct peer falls
|
||||
inside one of these ranges; the logged and login-recorded client
|
||||
address is then the rightmost forwarded entry that is not itself a
|
||||
trusted proxy. Otherwise the direct peer address is used and the header
|
||||
is ignored, so a client connecting directly from an address outside
|
||||
these ranges cannot spoof its address.
|
||||
An omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`,
|
||||
`172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a
|
||||
proxy on a private network; an explicitly empty list (`[]`) trusts no
|
||||
one, and an explicit list replaces the default. An invalid CIDR aborts
|
||||
startup. Set this to the address pixa sees for requests that come through
|
||||
your proxy, such as `172.17.0.1/32`, when the defaults do not cover it, or
|
||||
to trust nothing else (see the login limit under Routes). For a proxy on
|
||||
the Docker host that connects to pixa over `127.0.0.1`, that address is the
|
||||
gateway of the container's Docker network (`172.17.0.1` on the default
|
||||
bridge), not the proxy's own address; a proxy that connects through another of
|
||||
the host's addresses is seen with that address. To be sure which address it
|
||||
is, set this to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request
|
||||
through the proxy, and read `remoteIP` in pixa's request log line for it
|
||||
- `upstream_fetch_timeout` — time allowed for one fetch from an upstream
|
||||
host, as a duration such as `30s` (the default) or `2m`
|
||||
- `upstream_max_response_size` — largest upstream response accepted, in
|
||||
bytes; default `52428800` (50 MiB). It also limits the image data pixa
|
||||
decodes
|
||||
- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection, added
|
||||
to the always-enforced built-in ranges (loopback, private, link-local, CGNAT,
|
||||
benchmark, NAT64, and the like); an invalid CIDR aborts startup
|
||||
- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front of
|
||||
pixa. `X-Forwarded-For` is believed only when the direct peer falls inside one
|
||||
of these ranges; the logged and login-recorded client address is then the
|
||||
rightmost forwarded entry that is not itself a trusted proxy. Otherwise the
|
||||
direct peer address is used and the header is ignored, so a client connecting
|
||||
directly from an address outside these ranges cannot spoof its address. An
|
||||
omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`,
|
||||
`172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a proxy on a
|
||||
private network; an explicitly empty list (`[]`) trusts no one, and an
|
||||
explicit list replaces the default. An invalid CIDR aborts startup. Set this
|
||||
to the address pixa sees for requests that come through your proxy, such as
|
||||
`172.17.0.1/32`, when the defaults do not cover it, or to trust nothing else
|
||||
(see the login limit under Routes). For a proxy on the Docker host that
|
||||
connects to pixa over `127.0.0.1`, that address is the gateway of the
|
||||
container's Docker network (`172.17.0.1` on the default bridge), not the
|
||||
proxy's own address; a proxy that connects through another of the host's
|
||||
addresses is seen with that address. To be sure which address it is, set this
|
||||
to `[]` (or `PIXA_TRUSTED_PROXIES` to empty), send a request through the
|
||||
proxy, and read `remoteIP` in pixa's request log line for it
|
||||
- `upstream_fetch_timeout` — time allowed for one fetch from an upstream host,
|
||||
as a duration such as `30s` (the default) or `2m`
|
||||
- `upstream_max_response_size` — largest upstream response accepted, in bytes;
|
||||
default `52428800` (50 MiB). It also limits the image data pixa decodes
|
||||
- `downstream_timeout` — time allowed for answering one client request, as a
|
||||
duration; default `60s`. The upstream fetch counts toward it, and so do the
|
||||
waits for an upstream connection and for a processing slot (up to 10 seconds
|
||||
@@ -513,15 +551,16 @@ Key settings in more detail:
|
||||
- `signing_key` — HMAC secret for URL signatures
|
||||
- `db_url` — the SQLite database to open; omitted, it is
|
||||
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
|
||||
database in WAL mode. pixa adds `_pragma=busy_timeout(5000)` to any `db_url`,
|
||||
so a write that finds another in progress waits up to five seconds for it
|
||||
instead of failing. WAL mode comes only from the URL: keep
|
||||
`_pragma=journal_mode(WAL)` in one you set
|
||||
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the
|
||||
disk cache entirely; omitted defaults to 75% of the sum of the free space on
|
||||
the filesystem containing `<state_dir>/cache/` and the bytes of source and
|
||||
transformed images the cache already holds, worked out at startup (minimum
|
||||
500 MiB)
|
||||
database in WAL mode. pixa opens one connection to it, so its own reads and
|
||||
writes run one at a time. pixa adds `_pragma=busy_timeout(5000)` to any
|
||||
`db_url`, so a write that finds another program writing to the file waits up
|
||||
to five seconds for it instead of failing. WAL mode comes only from the URL:
|
||||
keep `_pragma=journal_mode(WAL)` in one you set
|
||||
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the disk
|
||||
cache entirely; omitted defaults to 75% of the sum of the free space on the
|
||||
filesystem containing `<state_dir>/cache/` and the bytes of source and
|
||||
transformed images the cache already holds, worked out at startup (minimum 500
|
||||
MiB)
|
||||
- `upstream_connections` — the most connections to upstream hosts at once, all
|
||||
hosts together, on top of `upstream_connections_per_host`; default `64`. A
|
||||
fetch holds its connection until its image has been processed. A fetch that
|
||||
@@ -536,10 +575,10 @@ Key settings in more detail:
|
||||
- `maintenance_mode` — while `true`, the image routes (`/v1/image/` and
|
||||
`/v1/e/`) answer every request for an image with 503, a `Retry-After` header
|
||||
and a JSON error body. The health check (`/.well-known/healthcheck.json`)
|
||||
still answers 200 and reports `"maintenance_mode": true`. It stays 200
|
||||
because the image's Docker `HEALTHCHECK` requests it: a 503 there would make
|
||||
the container unhealthy, and upaas marks a deploy failed when its container
|
||||
is unhealthy. The login and URL generator pages and `/metrics` keep working
|
||||
still answers 200 and reports `"maintenance_mode": true`. It stays 200 because
|
||||
the image's Docker `HEALTHCHECK` requests it: a 503 there would make the
|
||||
container unhealthy, and upaas marks a deploy failed when its container is
|
||||
unhealthy. The login and URL generator pages and `/metrics` keep working
|
||||
|
||||
See `configs/config.example.yml` for all options with defaults.
|
||||
|
||||
@@ -561,28 +600,98 @@ See `configs/config.example.yml` for all options with defaults.
|
||||
This repository adheres to the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
standard: normalized scripts in `script/` are the entrypoints for the
|
||||
development workflow, and the Makefile targets are thin shims that call
|
||||
them. We provide:
|
||||
development workflow, and the Makefile targets are thin shims that call them. We
|
||||
provide:
|
||||
|
||||
- `script/bootstrap` — install all dependencies (idempotent)
|
||||
- `script/setup` — make a fresh clone ready for development
|
||||
(bootstrap, then install-precommit)
|
||||
- `script/bootstrap` — install git, make, Go, Node, Yarn and prettier and
|
||||
download the Go modules (idempotent); with `--cgo`, the C compiler and the
|
||||
libvips (with its JPEG XL support) and libheif libraries that compiling and
|
||||
testing pixa need instead of Node, Yarn and prettier
|
||||
- `script/setup` — make a fresh clone ready for development (bootstrap, then
|
||||
install-precommit)
|
||||
- `script/projectname` — output the project name ("pixa")
|
||||
- `script/test` — run the test suite
|
||||
- `script/lint` — run golangci-lint, always in a container (builds
|
||||
`Dockerfile.lint` when run outside one)
|
||||
- `script/fmt` — format all code (writes)
|
||||
- `script/fmt-check` — check formatting (read-only)
|
||||
- `script/test` — run the test suite: build the `test` phase of the
|
||||
`Dockerfile`, tagged `pixa-test`
|
||||
- `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`,
|
||||
tagged `pixa-lint`; the linter never runs on the host
|
||||
- `script/fmt` — format the Go code with gofmt and the markdown with prettier
|
||||
(writes)
|
||||
- `script/fmt-check` — check the same formatting (read-only), on the host
|
||||
- `script/check` — run test, lint, and fmt-check
|
||||
- `script/docker` — build the Docker image tagged via `script/projectname`
|
||||
- `script/docker` — build the Docker image tagged via `script/projectname`, with
|
||||
the version from `git describe`; the image's build stage depends on the `lint`
|
||||
and `test` phases, so this runs them too
|
||||
- `script/docker-smoke` — build the image, start it, wait for it to be healthy
|
||||
- `script/cibuild` — CI entrypoint: `docker build .` with a new
|
||||
`CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of
|
||||
coming from the build cache, and a green run implies a green repo
|
||||
- `script/loadtest` — measure pixad's throughput, latency and peak memory; a
|
||||
benchmark, not part of `script/check` (see Load Test)
|
||||
- `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`),
|
||||
then `script/check`, then build the image as `script/docker` does
|
||||
- `script/precommit` — pre-commit checks (`go mod tidy` guard, then
|
||||
`script/check`)
|
||||
- `script/install-precommit` — install the git pre-commit hook that
|
||||
runs `script/precommit`
|
||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||
`script/precommit`
|
||||
|
||||
Every `docker build` in these scripts passes `--no-cache`, so the lint and test
|
||||
phases run on every build instead of coming from the build cache.
|
||||
`script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`,
|
||||
`script/setup` and `script/install-precommit` are the standard copies from
|
||||
`sneak/prompts`, kept identical to them. `script/fmt` and `script/fmt-check` are
|
||||
the standard copies with pixa's `gofmt` step kept before prettier. prettier
|
||||
formats the markdown only: not the HTML templates, as it cannot parse a Go
|
||||
template action inside a tag, and not `REPO_POLICIES.md` (see
|
||||
`.prettierignore`), a copy of the one in `sneak/prompts`.
|
||||
|
||||
## Load Test
|
||||
|
||||
`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how
|
||||
much memory it uses. It is a benchmark, not a check: `script/check` does not run
|
||||
it. It needs Docker and Go.
|
||||
|
||||
```bash
|
||||
script/loadtest # 10 seconds per scenario, 4 clients
|
||||
script/loadtest 30s 32 # 30 seconds per scenario, 32 clients
|
||||
```
|
||||
|
||||
It builds the image with `script/docker` and the load tool,
|
||||
[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario
|
||||
starts a new pixad container and a new origin container, `cmd/loadtest-origin`:
|
||||
an upstream host that answers every path with the same generated 1600x1200 JPEG.
|
||||
vegeta then sends requests from the given number of clients, each sending its
|
||||
next request as soon as its last one is answered, all for an image resized to
|
||||
400x300 WebP:
|
||||
|
||||
- `hit`: the same image every time, put in the cache first;
|
||||
- `miss`: a new source image every time, so pixad fetches and converts each one;
|
||||
- `herd`: each new source image once per client in a row, so that all clients
|
||||
ask for it at the same time and share one fetch and one conversion (see
|
||||
Routes).
|
||||
|
||||
pixad refuses upstream hosts with private or local addresses, so the containers
|
||||
share a Docker network in `203.0.113.0/24`, a range set aside for documentation.
|
||||
A second run on the same Docker host while one is going fails, as it cannot
|
||||
create that network.
|
||||
|
||||
For each scenario the script prints vegeta's report and two lines of its own:
|
||||
|
||||
- `Requests [total, rate, throughput]`: the requests sent, how many were sent
|
||||
per second, and how many were answered successfully per second; the last is
|
||||
the number to compare with the target under Storage;
|
||||
- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request
|
||||
to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th
|
||||
percentiles;
|
||||
- `Status Codes` and `Error Set`: anything other than `200` means the other
|
||||
numbers are not for the scenario described, such as `503` when pixad was busy;
|
||||
- `Bytes In`: `0`, as vegeta is told not to keep the images it receives;
|
||||
- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since
|
||||
its container started, in kB; for `hit` it includes the request that put the
|
||||
image in the cache;
|
||||
- `requests to the origin`: the fetches pixad made: one for `hit`, one per
|
||||
request for `miss`, and one per image for `herd`, that is the requests sent
|
||||
divided by the number of clients.
|
||||
|
||||
The numbers depend on the machine and on whatever else runs on it. The first
|
||||
measurement, made on a shared machine with few clients, is in `TODO.md`; it says
|
||||
nothing about the target.
|
||||
|
||||
## TODO
|
||||
|
||||
|
||||
@@ -1,28 +1,27 @@
|
||||
# Workflow
|
||||
|
||||
* branch per issue from `next`
|
||||
* do the work in Next Step
|
||||
* move Next Step to the top of Completed Steps
|
||||
* `TODO.md` merges with git's union merge (`.gitattributes`), which never
|
||||
- branch per issue from `next`
|
||||
- do the work in Next Step
|
||||
- move Next Step to the top of Completed Steps
|
||||
- `TODO.md` merges with git's union merge (`.gitattributes`), which never
|
||||
reports a conflict: read the merged entries after every merge or rebase
|
||||
* move the top item of Future Steps into Next Step
|
||||
* commit (`TODO.md` changes in the same commit as the work)
|
||||
* open a PR based on `next`
|
||||
* an independent reviewer who did not write the change gates it
|
||||
* the manager squash-merges the PR into `next` once review passes
|
||||
* `next` stays green and mergeable to `main` at any time; only the owner
|
||||
merges `next` into `main`, via the single milestone PR
|
||||
* push
|
||||
- move the top item of Future Steps into Next Step
|
||||
- commit (`TODO.md` changes in the same commit as the work)
|
||||
- open a PR based on `next`
|
||||
- an independent reviewer who did not write the change gates it
|
||||
- the manager squash-merges the PR into `next` once review passes
|
||||
- `next` stays green and mergeable to `main` at any time; only the owner merges
|
||||
`next` into `main`, via the single milestone PR
|
||||
- push
|
||||
|
||||
# Status
|
||||
|
||||
pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work
|
||||
lands on `next`, and `main` receives only the milestone PR that the
|
||||
owner merges. `next` is at the canonical `golangci-lint` v2.12.2 config
|
||||
and is green. Recent work extracted the internal/magic,
|
||||
internal/allowlist, internal/httpfetcher, and internal/signature
|
||||
packages. The gosec findings from the 2026-07-06 survey are resolved.
|
||||
The disk cache is now size-bounded with LRU eviction
|
||||
pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work lands on
|
||||
`next`, and `main` receives only the milestone PR that the owner merges. `next`
|
||||
is at the canonical `golangci-lint` v2.12.2 config and is green. Recent work
|
||||
extracted the internal/magic, internal/allowlist, internal/httpfetcher, and
|
||||
internal/signature packages. The gosec findings from the 2026-07-06 survey are
|
||||
resolved. The disk cache is now size-bounded with LRU eviction
|
||||
(`cache_max_bytes`), closing the unbounded disk growth DoS vector.
|
||||
|
||||
# Next Step
|
||||
@@ -31,6 +30,172 @@ P2: security: per-IP rate limiting on the image routes
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-08 a request past its deadline no longer waits for the database to
|
||||
count it (closes #224). On a busy host, `TestService_Get_ReturnsByItsDeadline`
|
||||
failed because its request, past its deadline, still waited for its miss count
|
||||
to be written, a write with no deadline, which in pixad waits for the one
|
||||
database connection every request shares. Each count write now keeps the
|
||||
request's deadline but not its cancellation. A count not written by then is
|
||||
kept in memory, where `Stats` sees it, and one goroutine of the cache writes
|
||||
those counts, one write at a time, and once more at shutdown before the
|
||||
database closes. The test phase of the `Dockerfile` also runs at most 4 tests
|
||||
of a package at once (`go test -parallel 4`), where it ran as many as the host
|
||||
has CPUs: on a busy host they then wait so long to be scheduled that a test
|
||||
that times a request can fail. `-p`, how many packages are tested at once, is
|
||||
unchanged, as capping it also slows compiling.
|
||||
- 2026-10-08 every output format is saved with settings pixa sets on purpose
|
||||
(closes #232): each format has its own govips export, as JPEG XL does, in
|
||||
place of govips' generic `Export`, which sent libvips a zero for some settings
|
||||
it was not given. PNG gets libvips' default compression, 6, where it had none,
|
||||
and WebP libvips' default effort, 4, where it had 0. GIF was already at
|
||||
libvips' default effort, 7, and JPEG output is unchanged. AVIF is saved at
|
||||
effort 1, the lowest govips can set, where it had libvips' default, 4. With
|
||||
one libvips thread, an 8192x8192 image of random pixels, the worst case, takes
|
||||
about 51 seconds as AVIF at effort 1 and 43 as WebP at effort 4, against the
|
||||
default `downstream_timeout` of 60 seconds. On an image of milder noise, which
|
||||
AVIF at effort 1 saves in about 12 seconds, effort 4 takes minutes. AVIF is
|
||||
also saved with 8 bits per sample from a 16-bit source, which libvips would
|
||||
save with 12: a 16-bit 8192x8192 image of milder noise takes about 54 seconds
|
||||
at effort 1 with 12 bits, nearly all of the default `downstream_timeout`, and
|
||||
about 12 with 8.
|
||||
- 2026-10-08 JPEG XL is the default output (closes #222): a `/v1/image/` URL
|
||||
whose last segment is a size with no format, such as `800x600` or `orig`, is
|
||||
served as JPEG XL and signed as `jxl`, so it has the signature of the same URL
|
||||
ending in `.jxl`. An encrypted URL whose token holds no format is served as
|
||||
JPEG XL (`encurl.DefaultFormat`). The generator page selects JPEG XL until
|
||||
another format is chosen, and a form with an empty format, or none, makes a
|
||||
JPEG XL URL whose name ends in `.jxl`. The image processor no longer takes an
|
||||
empty format as `orig`: both routes give every request a format, and it
|
||||
refuses a request with none. `auto` still ends with JPEG.
|
||||
- 2026-10-08 JPEG XL as an input and output format (part of #222): a source
|
||||
whose bytes start with either JPEG XL signature, the bare codestream's `FF 0A`
|
||||
or the container's, is detected as `image/jxl`, which the upstream fetch
|
||||
accepts; `orig` of such a source is JPEG XL. The format `jxl` works in plain
|
||||
and encrypted URLs and on the generator page, served as `image/jxl` and cached
|
||||
like the other formats. `auto` chooses JPEG XL first when `Accept` names
|
||||
`image/jxl`. govips sends libvips a JPEG XL distance with every save, so
|
||||
libvips ignores the quality: pixa turns `q` into a distance with libvips' own
|
||||
formula, keeping 100 lossy, and removes the metadata from the image before
|
||||
saving, as govips cannot have libvips strip it from JPEG XL. libvips 8.16 and
|
||||
later still write an EXIF block of their own into JPEG XL, from the image's
|
||||
size, orientation and resolution, and fixed values; the image is upright and
|
||||
is given 72 dpi before the save, so the block holds nothing from the source.
|
||||
An image with an ICC profile is converted to sRGB before the JPEG XL save too,
|
||||
and a CMYK image with none is converted to sRGB, as libvips cannot save CMYK
|
||||
as JPEG XL. libvips' default effort, 7, is kept. JPEG XL is not yet the
|
||||
default output.
|
||||
- 2026-10-08 pixad runs on the Alpine release it is built on (closes #229): the
|
||||
runtime stage of the `Dockerfile` uses `alpine:3.22`, the release of the
|
||||
`golang:1.25.4-alpine3.22` image that the test phase and the build stage use,
|
||||
so all three have libvips 8.16, where the runtime image had 8.15.
|
||||
- 2026-10-08 requests no longer wait behind eviction queries that read a whole
|
||||
table (closes #227): the new `cache_usage` table holds the total cache usage,
|
||||
kept up to date by triggers on `source_content` and `variant_content` in the
|
||||
statement that adds, removes or resizes a row, and `UsageBytes` reads it. The
|
||||
reconciliation pass reads the content tables 1000 rows per query, sums them
|
||||
and corrects the total when it differs, unless a row changed while it summed.
|
||||
Source rows get `last_accessed_at` when added, so choosing source images to
|
||||
evict reads that column's index instead of sorting the whole table.
|
||||
- 2026-10-08 libvips' JPEG XL support is installed and required (part of #222):
|
||||
`script/bootstrap --cgo` installs `vips-jxl` when its package manager is apk,
|
||||
as Alpine's `vips` package lacks the support, and the runtime stage of the
|
||||
`Dockerfile` installs it too. `imgcache.NewService` fails, naming the fix,
|
||||
when `imageprocessor.CheckJPEGXLSupport` finds that libvips cannot load and
|
||||
save JPEG XL, so pixad does not start without it. A test saves an image as
|
||||
JPEG XL with govips and loads it back. JPEG XL is not yet a format pixa
|
||||
serves.
|
||||
- 2026-10-08 SQLite writes no longer fail with "database is locked" under load
|
||||
(closes #223): `internal/database` opens the database with one connection, so
|
||||
pixa's own reads and writes run on it one at a time instead of competing for
|
||||
SQLite's lock, where a write that kept losing could wait past the five-second
|
||||
busy timeout and be lost. The busy timeout stays, for another program writing
|
||||
to the same file. No code in pixa keeps rows or a transaction open while it
|
||||
runs another query, which with one connection would wait forever.
|
||||
- 2026-10-05 the format `auto` (closes #88): a format in the `/v1/image/` path,
|
||||
an encrypted URL's token and the generator page's format choice, chosen for
|
||||
each request from `Accept` once the signature or token is checked: AVIF when
|
||||
the header names `image/avif`, else WebP when it names `image/webp`, else JPEG
|
||||
when the first of `image/jpeg`, `image/*` and `*/*` that it names allows it,
|
||||
or when it names nothing; `q=0` refuses a format. AVIF and WebP must be named,
|
||||
as clients that cannot show them send the wildcards too. A header that allows
|
||||
none of the three answers 406, one that does not parse 400. The signature and
|
||||
the token cover `auto` itself; the cache key and `ETag` use the format chosen.
|
||||
Answers from the point the format is chosen carry `Vary: Accept`, next to the
|
||||
CORS `Vary: Origin`; fixed-format answers do not.
|
||||
- 2026-10-05 the markdown is formatted with prettier (closes #100): `script/fmt`
|
||||
and `script/fmt-check` run prettier 3.8.1, pinned in `package.json` and
|
||||
`yarn.lock`, on `**/*.md` after `gofmt`, with four-space tabs and
|
||||
`proseWrap: always` as `.prettierrc` says; `.prettierignore` keeps it off
|
||||
`REPO_POLICIES.md`, the copy from `sneak/prompts`, and `vendor/`. Plain
|
||||
`script/bootstrap` installs Node and Yarn as the one in `sneak/prompts` does
|
||||
and then prettier; `script/bootstrap --cgo` does not, as the `Dockerfile`
|
||||
stages that run it format nothing. The HTML templates stay unformatted:
|
||||
prettier cannot parse a Go template action inside a tag. The markdown was
|
||||
reflowed in a commit of its own.
|
||||
- 2026-10-05 lint and tests run as the `lint` and `test` phases of the
|
||||
`Dockerfile`, built with `--no-cache` (closes #202): `script/check`,
|
||||
`script/cibuild`, `script/docker`, `script/lint`, `script/test`,
|
||||
`script/setup` and `script/install-precommit` are now the copies from
|
||||
`sneak/prompts` `main`, unchanged. The `lint` phase runs golangci-lint from
|
||||
the image `REPO_POLICIES.md` names, with `libvips-dev` from `apt-get`; the
|
||||
`test` phase runs the tests with a 90-second timeout; the build stage depends
|
||||
on both. `Dockerfile.lint` and the `CHECK_EPOCH` build argument are gone, the
|
||||
formatting check runs on the host, and `make docker-versioned` and
|
||||
`make docker-test` call the scripts. `script/bootstrap`, `script/fmt`,
|
||||
`script/fmt-check`, `script/precommit` and `script/projectname` stay pixa's
|
||||
own. `script/bootstrap` installs git, make and Go, refreshing apt's package
|
||||
lists before its first apt install; with `--cgo`, which only the `test` phase
|
||||
and the build stage pass, it also installs the C compiler and the libvips and
|
||||
libheif libraries. The stage that compiles still takes the version from
|
||||
`git describe` when no `VERSION` is given, per
|
||||
https://git.eeqj.de/sneak/pixa/issues/166, so the copied scripts' comment that
|
||||
`.dockerignore` leaves out `.git` does not hold for pixa.
|
||||
- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]`
|
||||
(`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check`
|
||||
does not run, measures three scenarios, each against a new pixad container and
|
||||
a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss`
|
||||
(a new source image every request) and `herd` (each new source image asked for
|
||||
by all clients at once). For each it prints vegeta's report (requests per
|
||||
second, latency percentiles, status codes), pixad's peak resident memory and
|
||||
the requests that reached the origin. `README.md` says how to run it and read
|
||||
it, and keeps 1-5k r/s as a target not yet measured. First measurement, with
|
||||
the defaults on a shared 48-CPU machine with other work running: a baseline
|
||||
for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95
|
||||
8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so
|
||||
not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100
|
||||
MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms,
|
||||
peak 60 MiB, 188 fetches for 749 requests.
|
||||
- 2026-10-04 the CI checkout fetches the tags (closes #208): the checkout step
|
||||
in `.gitea/workflows/check.yml` sets `fetch-depth: 0`, as `REPO_POLICIES.md`
|
||||
asks of a repo that takes its version from the tags, so a CI build of a tagged
|
||||
commit stamps the tag from `git describe` instead of a bare commit.
|
||||
- 2026-10-04 `config.yml` stays out of git and the Docker build context (closes
|
||||
#212): `.gitignore` now ignores `config.yml`, the config file Getting Started
|
||||
creates with the signing key, and `.dockerignore` leaves it out in every
|
||||
directory and in any letter case, as it already did `config.yaml` and
|
||||
`config.dev.yml`.
|
||||
- 2026-10-04 local config files stay out of the Docker build context (closes
|
||||
#211): `.dockerignore` now leaves out `config.yaml` and `config.dev.yml` in
|
||||
every directory and in any letter case, the local config files `.gitignore`
|
||||
keeps out of git because they can hold the signing key.
|
||||
`configs/config.example.yml` is still sent. `config.yml`, which Getting
|
||||
Started creates, is in neither file:
|
||||
https://git.eeqj.de/sneak/pixa/issues/212.
|
||||
- 2026-10-04 `cmd/pixad/main.go` is one call into `internal/` (closes #206):
|
||||
what it did (the command line and its `--config` flag, setting
|
||||
`PIXA_CONFIG_PATH`, ignoring `SIGPIPE`, starting the fx app) is now `Run` in
|
||||
`internal/app`, unchanged, and `main` calls it with `Version`, which the build
|
||||
still sets through `-X main.Version`. That code had no tests to move.
|
||||
- 2026-10-04 `.gitignore` ignores `.claude/` (closes #204): the entry and its
|
||||
comment are copied from the canonical `.gitignore` in `sneak/prompts`,
|
||||
unanchored so it matches at every depth. `.dockerignore` already has
|
||||
`.claude`.
|
||||
- 2026-10-04 `.dockerignore` keeps secrets out at every depth (closes #205): the
|
||||
file is now the standard one from `sneak/prompts`, whose patterns match in
|
||||
every directory and, for environment files and private keys, in any letter
|
||||
case, so a nested `.env` or `server.key` no longer reaches the build context.
|
||||
pixa still sends `.git` without `.git/config` in place of the standard file's
|
||||
`.git` line, and still leaves out `.gitignore`, `/bin` and `/data`.
|
||||
- 2026-10-04 `REPO_POLICIES.md` matches the canonical copy again (closes #196):
|
||||
it is replaced, unchanged, by `prompts/REPO_POLICIES.md` from `sneak/prompts`
|
||||
`main`. The rules it adds that pixa's tree breaks are filed:
|
||||
@@ -39,11 +204,33 @@ P2: security: per-IP rate limiting on the image routes
|
||||
(the workflow's `script/docker-smoke` step),
|
||||
https://git.eeqj.de/sneak/pixa/issues/204 (`.claude/` in `.gitignore`),
|
||||
https://git.eeqj.de/sneak/pixa/issues/205 (`.dockerignore` patterns at every
|
||||
depth) and https://git.eeqj.de/sneak/pixa/issues/206 (a thin
|
||||
`cmd/pixad/main.go`). Its rule that no build stage runs `git describe` is not
|
||||
followed: pixa takes the version from the `.git` in the build context, per
|
||||
https://git.eeqj.de/sneak/pixa/issues/166, as the copy on `sneak/prompts`
|
||||
`next` already says.
|
||||
depth), https://git.eeqj.de/sneak/pixa/issues/206 (a thin `cmd/pixad/main.go`)
|
||||
and https://git.eeqj.de/sneak/pixa/issues/208 (`fetch-depth: 0` on the CI
|
||||
checkout, so the build sees the tags). Its rule that no build stage runs
|
||||
`git describe` is not followed: pixa takes the version from the `.git` in the
|
||||
build context, per https://git.eeqj.de/sneak/pixa/issues/166, as the copy on
|
||||
`sneak/prompts` `next` already says.
|
||||
- 2026-10-04 an integration test of the image proxy flow (closes #80):
|
||||
`TestImageProxyFlow` in `internal/server` starts the database, handlers and
|
||||
middleware from the constructors `pixad` uses, with a fresh state directory,
|
||||
and replaces only the upstream origin with a local test server. For a resize
|
||||
with a change to JPEG and for `orig`, the first request goes through the
|
||||
router, the real fetcher, libvips, the disk cache and SQLite and answers 200
|
||||
with the right content type and size and `X-Pixa-Cache: MISS`; the second
|
||||
answers `HIT` with the same image and the upstream has had one request; the
|
||||
source and the converted image are then in `cache/sources` and
|
||||
`cache/variants`, with their rows in `source_content`, `source_metadata` and
|
||||
`variant_content`. Two optional fields make this possible, which `pixad` does
|
||||
not set and the config file and environment cannot:
|
||||
`httpfetcher.Config.DialContext` connects in place of the dialer that refuses
|
||||
internal addresses, the URL and redirect checks still running, and
|
||||
`handlers.Params.Fetcher` replaces the fetcher the handlers build.
|
||||
- 2026-10-04 a URL made on the generator page with a `ttl` is tested to expire
|
||||
(closes #199): a new test in `internal/handlers` makes a URL on the generator
|
||||
page with a `ttl` of one second, checks that `/v1/e/` serves it at once, waits
|
||||
two seconds and checks that it then answers 410. The test waits for real, as
|
||||
pixa reads the clock directly when it makes and checks a URL; it waits two
|
||||
seconds because the time a URL expires is kept in whole seconds. Test only.
|
||||
- 2026-10-04 referer blocklist (closes #90): `referer_blocklist`
|
||||
(`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for
|
||||
`allowlist_hosts` with the same matcher; an entry of either list that is
|
||||
@@ -61,30 +248,29 @@ P2: security: per-IP rate limiting on the image routes
|
||||
`README.md`, the comments in `internal/config/config.go` and the startup error
|
||||
for the placeholder signing key name the new path; `scripts/manual-test.sh`
|
||||
and its directory are deleted, as the handler tests in `internal/handlers`
|
||||
cover every check it made except two: fetching a real image from the
|
||||
internet, and a URL made on the generator page with a `ttl` answering 410 once
|
||||
the `ttl` has passed (https://git.eeqj.de/sneak/pixa/issues/199);
|
||||
`CONVENTIONS.md` is deleted, as `REPO_POLICIES.md` links the canonical Go HTTP
|
||||
server conventions.
|
||||
cover every check it made except two: fetching a real image from the internet,
|
||||
and a URL made on the generator page with a `ttl` answering 410 once the `ttl`
|
||||
has passed (https://git.eeqj.de/sneak/pixa/issues/199); `CONVENTIONS.md` is
|
||||
deleted, as `REPO_POLICIES.md` links the canonical Go HTTP server conventions.
|
||||
- 2026-10-04 SQLite writes no longer fail with "database is locked" (closes
|
||||
#198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write
|
||||
that finds another in progress on another connection waits up to five seconds
|
||||
for it, and the default `db_url` turns on WAL mode with
|
||||
`_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a
|
||||
parameter the driver reads, so the database was never in WAL mode.
|
||||
- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup`
|
||||
only passes through a periodic pass (closes #189): it slept for three
|
||||
eviction intervals before writing its file, and a startup pass still running
|
||||
then could adopt the file itself. It now holds the test database's only
|
||||
connection until the startup pass waits for it after walking the empty
|
||||
variant directory, writes the file and lets the connection go, as
|
||||
- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup` only
|
||||
passes through a periodic pass (closes #189): it slept for three eviction
|
||||
intervals before writing its file, and a startup pass still running then could
|
||||
adopt the file itself. It now holds the test database's only connection until
|
||||
the startup pass waits for it after walking the empty variant directory,
|
||||
writes the file and lets the connection go, as
|
||||
`TestEvictionRunsOnPeriodicSchedule` does, so only a periodic reconciliation
|
||||
pass can adopt the file. Test only.
|
||||
- 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have
|
||||
handler tests (closes #77): new tests in `internal/handlers`, with no
|
||||
network, check that `GET /` without a login session shows the login form; a
|
||||
wrong key shows it again with an error and sets no session cookie; the right
|
||||
key answers 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and
|
||||
handler tests (closes #77): new tests in `internal/handlers`, with no network,
|
||||
check that `GET /` without a login session shows the login form; a wrong key
|
||||
shows it again with an error and sets no session cookie; the right key answers
|
||||
303 to `/` with a session cookie marked `Secure`, `HttpOnly` and
|
||||
`SameSite=Strict`, with which `GET /` shows the generator page; `GET /logout`
|
||||
answers 303 to `/` with an empty session cookie sent with `Max-Age=0`;
|
||||
`POST /generate` without a login session answers 303 to `/`; `/v1/e/` serves
|
||||
@@ -94,11 +280,11 @@ P2: security: per-IP rate limiting on the image routes
|
||||
- 2026-10-04 `TODO.md` merges with git's union merge (closes #190): a root
|
||||
`.gitattributes`, copied from `sneak/prompts`, marks it `merge=union`, so two
|
||||
branches that each add an entry at the top of Completed Steps merge without a
|
||||
conflict and keep both entries. Git now never reports a conflict in
|
||||
`TODO.md`: a real one keeps both versions of the lines, and two entries that
|
||||
share an identical line can end up one inside the other, which a rebase can
|
||||
do to an entry already on `next`. The Workflow above says to read the merged
|
||||
entries after every merge or rebase.
|
||||
conflict and keep both entries. Git now never reports a conflict in `TODO.md`:
|
||||
a real one keeps both versions of the lines, and two entries that share an
|
||||
identical line can end up one inside the other, which a rebase can do to an
|
||||
entry already on `next`. The Workflow above says to read the merged entries
|
||||
after every merge or rebase.
|
||||
- 2026-10-04 the default `cache_max_bytes` no longer shrinks as the cache fills
|
||||
(closes #184): for an omitted key, the cache works out the limit when it
|
||||
opens, after the database is open, as 75% of the sum of the free space on the
|
||||
@@ -122,10 +308,10 @@ P2: security: per-IP rate limiting on the image routes
|
||||
that pixa may not enter, aborts startup naming the file, as a file that does
|
||||
not parse already did.
|
||||
- 2026-10-04 `.golangci.yml` re-vendored from the canonical copy (closes #57):
|
||||
the deprecated `gomodguard` is switched off, so lint runs print no
|
||||
deprecation warning; its successor `gomodguard_v2` runs with the shared
|
||||
module block list, and `depguard` keeps `net/http/httptest` out of files that
|
||||
are not tests. The tree needed no code changes.
|
||||
the deprecated `gomodguard` is switched off, so lint runs print no deprecation
|
||||
warning; its successor `gomodguard_v2` runs with the shared module block list,
|
||||
and `depguard` keeps `net/http/httptest` out of files that are not tests. The
|
||||
tree needed no code changes.
|
||||
- 2026-10-04 the Content-Security-Policy allows no inline script or style
|
||||
(closes #125): `script-src` and `style-src` are `'self'` only. The generator
|
||||
page's two inline `onclick` handlers moved into
|
||||
@@ -144,30 +330,30 @@ P2: security: per-IP rate limiting on the image routes
|
||||
counts, the health check for a load balancer, what a stop does and its exit
|
||||
codes, and what running outside Docker needs; `configs/Caddyfile` is the
|
||||
example, checked with `caddy validate`.
|
||||
- 2026-10-04 the metrics basic auth, CORS preflight, request logging and
|
||||
metrics recording have tests (closes #79): `MetricsAuth` on its own answers
|
||||
401 with a challenge without credentials or with a wrong username or password
|
||||
and lets the configured ones through; a preflight request gets `*` for any
|
||||
origin when `access_control_allow_origin` is `*` and no
|
||||
`Access-Control-Allow-Origin` from another origin than the configured one; a
|
||||
`POST /` carrying the signing key leaves no trace of it in the request log
|
||||
line, and the login handler's own log lines leave out the submitted key; the
|
||||
metrics middleware on its own records a request it served, and the router
|
||||
records nothing while no metrics username is set. Not tested: that the router
|
||||
puts the basic auth in front of `/metrics` and records requests when a
|
||||
metrics username is set. Only one test per package can set up `/metrics`, and
|
||||
in `internal/server` that is `TestMaintenanceModeKeepsOtherRoutes`, which
|
||||
needs the owner's approval to change; #180 holds it. Tests only; the basic
|
||||
auth library already compares the password in constant time.
|
||||
- 2026-10-04 the metrics basic auth, CORS preflight, request logging and metrics
|
||||
recording have tests (closes #79): `MetricsAuth` on its own answers 401 with a
|
||||
challenge without credentials or with a wrong username or password and lets
|
||||
the configured ones through; a preflight request gets `*` for any origin when
|
||||
`access_control_allow_origin` is `*` and no `Access-Control-Allow-Origin` from
|
||||
another origin than the configured one; a `POST /` carrying the signing key
|
||||
leaves no trace of it in the request log line, and the login handler's own log
|
||||
lines leave out the submitted key; the metrics middleware on its own records a
|
||||
request it served, and the router records nothing while no metrics username is
|
||||
set. Not tested: that the router puts the basic auth in front of `/metrics`
|
||||
and records requests when a metrics username is set. Only one test per package
|
||||
can set up `/metrics`, and in `internal/server` that is
|
||||
`TestMaintenanceModeKeepsOtherRoutes`, which needs the owner's approval to
|
||||
change; #180 holds it. Tests only; the basic auth library already compares the
|
||||
password in constant time.
|
||||
- 2026-10-04 the image route's signature check and error answers are tested
|
||||
(closes #76): new tests in `internal/handlers`, with no network, check the
|
||||
status and JSON error body for a missing, wrong, unpadded, upper-case or
|
||||
expired signature on a host not on the allowlist, or a valid one sent for
|
||||
its parent domain, a sibling host, a subdomain or the host with another
|
||||
domain appended (401), an unparseable path (400), `localhost` as the
|
||||
upstream host (403) and an upstream error (502); that an allowlisted host is
|
||||
served without a signature, another host only with a valid one; and the
|
||||
answers of `/robots.txt` and the health check. No code changes.
|
||||
expired signature on a host not on the allowlist, or a valid one sent for its
|
||||
parent domain, a sibling host, a subdomain or the host with another domain
|
||||
appended (401), an unparseable path (400), `localhost` as the upstream host
|
||||
(403) and an upstream error (502); that an allowlisted host is served without
|
||||
a signature, another host only with a valid one; and the answers of
|
||||
`/robots.txt` and the health check. No code changes.
|
||||
- 2026-10-04 request IDs returned and passed on, and `/v1/e/` revalidates
|
||||
(closes #84): pixa's own `RequestID` middleware, in place of chi's, gives each
|
||||
request an ID, its own `X-Request-ID` when that is at most 64 letters, digits,
|
||||
@@ -184,26 +370,26 @@ P2: security: per-IP rate limiting on the image routes
|
||||
`Vary: Accept` is left to #88.
|
||||
- 2026-10-04 routes, encrypted URLs and config file documented (closes #75):
|
||||
"Routes" in `README.md` lists every route with its method, purpose, what it
|
||||
needs and the status codes it answers with, and says `q` and `fit` are part
|
||||
of what is cached; "Encrypted URLs" covers logging in, making one on the
|
||||
needs and the status codes it answers with, and says `q` and `fit` are part of
|
||||
what is cached; "Encrypted URLs" covers logging in, making one on the
|
||||
generator page, how long it lasts and the 410 once it has expired;
|
||||
"Configuration" gives the order in which pixa looks for its config file;
|
||||
`config.example.yml` lists `db_url` and `env` and gives every key's default;
|
||||
`scripts/manual-test.sh` is left to #97.
|
||||
- 2026-10-04 shutdown stops cache eviction in progress (closes #102):
|
||||
`StartEviction` runs the eviction goroutine with its own context, which
|
||||
`StopEviction` cancels, so a pass in progress stops at its next database
|
||||
call, file, row or eviction candidate instead of running to completion, and
|
||||
no pass starts after it, so a stop logs at most one warning;
|
||||
`StopEviction` takes a context and, when that context ends before the
|
||||
goroutine exits, stops waiting and returns its error; the handlers' stop hook
|
||||
passes fx's stop context, so an eviction still running when fx's stop
|
||||
deadline ends fails the stop and makes the exit code 1.
|
||||
`StopEviction` cancels, so a pass in progress stops at its next database call,
|
||||
file, row or eviction candidate instead of running to completion, and no pass
|
||||
starts after it, so a stop logs at most one warning; `StopEviction` takes a
|
||||
context and, when that context ends before the goroutine exits, stops waiting
|
||||
and returns its error; the handlers' stop hook passes fx's stop context, so an
|
||||
eviction still running when fx's stop deadline ends fails the stop and makes
|
||||
the exit code 1.
|
||||
- 2026-10-04 dead code in `internal/imgcache` is gone (closes #73): `Purge`,
|
||||
which only returned an error and which nothing called, is no longer part of
|
||||
the `ImageCache` interface or `Service`; the `SignatureValidator`,
|
||||
`Allowlist` and `Storage` interfaces, which nothing implemented or used, are
|
||||
deleted. Nothing else changes.
|
||||
the `ImageCache` interface or `Service`; the `SignatureValidator`, `Allowlist`
|
||||
and `Storage` interfaces, which nothing implemented or used, are deleted.
|
||||
Nothing else changes.
|
||||
- 2026-10-04 upstream host semaphores and variant `.meta` files no longer
|
||||
outlive their use (closes #87): the fetcher counts the fetches holding or
|
||||
waiting for a slot of each upstream host's semaphore and removes the host's
|
||||
@@ -215,21 +401,20 @@ P2: security: per-IP rate limiting on the image routes
|
||||
cache directories pixa uses (`cache/sources`, `cache/metadata`,
|
||||
`cache/variants`) and how files are named in each, and the comments in
|
||||
`001_schema.sql` name the same paths; the routes and the signature section
|
||||
list the same output formats, `jpg` and `original` included; the TLS
|
||||
sentence names `allow_http` as its exception; "Metrics" says only generic
|
||||
HTTP and Go runtime metrics exist, measured and served only when the metrics
|
||||
username and password are set.
|
||||
- 2026-10-03 shutdown sets the exit code and waits for image processing
|
||||
(closes #86): fx alone handles SIGINT and SIGTERM, and the server's own
|
||||
signal handler is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's
|
||||
code: 0 for a signal, 1 when the HTTP server cannot listen or the app fails
|
||||
to start or to stop; the server's stop hook, which fx waits for, stops the
|
||||
HTTP server, waits for the images still being processed, both within 5
|
||||
seconds, then flushes Sentry; images still being processed after that are
|
||||
logged with their count and make the exit code 1; a Sentry DSN that cannot be
|
||||
used fails startup, so the stop hooks of what had already started run,
|
||||
instead of exiting the process from a goroutine; the eviction loop is left to
|
||||
#102.
|
||||
list the same output formats, `jpg` and `original` included; the TLS sentence
|
||||
names `allow_http` as its exception; "Metrics" says only generic HTTP and Go
|
||||
runtime metrics exist, measured and served only when the metrics username and
|
||||
password are set.
|
||||
- 2026-10-03 shutdown sets the exit code and waits for image processing (closes
|
||||
#86): fx alone handles SIGINT and SIGTERM, and the server's own signal handler
|
||||
is gone; fx's `Run` in `cmd/pixad` exits with the shutdown's code: 0 for a
|
||||
signal, 1 when the HTTP server cannot listen or the app fails to start or to
|
||||
stop; the server's stop hook, which fx waits for, stops the HTTP server, waits
|
||||
for the images still being processed, both within 5 seconds, then flushes
|
||||
Sentry; images still being processed after that are logged with their count
|
||||
and make the exit code 1; a Sentry DSN that cannot be used fails startup, so
|
||||
the stop hooks of what had already started run, instead of exiting the process
|
||||
from a goroutine; the eviction loop is left to #102.
|
||||
- 2026-10-03 every `script/cibuild` and `script/docker` run executes the checks
|
||||
(closes #101): the `Dockerfile` declares `CHECK_EPOCH` above `make fmt-check`
|
||||
and `make lint` in the lint stage and above `make test` in the build stage,
|
||||
@@ -244,11 +429,11 @@ P2: security: per-IP rate limiting on the image routes
|
||||
upstream fetch or cached source read and one transcode through
|
||||
`golang.org/x/sync/singleflight`; the first request's processing ignores its
|
||||
cancellation but keeps its deadline, and the others wait for its image or
|
||||
error holding no upstream connection or processing slot, and stop waiting
|
||||
when their own context ends; the request doing the processing waits for it
|
||||
even then, up to its deadline; a request whose context has already ended
|
||||
starts nothing; each request counts one miss, and the processing counts its
|
||||
fetch and transcode once; a panic while processing is reported to Sentry when
|
||||
error holding no upstream connection or processing slot, and stop waiting when
|
||||
their own context ends; the request doing the processing waits for it even
|
||||
then, up to its deadline; a request whose context has already ended starts
|
||||
nothing; each request counts one miss, and the processing counts its fetch and
|
||||
transcode once; a panic while processing is reported to Sentry when
|
||||
`sentry_dsn` is set and becomes an error for every waiting request instead of
|
||||
stopping pixad; documented in `README.md`.
|
||||
- 2026-09-29 only the image routes send CORS headers (closes #98): the CORS
|
||||
@@ -257,20 +442,20 @@ P2: security: per-IP rate limiting on the image routes
|
||||
still answers a preflight `OPTIONS` request; the login and URL generator
|
||||
pages, `/metrics` and the other routes send no `Access-Control-Allow-Origin`;
|
||||
documented in `README.md` and `config.example.yml`.
|
||||
- 2026-10-02 a plain `docker build .` stamps the tag or short commit, not
|
||||
`dev` (closes #166): `.dockerignore` lets `.git` into the build context,
|
||||
without `.git/config`; with no `VERSION` build argument the `Dockerfile`
|
||||
takes the version from `git describe --tags --always`, and fails the build if
|
||||
the context carries `.git` and no version comes out; `ARG VERSION` has no
|
||||
default; pixad logs its version, with its name and architecture, as its first
|
||||
log line at startup.
|
||||
- 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes
|
||||
#159): `deploy/docker-entrypoint.sh` creates the directory if it is missing,
|
||||
gives the directory and everything in it to `pixad` when the directory or one
|
||||
of its top-level entries belongs to another user or group, sets its mode to
|
||||
`750`, then runs the server as `pixad`; data left by an earlier run under
|
||||
another uid is taken over this way; "Running under upaas" in `README.md` no
|
||||
longer tells the operator to create or chown the host directory.
|
||||
- 2026-10-02 a plain `docker build .` stamps the tag or short commit, not `dev`
|
||||
(closes #166): `.dockerignore` lets `.git` into the build context, without
|
||||
`.git/config`; with no `VERSION` build argument the `Dockerfile` takes the
|
||||
version from `git describe --tags --always`, and fails the build if the
|
||||
context carries `.git` and no version comes out; `ARG VERSION` has no default;
|
||||
pixad logs its version, with its name and architecture, as its first log line
|
||||
at startup.
|
||||
- 2026-09-29 the container makes `/var/lib/pixa` usable by itself (closes #159):
|
||||
`deploy/docker-entrypoint.sh` creates the directory if it is missing, gives
|
||||
the directory and everything in it to `pixad` when the directory or one of its
|
||||
top-level entries belongs to another user or group, sets its mode to `750`,
|
||||
then runs the server as `pixad`; data left by an earlier run under another uid
|
||||
is taken over this way; "Running under upaas" in `README.md` no longer tells
|
||||
the operator to create or chown the host directory.
|
||||
- 2026-09-29 variant content types kept in memory (closes #70):
|
||||
`Cache.metaCache` holds the content types of up to 10,000 variants in an LRU
|
||||
(`github.com/hashicorp/golang-lru/v2`), filled by `StoreVariant` and by
|
||||
@@ -309,8 +494,8 @@ P2: security: per-IP rate limiting on the image routes
|
||||
`ARG VERSION` sits just above the build, so a new version reruns neither
|
||||
`script/bootstrap` nor the tests.
|
||||
- 2026-09-29 migrations at the path `REPO_POLICIES.md` sets (closes #96): the
|
||||
migration files moved, contents unchanged, from `internal/database/schema/`
|
||||
to `internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the
|
||||
migration files moved, contents unchanged, from `internal/database/schema/` to
|
||||
`internal/db/migrations/` as `000_migration.sql` and `001_schema.sql`; the
|
||||
`internal/db/migrations` package embeds them and `internal/database` reads
|
||||
them through its `FS()`; the `internal/database` package itself stays; the
|
||||
version still comes from the filename prefix, so a database that has recorded
|
||||
@@ -325,8 +510,8 @@ P2: security: per-IP rate limiting on the image routes
|
||||
`=` padding kept, and gives the example's `sig` for a stated signing key.
|
||||
- 2026-09-29 fixed uid and gid for `pixad` (closes #151): the image creates the
|
||||
`pixad` group with gid 65532 and the `pixad` user with uid 65532, instead of
|
||||
the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad`
|
||||
is not owned on the host by a person's login account; the first-run step of
|
||||
the first free uid 1000, so a bind-mounted `/var/lib/pixa` given to `pixad` is
|
||||
not owned on the host by a person's login account; the first-run step of
|
||||
"Running under upaas" in `README.md` names the uid and gid.
|
||||
- 2026-09-29 `max-age` never outlives an expiring URL (closes #63): both image
|
||||
routes build `Cache-Control` from the request's `Expires`, which an encrypted
|
||||
@@ -335,29 +520,27 @@ P2: security: per-IP rate limiting on the image routes
|
||||
that is sooner, never negative; an allowlisted host's URL that has an `exp`
|
||||
follows it too; `immutable` stays, as freshness now ends at the expiry;
|
||||
documented in `README.md`.
|
||||
- 2026-09-28 add the four settings `README.md` documented but pixa did not
|
||||
have, which aborted startup as unknown keys (closes #61):
|
||||
- 2026-09-28 add the four settings `README.md` documented but pixa did not have,
|
||||
which aborted startup as unknown keys (closes #61):
|
||||
`access_control_allow_origin` (default `*`, the CORS origin),
|
||||
`upstream_fetch_timeout` (default `30s`), `upstream_max_response_size`
|
||||
(default 50 MiB) and `downstream_timeout` (default `60s`, both the
|
||||
server's write timeout and the per-request timeout); each has a
|
||||
`PIXA_` variable; durations are positive Go duration strings, the size a
|
||||
whole number of bytes up to 1 GiB, the origin `*` or one `http` or
|
||||
`https` origin as `README.md` describes it; an invalid value
|
||||
aborts startup naming the key and the value; documented in
|
||||
`config.example.yml` and `README.md`.
|
||||
- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats`
|
||||
counts the cached source images and processed variants (`source_content`
|
||||
plus `variant_content`) and takes their size from `Cache.UsageBytes`,
|
||||
instead of reading `request_cache` and `output_content`, which nothing
|
||||
writes; those two tables are left in the schema; a disabled disk cache
|
||||
reports no items and no size. A hit is counted even when the request
|
||||
context has ended. A miss is counted after it is served or fails, also
|
||||
when the request context has ended by then, with the bytes it read from
|
||||
upstream, so `upstream_fetch_count` and `upstream_fetch_bytes` move,
|
||||
including for an upstream body that fails partway or a fetched source
|
||||
that then fails the magic byte check; `transform_count` counts each image
|
||||
the image processor transcodes.
|
||||
(default 50 MiB) and `downstream_timeout` (default `60s`, both the server's
|
||||
write timeout and the per-request timeout); each has a `PIXA_` variable;
|
||||
durations are positive Go duration strings, the size a whole number of bytes
|
||||
up to 1 GiB, the origin `*` or one `http` or `https` origin as `README.md`
|
||||
describes it; an invalid value aborts startup naming the key and the value;
|
||||
documented in `config.example.yml` and `README.md`.
|
||||
- 2026-09-28 cache stats report real numbers (closes #56): `Cache.Stats` counts
|
||||
the cached source images and processed variants (`source_content` plus
|
||||
`variant_content`) and takes their size from `Cache.UsageBytes`, instead of
|
||||
reading `request_cache` and `output_content`, which nothing writes; those two
|
||||
tables are left in the schema; a disabled disk cache reports no items and no
|
||||
size. A hit is counted even when the request context has ended. A miss is
|
||||
counted after it is served or fails, also when the request context has ended
|
||||
by then, with the bytes it read from upstream, so `upstream_fetch_count` and
|
||||
`upstream_fetch_bytes` move, including for an upstream body that fails partway
|
||||
or a fetched source that then fails the magic byte check; `transform_count`
|
||||
counts each image the image processor transcodes.
|
||||
- 2026-09-28 strip metadata from processed images (closes #82): every output is
|
||||
exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC
|
||||
profile; the image is first turned upright with `AutoRotate` (before sizes are
|
||||
@@ -368,220 +551,201 @@ P2: security: per-IP rate limiting on the image routes
|
||||
attempts per minute per client address, and an attempt over the limit is
|
||||
refused with 429 and a `Retry-After` header; the address is the one
|
||||
`internal/clientip` resolves through `trusted_proxies`, an IPv6 client is
|
||||
counted by its /64, and an IPv4-mapped address as the IPv4 address it
|
||||
carries; the limit is a `RateLimit` middleware in `internal/middleware` on
|
||||
counted by its /64, and an IPv4-mapped address as the IPv4 address it carries;
|
||||
the limit is a `RateLimit` middleware in `internal/middleware` on
|
||||
`github.com/go-chi/httprate`, which the image routes can reuse; the library
|
||||
keeps counts for the current and the previous minute only; documented in
|
||||
`README.md`.
|
||||
- 2026-09-28 refuse an unparseable `exp` on `/v1/image/` and log swallowed
|
||||
cache errors (closes #72): an `exp` in the URL that is not a whole
|
||||
number, an empty `exp=` included, is a 400 naming `exp` and the value,
|
||||
instead of being ignored and answered with 401 as if the URL had no
|
||||
`exp`; only an `exp` missing from the URL is unchanged; `README.md` says
|
||||
so where it documents `exp`. A failed variant `.meta` write, source
|
||||
metadata JSON write, `Stats` count query, stats counter update, negative
|
||||
cache write or expired negative cache delete is now logged at `warn`
|
||||
with the path or key and the error, and stays non-fatal.
|
||||
- 2026-09-28 refuse an empty `fit` on `/v1/image/` (closes #139): a
|
||||
`fit` in the URL with an empty value (`fit=`) is a 400 naming `fit`,
|
||||
instead of being served as `cover` and verified against a signature
|
||||
made for `cover`; only a `fit` missing from the URL is still `cover`;
|
||||
any other value still goes through the existing fit-mode check;
|
||||
`README.md` says so where it documents `fit`.
|
||||
- 2026-09-28 refuse an invalid `q` on `/v1/image/` (closes #134): a `q`
|
||||
that is not a whole number from 1 to 100, an empty `q` included, is a
|
||||
400 naming `q` and the value, instead of being served at the default
|
||||
85; the route reads `q` with the generator's quality check
|
||||
(`parseFormInt` with `minQuality` and `maxQuality`); only a `q` missing
|
||||
from the URL is still 85; a query string that cannot be decoded, such
|
||||
as `q=80%`, is a 400 showing it; any query parameter given more than
|
||||
once (`q`, `fit`, `sig`, `exp` alike) is a 400 naming it, so none is
|
||||
read from its first value only; `README.md` states the range and both
|
||||
query-string rules.
|
||||
- 2026-09-28 unknown `PIXA_` environment variables abort startup (closes
|
||||
#133): a variable whose name starts with `PIXA_` but is neither a
|
||||
setting's variable nor `PIXA_CONFIG_PATH` aborts startup naming it, as
|
||||
an unknown config key does, and `PIXA_PORT` is named with a pointer to
|
||||
`PORT`; the check runs after the config file loads, so the variables
|
||||
the file's `env:` section sets are checked too; documented in
|
||||
`README.md`.
|
||||
- 2026-09-28 start on a fresh upaas volume (closes #129): the image
|
||||
starts as root only to give `/var/lib/pixa` to `pixad` when `pixad`
|
||||
does not own it (`deploy/docker-entrypoint.sh`), then runs the server
|
||||
as `pixad` through `su-exec`, so a root-owned host directory
|
||||
bind-mounted there no longer stops the container at startup;
|
||||
`README.md` gains a "Running under upaas" section.
|
||||
- 2026-09-28 run all linting in Docker via `Dockerfile.lint` +
|
||||
`script/lint` (closes #104): `make lint` calls `script/lint`, the only
|
||||
way the linter is run; inside a container (both Dockerfiles set
|
||||
`container=docker`) it runs `golangci-lint`, anywhere else it builds the
|
||||
hash-pinned `Dockerfile.lint`, whose last step runs `script/lint` again;
|
||||
the `Dockerfile` lint stage runs `make lint`; no host or nix-shell
|
||||
`golangci-lint` path remains (`script/bootstrap` installs no linter);
|
||||
a per-run `CACHEBUST` build-arg keeps the lint step from being served
|
||||
from cache, and a tmpfs mount on that step keeps Go's and
|
||||
golangci-lint's caches out of its layer, so a run leaves no large build
|
||||
cache behind; `golangci-lint config verify` stays out, as it fetches its
|
||||
schema over an unpinned live HTTPS call
|
||||
- 2026-09-28 every setting as an environment variable (closes #128, also
|
||||
covers #99): each config key can be set by `PIXA_` plus the key in upper
|
||||
case (`.` written as `_`), and the port by `PORT`; a variable present in
|
||||
the environment, even empty, wins over the config file, which wins over
|
||||
the default; the typed getters read the variable first, so every existing
|
||||
check applies to it and a bad value aborts startup naming the variable;
|
||||
lists are comma-separated, and an empty variable (or `""` in the file) is
|
||||
an empty list; the Docker image no longer bakes in `config.docker.yml` or
|
||||
passes `--config`, and its `HEALTHCHECK` probes `PORT` (default `8080`);
|
||||
the config file is looked for under `/etc/pixa` and `~/.config/pixa`
|
||||
instead of the daemon name `pixad`; documented in `README.md` and
|
||||
`config.example.yml`.
|
||||
- 2026-09-28 quality and fit in the URL signature (closes #60): the signed
|
||||
data is now `host:path:query:width:height:format:expiration:quality:fit`,
|
||||
using `85` and `cover` when the URL has no `q` or `fit`, so one signed
|
||||
URL can no longer be replayed across other quality and fit values to
|
||||
create unauthorized cache entries and transcodes; the known-answer
|
||||
vectors in `internal/signature/golden_test.go` and the README signature
|
||||
specification describe the new format.
|
||||
- 2026-09-28 Docker image healthcheck (closes #111): a `HEALTHCHECK` in
|
||||
the runtime stage probing `/.well-known/healthcheck.json` with busybox
|
||||
`wget`; `script/docker-smoke` (`make docker-smoke`) builds the image,
|
||||
starts it with a throwaway `PIXA_SIGNING_KEY`, and passes only once
|
||||
Docker reports it healthy within 30 seconds, removing the container on
|
||||
exit; the Gitea workflow runs it after `script/cibuild`.
|
||||
- 2026-09-28 refuse an unparseable `exp` on `/v1/image/` and log swallowed cache
|
||||
errors (closes #72): an `exp` in the URL that is not a whole number, an empty
|
||||
`exp=` included, is a 400 naming `exp` and the value, instead of being ignored
|
||||
and answered with 401 as if the URL had no `exp`; only an `exp` missing from
|
||||
the URL is unchanged; `README.md` says so where it documents `exp`. A failed
|
||||
variant `.meta` write, source metadata JSON write, `Stats` count query, stats
|
||||
counter update, negative cache write or expired negative cache delete is now
|
||||
logged at `warn` with the path or key and the error, and stays non-fatal.
|
||||
- 2026-09-28 refuse an empty `fit` on `/v1/image/` (closes #139): a `fit` in the
|
||||
URL with an empty value (`fit=`) is a 400 naming `fit`, instead of being
|
||||
served as `cover` and verified against a signature made for `cover`; only a
|
||||
`fit` missing from the URL is still `cover`; any other value still goes
|
||||
through the existing fit-mode check; `README.md` says so where it documents
|
||||
`fit`.
|
||||
- 2026-09-28 refuse an invalid `q` on `/v1/image/` (closes #134): a `q` that is
|
||||
not a whole number from 1 to 100, an empty `q` included, is a 400 naming `q`
|
||||
and the value, instead of being served at the default 85; the route reads `q`
|
||||
with the generator's quality check (`parseFormInt` with `minQuality` and
|
||||
`maxQuality`); only a `q` missing from the URL is still 85; a query string
|
||||
that cannot be decoded, such as `q=80%`, is a 400 showing it; any query
|
||||
parameter given more than once (`q`, `fit`, `sig`, `exp` alike) is a 400
|
||||
naming it, so none is read from its first value only; `README.md` states the
|
||||
range and both query-string rules.
|
||||
- 2026-09-28 unknown `PIXA_` environment variables abort startup (closes #133):
|
||||
a variable whose name starts with `PIXA_` but is neither a setting's variable
|
||||
nor `PIXA_CONFIG_PATH` aborts startup naming it, as an unknown config key
|
||||
does, and `PIXA_PORT` is named with a pointer to `PORT`; the check runs after
|
||||
the config file loads, so the variables the file's `env:` section sets are
|
||||
checked too; documented in `README.md`.
|
||||
- 2026-09-28 start on a fresh upaas volume (closes #129): the image starts as
|
||||
root only to give `/var/lib/pixa` to `pixad` when `pixad` does not own it
|
||||
(`deploy/docker-entrypoint.sh`), then runs the server as `pixad` through
|
||||
`su-exec`, so a root-owned host directory bind-mounted there no longer stops
|
||||
the container at startup; `README.md` gains a "Running under upaas" section.
|
||||
- 2026-09-28 run all linting in Docker via `Dockerfile.lint` + `script/lint`
|
||||
(closes #104): `make lint` calls `script/lint`, the only way the linter is
|
||||
run; inside a container (both Dockerfiles set `container=docker`) it runs
|
||||
`golangci-lint`, anywhere else it builds the hash-pinned `Dockerfile.lint`,
|
||||
whose last step runs `script/lint` again; the `Dockerfile` lint stage runs
|
||||
`make lint`; no host or nix-shell `golangci-lint` path remains
|
||||
(`script/bootstrap` installs no linter); a per-run `CACHEBUST` build-arg keeps
|
||||
the lint step from being served from cache, and a tmpfs mount on that step
|
||||
keeps Go's and golangci-lint's caches out of its layer, so a run leaves no
|
||||
large build cache behind; `golangci-lint config verify` stays out, as it
|
||||
fetches its schema over an unpinned live HTTPS call
|
||||
- 2026-09-28 every setting as an environment variable (closes #128, also covers
|
||||
#99): each config key can be set by `PIXA_` plus the key in upper case (`.`
|
||||
written as `_`), and the port by `PORT`; a variable present in the
|
||||
environment, even empty, wins over the config file, which wins over the
|
||||
default; the typed getters read the variable first, so every existing check
|
||||
applies to it and a bad value aborts startup naming the variable; lists are
|
||||
comma-separated, and an empty variable (or `""` in the file) is an empty list;
|
||||
the Docker image no longer bakes in `config.docker.yml` or passes `--config`,
|
||||
and its `HEALTHCHECK` probes `PORT` (default `8080`); the config file is
|
||||
looked for under `/etc/pixa` and `~/.config/pixa` instead of the daemon name
|
||||
`pixad`; documented in `README.md` and `config.example.yml`.
|
||||
- 2026-09-28 quality and fit in the URL signature (closes #60): the signed data
|
||||
is now `host:path:query:width:height:format:expiration:quality:fit`, using
|
||||
`85` and `cover` when the URL has no `q` or `fit`, so one signed URL can no
|
||||
longer be replayed across other quality and fit values to create unauthorized
|
||||
cache entries and transcodes; the known-answer vectors in
|
||||
`internal/signature/golden_test.go` and the README signature specification
|
||||
describe the new format.
|
||||
- 2026-09-28 Docker image healthcheck (closes #111): a `HEALTHCHECK` in the
|
||||
runtime stage probing `/.well-known/healthcheck.json` with busybox `wget`;
|
||||
`script/docker-smoke` (`make docker-smoke`) builds the image, starts it with a
|
||||
throwaway `PIXA_SIGNING_KEY`, and passes only once Docker reports it healthy
|
||||
within 30 seconds, removing the container on exit; the Gitea workflow runs it
|
||||
after `script/cibuild`.
|
||||
- 2026-09-21 trusted-proxy client IP resolution (closes #94): a
|
||||
`trusted_proxies` config key taking a list of CIDRs, parsed by the same
|
||||
`net/netip` list parser as `blocked_networks` (an invalid entry aborts
|
||||
startup naming the key and value; an omitted key defaults to the RFC 1918
|
||||
private ranges, an explicitly empty list trusts no one, and an explicit
|
||||
list replaces the default); a new
|
||||
`internal/clientip` package resolves the client address by honoring
|
||||
`X-Forwarded-For` only when the direct peer is a trusted proxy, walking
|
||||
the chain right-to-left to the rightmost non-proxy entry, so a client
|
||||
connecting directly cannot spoof its address; the resolved address is
|
||||
stored in the request context by a new middleware and used by the
|
||||
request-logging middleware and the login-attempt logs in place of the
|
||||
raw peer address; documented in `README.md` and `config.example.yml`.
|
||||
`net/netip` list parser as `blocked_networks` (an invalid entry aborts startup
|
||||
naming the key and value; an omitted key defaults to the RFC 1918 private
|
||||
ranges, an explicitly empty list trusts no one, and an explicit list replaces
|
||||
the default); a new `internal/clientip` package resolves the client address by
|
||||
honoring `X-Forwarded-For` only when the direct peer is a trusted proxy,
|
||||
walking the chain right-to-left to the rightmost non-proxy entry, so a client
|
||||
connecting directly cannot spoof its address; the resolved address is stored
|
||||
in the request context by a new middleware and used by the request-logging
|
||||
middleware and the login-attempt logs in place of the raw peer address;
|
||||
documented in `README.md` and `config.example.yml`.
|
||||
- 2026-09-21 blocked networks configuration extending SSRF protection: a
|
||||
`blocked_networks` config key taking a list of CIDRs (parsed with
|
||||
`net/netip`, an invalid entry aborts startup naming the key and value),
|
||||
added to the built-in blocklist rather than replacing it; the built-in
|
||||
ranges extended to CGNAT `100.64.0.0/10`, IETF protocol assignments
|
||||
`192.0.0.0/24`, benchmark `198.18.0.0/15`, and NAT64 `64:ff9b::/96`
|
||||
(IPv4-mapped forms covered); enforcement stays in the dial-time
|
||||
re-resolution so the DNS-rebinding window remains closed; documented in
|
||||
`README.md` and `config.example.yml`.
|
||||
- 2026-09-21 validate dimensions and fit mode on the encrypted-URL
|
||||
route and the token generator (closes #62): `imgcache.ValidateDimension`
|
||||
alone holds the `MaxDimension` bound and is used by the path parser, by
|
||||
the new `ValidateImageRequest` (which also applies `ValidateFitMode`)
|
||||
and by the generator; both the `/v1/image/` and `/v1/e/` routes call
|
||||
`ValidateImageRequest`, so an over-limit size or an unknown fit mode is a
|
||||
400 rather than an out-of-memory or a 500 from the processor; the URL
|
||||
generator answers 400 naming the field for a `width` or `height` that is
|
||||
not a number or fails the shared check, a `quality` that is not a number
|
||||
from 1 to 100, a `ttl` that is not a number from 0 to the largest number
|
||||
of seconds the expiry calculation can hold, or an unknown `fit`; an empty
|
||||
`quality` is 85 and
|
||||
an empty `ttl` never expires; the form's width and height inputs stop at
|
||||
8192
|
||||
- 2026-09-21 http.Server hardening (closes #92): added
|
||||
`HTTPReadHeaderTimeout` (10s, bounds the slowloris header dribble) and
|
||||
`HTTPIdleTimeout` (120s, bounds keep-alive reuse) alongside the
|
||||
existing timeouts and wired them onto the server; added a `LimitBody`
|
||||
middleware capping the two form POST bodies (`POST /`, `POST /generate`)
|
||||
at `MaxFormBytes` (1 MiB) and returning 413, applied ahead of the CSRF
|
||||
middleware so an oversized body is refused as 413 rather than being read
|
||||
as a missing CSRF token (403); left `WriteTimeout` at 60s unchanged
|
||||
- 2026-08-07 update golangci-lint to v2.12.2 with the canonical
|
||||
`.golangci.yml` (v2 schema, `default: all` minus six disabled
|
||||
linters, `lll` 88, tests included): bumped the pinned
|
||||
`golangci/golangci-lint:v2.12.2-alpine` image in `Dockerfile` and the
|
||||
release-archive sha256 pins in `script/bootstrap`; fixed the findings
|
||||
the stricter config surfaced (notably `paralleltest`, `wsl_v5`,
|
||||
`blocked_networks` config key taking a list of CIDRs (parsed with `net/netip`,
|
||||
an invalid entry aborts startup naming the key and value), added to the
|
||||
built-in blocklist rather than replacing it; the built-in ranges extended to
|
||||
CGNAT `100.64.0.0/10`, IETF protocol assignments `192.0.0.0/24`, benchmark
|
||||
`198.18.0.0/15`, and NAT64 `64:ff9b::/96` (IPv4-mapped forms covered);
|
||||
enforcement stays in the dial-time re-resolution so the DNS-rebinding window
|
||||
remains closed; documented in `README.md` and `config.example.yml`.
|
||||
- 2026-09-21 validate dimensions and fit mode on the encrypted-URL route and the
|
||||
token generator (closes #62): `imgcache.ValidateDimension` alone holds the
|
||||
`MaxDimension` bound and is used by the path parser, by the new
|
||||
`ValidateImageRequest` (which also applies `ValidateFitMode`) and by the
|
||||
generator; both the `/v1/image/` and `/v1/e/` routes call
|
||||
`ValidateImageRequest`, so an over-limit size or an unknown fit mode is a 400
|
||||
rather than an out-of-memory or a 500 from the processor; the URL generator
|
||||
answers 400 naming the field for a `width` or `height` that is not a number or
|
||||
fails the shared check, a `quality` that is not a number from 1 to 100, a
|
||||
`ttl` that is not a number from 0 to the largest number of seconds the expiry
|
||||
calculation can hold, or an unknown `fit`; an empty `quality` is 85 and an
|
||||
empty `ttl` never expires; the form's width and height inputs stop at 8192
|
||||
- 2026-09-21 http.Server hardening (closes #92): added `HTTPReadHeaderTimeout`
|
||||
(10s, bounds the slowloris header dribble) and `HTTPIdleTimeout` (120s, bounds
|
||||
keep-alive reuse) alongside the existing timeouts and wired them onto the
|
||||
server; added a `LimitBody` middleware capping the two form POST bodies
|
||||
(`POST /`, `POST /generate`) at `MaxFormBytes` (1 MiB) and returning 413,
|
||||
applied ahead of the CSRF middleware so an oversized body is refused as 413
|
||||
rather than being read as a missing CSRF token (403); left `WriteTimeout` at
|
||||
60s unchanged
|
||||
- 2026-08-07 update golangci-lint to v2.12.2 with the canonical `.golangci.yml`
|
||||
(v2 schema, `default: all` minus six disabled linters, `lll` 88, tests
|
||||
included): bumped the pinned `golangci/golangci-lint:v2.12.2-alpine` image in
|
||||
`Dockerfile` and the release-archive sha256 pins in `script/bootstrap`; fixed
|
||||
the findings the stricter config surfaced (notably `paralleltest`, `wsl_v5`,
|
||||
`goconst`, `lll`, `noinlineerr`, `err113`, `errcheck`, `testpackage` —
|
||||
white-box test files renamed to `*_internal_test.go`), including #55's
|
||||
code absorbed after it merged, iterating the pinned linter to
|
||||
`0 issues.`; no single finding total is substantiable, since
|
||||
golangci-lint's `uniq-by-line` reveals new findings on a line as
|
||||
others there are fixed — the documented re-measurements were 81 after
|
||||
the #53 merge and 149 after the #55 merge; three behavior changes, so
|
||||
not a pure no-op: `Cache.StoreVariant` now takes a `context.Context`
|
||||
(`noctx`), so a cancelled request skips its best-effort accounting
|
||||
row; `MetadataStorage.Store`'s cleanup defer was dead on `main` and
|
||||
leaked `.tmp-*.json` on failure, now fixed with explicit removals; and
|
||||
the `signing_key` validation error text gained `value too short: `;
|
||||
the eviction loop's uncancellable context is deferred to #102 under a
|
||||
`//nolint:contextcheck`; three `//nolint:tagliatelle` directives keep
|
||||
the snake_case JSON wire/disk formats unchanged; `make check` green
|
||||
- 2026-08-07 implement cache size management and eviction (closes
|
||||
#51): new `cache_max_bytes` config key validated by the startup
|
||||
framework (explicit values used exactly with no floor, `0` disables
|
||||
the disk cache entirely, omitted defaults to max(75% of free space
|
||||
on the filesystem containing `<state_dir>/cache/`, 500 MiB), logged
|
||||
at startup); processed variants are now tracked in the database (a
|
||||
new `variant_content` table and an LRU timestamp on `source_content`)
|
||||
so total usage is two SUMs, never a directory scan on the hot path; a
|
||||
background goroutine evicts globally least-recently-used entries
|
||||
(variants and source blobs merged) to the limit, woken by a periodic
|
||||
ticker and by write-pressure notifications from stores; a source
|
||||
blob and ALL of its `source_metadata` references are deleted in one
|
||||
transaction before the file is unlinked, so multi-referenced blobs
|
||||
are never removed while referenced and rows never point at deleted
|
||||
files; a startup and periodic reconciliation pass adopts untracked
|
||||
variant files, drops rows for missing files, removes unreachable
|
||||
source blobs, and sweeps stale temp files
|
||||
- 2026-08-07 validate configuration on startup, fail fast on bad
|
||||
config (closes #52): a config value that is set but unparseable or
|
||||
invalid aborts startup naming the key and value (defaults apply only
|
||||
to omitted keys), unknown config keys abort startup, a malformed
|
||||
config file aborts instead of being skipped, and `state_dir` is
|
||||
verified creatable and writable before the listener binds
|
||||
- 2026-08-07 manual test pass of the auth and encrypted URL flows
|
||||
against a locally built and running `pixad` (built from `main` at
|
||||
`6573b9d`, port 18099, local throwaway config); all six checks
|
||||
passed, plus all nine tests in `scripts/manual-test.sh` (closes #49):
|
||||
- [x] visit `/` and see the login form: HTTP 200, `Pixa - Login`
|
||||
page with `name="key"` password form
|
||||
- [x] wrong key shows an error: POST `/` with `key=wrong-key`
|
||||
returned HTTP 200 login page containing "Invalid signing key"
|
||||
- [x] correct signing key shows the generator form: POST `/`
|
||||
returned HTTP 303 to `/` with
|
||||
`Set-Cookie: pixa_session=...; HttpOnly; Secure; SameSite=Strict`;
|
||||
GET `/` with that cookie rendered `Pixa - URL Generator` with the
|
||||
`/generate` form and logout link
|
||||
white-box test files renamed to `*_internal_test.go`), including #55's code
|
||||
absorbed after it merged, iterating the pinned linter to `0 issues.`; no
|
||||
single finding total is substantiable, since golangci-lint's `uniq-by-line`
|
||||
reveals new findings on a line as others there are fixed — the documented
|
||||
re-measurements were 81 after the #53 merge and 149 after the #55 merge; three
|
||||
behavior changes, so not a pure no-op: `Cache.StoreVariant` now takes a
|
||||
`context.Context` (`noctx`), so a cancelled request skips its best-effort
|
||||
accounting row; `MetadataStorage.Store`'s cleanup defer was dead on `main` and
|
||||
leaked `.tmp-*.json` on failure, now fixed with explicit removals; and the
|
||||
`signing_key` validation error text gained `value too short: `; the eviction
|
||||
loop's uncancellable context is deferred to #102 under a
|
||||
`//nolint:contextcheck`; three `//nolint:tagliatelle` directives keep the
|
||||
snake_case JSON wire/disk formats unchanged; `make check` green
|
||||
- 2026-08-07 implement cache size management and eviction (closes #51): new
|
||||
`cache_max_bytes` config key validated by the startup framework (explicit
|
||||
values used exactly with no floor, `0` disables the disk cache entirely,
|
||||
omitted defaults to max(75% of free space on the filesystem containing
|
||||
`<state_dir>/cache/`, 500 MiB), logged at startup); processed variants are now
|
||||
tracked in the database (a new `variant_content` table and an LRU timestamp on
|
||||
`source_content`) so total usage is two SUMs, never a directory scan on the
|
||||
hot path; a background goroutine evicts globally least-recently-used entries
|
||||
(variants and source blobs merged) to the limit, woken by a periodic ticker
|
||||
and by write-pressure notifications from stores; a source blob and ALL of its
|
||||
`source_metadata` references are deleted in one transaction before the file is
|
||||
unlinked, so multi-referenced blobs are never removed while referenced and
|
||||
rows never point at deleted files; a startup and periodic reconciliation pass
|
||||
adopts untracked variant files, drops rows for missing files, removes
|
||||
unreachable source blobs, and sweeps stale temp files
|
||||
- 2026-08-07 validate configuration on startup, fail fast on bad config (closes
|
||||
#52): a config value that is set but unparseable or invalid aborts startup
|
||||
naming the key and value (defaults apply only to omitted keys), unknown config
|
||||
keys abort startup, a malformed config file aborts instead of being skipped,
|
||||
and `state_dir` is verified creatable and writable before the listener binds
|
||||
- 2026-08-07 manual test pass of the auth and encrypted URL flows against a
|
||||
locally built and running `pixad` (built from `main` at `6573b9d`, port 18099,
|
||||
local throwaway config); all six checks passed, plus all nine tests in
|
||||
`scripts/manual-test.sh` (closes #49):
|
||||
- [x] visit `/` and see the login form: HTTP 200, `Pixa - Login` page with
|
||||
`name="key"` password form
|
||||
- [x] wrong key shows an error: POST `/` with `key=wrong-key` returned HTTP
|
||||
200 login page containing "Invalid signing key"
|
||||
- [x] correct signing key shows the generator form: POST `/` returned HTTP
|
||||
303 to `/` with
|
||||
`Set-Cookie: pixa_session=...; HttpOnly; Secure; SameSite=Strict`; GET
|
||||
`/` with that cookie rendered `Pixa - URL Generator` with the
|
||||
`/generate` form and logout link
|
||||
- [x] a generated encrypted URL serves the image: POST `/generate`
|
||||
(ttl=3600) produced a `/v1/e/<token>/img.jpeg` URL that returned
|
||||
HTTP 200, `Content-Type: image/jpeg`, an 800x600 baseline JPEG of
|
||||
61706 bytes
|
||||
- [x] an expired URL (short TTL) returns 410: a ttl=1 URL fetched
|
||||
after 3 s returned HTTP 410 Gone with
|
||||
`{"error":"URL has expired","status":410,...}`
|
||||
- [x] logout redirects back to login: GET `/logout` returned HTTP
|
||||
303 to `/` with `Set-Cookie: pixa_session=; Max-Age=0`;
|
||||
subsequent GET `/` rendered the login form again
|
||||
- 2026-08-07 fix the two remaining gosec findings (G124 in
|
||||
internal/session): session cookies now always carry
|
||||
Secure/HttpOnly/SameSite=Strict on both the set and clear paths;
|
||||
`make check` green (closes #47)
|
||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||
Makefile shims, README Entrypoints section
|
||||
(ttl=3600) produced a `/v1/e/<token>/img.jpeg` URL that returned HTTP
|
||||
200, `Content-Type: image/jpeg`, an 800x600 baseline JPEG of 61706
|
||||
bytes
|
||||
- [x] an expired URL (short TTL) returns 410: a ttl=1 URL fetched after 3 s
|
||||
returned HTTP 410 Gone with
|
||||
`{"error":"URL has expired","status":410,...}`
|
||||
- [x] logout redirects back to login: GET `/logout` returned HTTP 303 to `/`
|
||||
with `Set-Cookie: pixa_session=; Max-Age=0`; subsequent GET `/`
|
||||
rendered the login form again
|
||||
- 2026-08-07 fix the two remaining gosec findings (G124 in internal/session):
|
||||
session cookies now always carry Secure/HttpOnly/SameSite=Strict on both the
|
||||
set and clear paths; `make check` green (closes #47)
|
||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
||||
shims, README Entrypoints section
|
||||
- 2026-04-07 extract magic byte detection into internal/magic (#42)
|
||||
- 2026-03-25 extract allowlist package from internal/imgcache (#41)
|
||||
- 2026-03-25 move schema_migrations table creation into 000.sql (#36)
|
||||
- 2026-03-20 enforce and document exact-match-only signature
|
||||
verification (#40)
|
||||
- 2026-03-20 bound imageprocessor.Process input read to prevent
|
||||
unbounded memory use (#37); consolidate appname into an
|
||||
internal/globals constant (#34)
|
||||
- 2026-03-20 enforce and document exact-match-only signature verification (#40)
|
||||
- 2026-03-20 bound imageprocessor.Process input read to prevent unbounded memory
|
||||
use (#37); consolidate appname into an internal/globals constant (#34)
|
||||
- 2026-03-18 parse version prefix from migration filenames (#33)
|
||||
- 2026-03-15 QA audit fixes for 1.0/MVP readiness (#25)
|
||||
- 2026-03-02 split Dockerfile with pre-built golangci-lint stage for
|
||||
faster CI (#23)
|
||||
- 2026-03-02 split Dockerfile with pre-built golangci-lint stage for faster CI
|
||||
(#23)
|
||||
- 2026-02-25 repo policy compliance: CI workflow, hash-pinned images,
|
||||
golangci-lint and gosec fixes of that date (#14); arm64 Docker build
|
||||
fix (#16)
|
||||
- 2026-01-08 WebP and AVIF encoding support via govips (both former P0
|
||||
image processing items, now done)
|
||||
golangci-lint and gosec fixes of that date (#14); arm64 Docker build fix (#16)
|
||||
- 2026-01-08 WebP and AVIF encoding support via govips (both former P0 image
|
||||
processing items, now done)
|
||||
|
||||
# Future Steps
|
||||
|
||||
@@ -589,13 +753,11 @@ P2: security: per-IP rate limiting on the image routes
|
||||
- per-origin rate limiting
|
||||
- P2: HTTP response handling
|
||||
- Last-Modified headers
|
||||
- Vary header for content negotiation
|
||||
- P2: auto format selection (format=auto based on Accept header)
|
||||
- P2: configuration
|
||||
- YAML config file support
|
||||
- P2: operational
|
||||
- optional Sentry error reporting
|
||||
- comprehensive request logging
|
||||
- Prometheus performance metrics
|
||||
- integration tests for the image proxy flow
|
||||
- load tests to verify the 1k to 5k req/s target
|
||||
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
||||
shared with other work
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
// Command loadtest-origin is the upstream host script/loadtest points pixad
|
||||
// at; internal/loadtestorigin says what it does.
|
||||
package main
|
||||
|
||||
import "sneak.berlin/go/pixa/internal/loadtestorigin"
|
||||
|
||||
func main() {
|
||||
loadtestorigin.Run()
|
||||
}
|
||||
+2
-61
@@ -1,69 +1,10 @@
|
||||
// Package main is the entry point for the pixad image proxy server.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
"sneak.berlin/go/pixa/internal/server"
|
||||
)
|
||||
import "sneak.berlin/go/pixa/internal/app"
|
||||
|
||||
var Version string //nolint:gochecknoglobals // set by ldflags
|
||||
|
||||
var configPath string //nolint:gochecknoglobals // cobra flag
|
||||
|
||||
func main() {
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "pixad",
|
||||
Short: "Pixa image caching proxy server",
|
||||
Run: run,
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(_ *cobra.Command, _ []string) {
|
||||
globals.Version = Version
|
||||
|
||||
// Set config path in environment if specified via flag
|
||||
if configPath != "" {
|
||||
_ = os.Setenv("PIXA_CONFIG_PATH", configPath)
|
||||
}
|
||||
|
||||
// A write to a closed stdout or stderr must not end the process.
|
||||
signal.Ignore(syscall.SIGPIPE)
|
||||
|
||||
fx.New(
|
||||
fx.Provide(
|
||||
config.New,
|
||||
database.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
logger.New,
|
||||
server.New,
|
||||
middleware.New,
|
||||
healthcheck.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
func(log *logger.Logger) { log.Identify() },
|
||||
func(*server.Server) {},
|
||||
),
|
||||
).Run()
|
||||
app.Run(Version)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
// Package app reads the pixad command line and runs the server.
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
"sneak.berlin/go/pixa/internal/server"
|
||||
)
|
||||
|
||||
var configPath string //nolint:gochecknoglobals // cobra flag
|
||||
|
||||
// Run reads the command line and runs the server until it stops, with
|
||||
// version as the version pixad logs and reports. It exits the process
|
||||
// with status 1 when the command line is not valid.
|
||||
func Run(version string) {
|
||||
globals.Version = version
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "pixad",
|
||||
Short: "Pixa image caching proxy server",
|
||||
Run: run,
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(_ *cobra.Command, _ []string) {
|
||||
// Set config path in environment if specified via flag
|
||||
if configPath != "" {
|
||||
_ = os.Setenv("PIXA_CONFIG_PATH", configPath)
|
||||
}
|
||||
|
||||
// A write to a closed stdout or stderr must not end the process.
|
||||
signal.Ignore(syscall.SIGPIPE)
|
||||
|
||||
fx.New(
|
||||
fx.Provide(
|
||||
config.New,
|
||||
database.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
logger.New,
|
||||
server.New,
|
||||
middleware.New,
|
||||
healthcheck.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
func(log *logger.Logger) { log.Identify() },
|
||||
func(*server.Server) {},
|
||||
),
|
||||
).Run()
|
||||
}
|
||||
@@ -13,11 +13,10 @@ import (
|
||||
)
|
||||
|
||||
// TestConcurrentWritesAllSucceed opens a database the way pixad does and
|
||||
// writes to it from several goroutines at once, so the writes run on
|
||||
// separate connections, as one request's writes and the background eviction
|
||||
// pass do. Every write must succeed, none failing with "database is locked",
|
||||
// whether or not db_url already has parameters, and the parameters it has
|
||||
// must still apply.
|
||||
// writes to it from several goroutines at once, as one request's writes and
|
||||
// the background eviction pass do. Every write must succeed, none failing
|
||||
// with "database is locked", whether or not db_url already has parameters,
|
||||
// and the parameters it has must still apply.
|
||||
func TestConcurrentWritesAllSucceed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -237,17 +237,18 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DB returns the underlying sql.DB.
|
||||
// DB returns the underlying sql.DB. It has one connection, so close any
|
||||
// rows and end any transaction before running another query on it; a
|
||||
// query run while they are open waits forever.
|
||||
func (s *Database) DB() *sql.DB {
|
||||
return s.db
|
||||
}
|
||||
|
||||
func (s *Database) connect(ctx context.Context) error {
|
||||
// Requests and the eviction pass write on separate connections. With
|
||||
// a busy timeout, a write that finds another one in progress waits up
|
||||
// to five seconds for it instead of failing at once with "database is
|
||||
// locked". The driver runs each _pragma parameter on every connection
|
||||
// it opens.
|
||||
// With a busy timeout, a write that finds another program writing to
|
||||
// the same database file waits up to five seconds for it instead of
|
||||
// failing at once with "database is locked". The driver runs each
|
||||
// _pragma parameter on every connection it opens.
|
||||
separator := "?"
|
||||
if strings.Contains(s.config.DBURL, "?") {
|
||||
separator = "&"
|
||||
@@ -264,6 +265,11 @@ func (s *Database) connect(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// One connection: pixa's own reads and writes run on it one at a
|
||||
// time instead of competing for SQLite's lock, where a write that
|
||||
// keeps losing can wait past the busy timeout and be lost.
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
err = db.PingContext(ctx)
|
||||
if err != nil {
|
||||
s.log.Error("failed to ping database", "error", err)
|
||||
|
||||
@@ -3,14 +3,12 @@
|
||||
|
||||
-- Source content blobs
|
||||
-- Files stored at: cache/sources/<ab>/<cd>/<sha256>
|
||||
-- last_accessed_at is NULL until the first LRU touch; eviction falls
|
||||
-- back to fetched_at for rows that have never been touched.
|
||||
CREATE TABLE IF NOT EXISTS source_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_accessed_at DATETIME
|
||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
||||
ON source_content(last_accessed_at);
|
||||
@@ -42,9 +40,8 @@ CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(conte
|
||||
-- Processed variant blobs
|
||||
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
|
||||
-- sidecar with the content type). Tracked here (like source content
|
||||
-- blobs above) so total cache usage can be computed with a SUM query,
|
||||
-- never a directory scan, and so LRU eviction has a timestamp to order
|
||||
-- on.
|
||||
-- blobs above) so total cache usage is known without a directory scan,
|
||||
-- and so LRU eviction has a timestamp to order on.
|
||||
CREATE TABLE IF NOT EXISTS variant_content (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
@@ -55,6 +52,74 @@ CREATE TABLE IF NOT EXISTS variant_content (
|
||||
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
||||
ON variant_content(last_accessed_at);
|
||||
|
||||
-- Total cache usage: the sum of size_bytes over source_content and
|
||||
-- variant_content, kept by the triggers below in the same statement
|
||||
-- that adds, removes or resizes a row, so eviction reads this one row
|
||||
-- instead of summing both tables. Each trigger also adds one to
|
||||
-- change_count; the reconciliation pass, which sums both tables a page
|
||||
-- at a time, corrects total_size_bytes only if change_count did not
|
||||
-- move while it summed.
|
||||
CREATE TABLE IF NOT EXISTS cache_usage (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
total_size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
change_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT OR IGNORE INTO cache_usage (id) VALUES (1);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_insert
|
||||
AFTER INSERT ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_delete
|
||||
AFTER DELETE ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_insert
|
||||
AFTER INSERT ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_delete
|
||||
AFTER DELETE ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
-- Output/transformed content blobs
|
||||
-- Not written: transformed images are stored in cache/variants and
|
||||
-- tracked in variant_content above.
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
// Default values for optional fields.
|
||||
const (
|
||||
DefaultQuality = 85
|
||||
DefaultFormat = imgcache.FormatOriginal
|
||||
DefaultFormat = imgcache.FormatJXL
|
||||
DefaultFitMode = imgcache.FitCover
|
||||
|
||||
// HKDF salt for URL encryption key derivation
|
||||
@@ -37,7 +37,7 @@ type Payload struct {
|
||||
SourceQuery string `cbor:"q,omitempty"` // optional
|
||||
Width int `cbor:"w,omitempty"` // 0 = original
|
||||
Height int `cbor:"ht,omitempty"` // 0 = original
|
||||
Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: orig
|
||||
Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: jxl
|
||||
Quality int `cbor:"ql,omitempty"` // default: 85
|
||||
FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover
|
||||
ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
|
||||
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
|
||||
const appname = "pixad"
|
||||
|
||||
// Version is populated from main() via ldflags.
|
||||
var Version string //nolint:gochecknoglobals // set from main
|
||||
// Version is set by app.Run to the version main was built with.
|
||||
var Version string //nolint:gochecknoglobals // set by app.Run
|
||||
|
||||
// Globals holds application-wide constants.
|
||||
type Globals struct {
|
||||
|
||||
@@ -369,10 +369,15 @@ func (s *Handlers) buildGeneratedURL(r *http.Request, token, format string) stri
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
// Determine file extension for the trailing filename
|
||||
// Determine file extension for the trailing filename. A form with no
|
||||
// format makes a token with none, which is served as encurl.DefaultFormat.
|
||||
ext := format
|
||||
if ext == "" || ext == "orig" {
|
||||
ext = "jpg" // Default extension
|
||||
|
||||
switch format {
|
||||
case "":
|
||||
ext = string(encurl.DefaultFormat)
|
||||
case "orig", "auto":
|
||||
ext = "jpg"
|
||||
}
|
||||
|
||||
return scheme + "://" + r.Host + "/v1/e/" + url.PathEscape(token) + "/img." + ext
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/session"
|
||||
@@ -241,3 +242,51 @@ func TestGeneratePost_URLServesImage(t *testing.T) {
|
||||
|
||||
requireServedPhoto(t, imageRec)
|
||||
}
|
||||
|
||||
// TestGeneratePost_URLWithTTLExpires verifies that a URL the generator page
|
||||
// makes with a ttl of one second is served by /v1/e/ at once and answers 410
|
||||
// once the ttl has passed.
|
||||
func TestGeneratePost_URLWithTTLExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
formatField: {string(imgcache.FormatJPEG)},
|
||||
ttlField: {"1"},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireServedPhoto(t, imageRec)
|
||||
|
||||
// The URL keeps the time it expires in whole seconds and is served
|
||||
// through the whole of that second, so a ttl of one second has passed
|
||||
// for certain two seconds after the URL was made.
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
imageRec = httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
t.Logf("GET %s after the ttl: %d %q", match[1], imageRec.Code, imageRec.Body)
|
||||
|
||||
if imageRec.Code != http.StatusGone {
|
||||
t.Errorf("status after the ttl = %d, want %d",
|
||||
imageRec.Code, http.StatusGone)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxtest"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
)
|
||||
|
||||
// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as
|
||||
// pixad does, in an fx app that provides no fetcher, and requests an image
|
||||
// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL
|
||||
// check accepts that address; only the dialer that refuses internal
|
||||
// addresses checks blocked_networks, so the answer is 403 only if the
|
||||
// fetcher the handlers build from the config connects with that dialer. Any
|
||||
// other dialer would try to connect until the upstream fetch timeout, which
|
||||
// is short so that the test then fails quickly.
|
||||
func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const host = "192.0.2.10"
|
||||
|
||||
stateDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
SigningKey: testSigningKey,
|
||||
StateDir: stateDir,
|
||||
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||
AllowlistHosts: []string{host},
|
||||
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
|
||||
UpstreamFetchTimeout: 2 * time.Second,
|
||||
// With no connection slots, the fetch would fail before dialing.
|
||||
UpstreamConnections: config.DefaultUpstreamConnections,
|
||||
}
|
||||
|
||||
var h *Handlers
|
||||
|
||||
app := fxtest.New(t,
|
||||
fx.Supply(cfg),
|
||||
fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New),
|
||||
fx.Populate(&h),
|
||||
)
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
|
||||
rec := sendGet(t, r, photoURL(host))
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "forbidden")
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// Errors for an Accept header that an auto URL cannot be served for.
|
||||
var (
|
||||
errInvalidAccept = errors.New("invalid Accept header")
|
||||
errNotAcceptable = errors.New("not acceptable: auto serves " +
|
||||
"image/jxl, image/avif, image/webp or image/jpeg")
|
||||
)
|
||||
|
||||
// chooseAutoFormat replaces the format auto in req with the format
|
||||
// formatForAccept chooses from r's Accept header, and adds Vary: Accept to the
|
||||
// response, which then depends on that header. It answers 400 for an Accept
|
||||
// header that is not valid and 406 for one that allows none of the formats,
|
||||
// and reports whether req can be served. Any other format is left as it is.
|
||||
func (s *Handlers) chooseAutoFormat(
|
||||
w http.ResponseWriter, r *http.Request, req *imgcache.ImageRequest,
|
||||
) bool {
|
||||
if req.Format != imgcache.FormatAuto {
|
||||
return true
|
||||
}
|
||||
|
||||
w.Header().Add("Vary", "Accept")
|
||||
|
||||
format, err := formatForAccept(strings.Join(r.Header.Values("Accept"), ","))
|
||||
if errors.Is(err, errNotAcceptable) {
|
||||
s.respondError(w, err.Error(), http.StatusNotAcceptable)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
s.respondError(w, err.Error(), http.StatusBadRequest)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
req.Format = format
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// formatForAccept returns the format an auto URL is served in for the Accept
|
||||
// header accept: JPEG XL when it names image/jxl, else AVIF when it names
|
||||
// image/avif, else WebP when it names image/webp, else JPEG when its most
|
||||
// specific entry of image/jpeg, image/* and */* allows it, or when it names
|
||||
// nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as
|
||||
// clients that cannot show them send image/* and */* too.
|
||||
func formatForAccept(accept string) (imgcache.ImageFormat, error) {
|
||||
qualities, err := parseAccept(accept)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(qualities) == 0 {
|
||||
return imgcache.FormatJPEG, nil
|
||||
}
|
||||
|
||||
if qualities["image/jxl"] > 0 {
|
||||
return imgcache.FormatJXL, nil
|
||||
}
|
||||
|
||||
if qualities["image/avif"] > 0 {
|
||||
return imgcache.FormatAVIF, nil
|
||||
}
|
||||
|
||||
if qualities["image/webp"] > 0 {
|
||||
return imgcache.FormatWebP, nil
|
||||
}
|
||||
|
||||
// For JPEG, the most specific entry the header has decides
|
||||
quality, named := qualities["image/jpeg"]
|
||||
if !named {
|
||||
quality, named = qualities["image/*"]
|
||||
}
|
||||
|
||||
if !named {
|
||||
quality = qualities["*/*"]
|
||||
}
|
||||
|
||||
if quality > 0 {
|
||||
return imgcache.FormatJPEG, nil
|
||||
}
|
||||
|
||||
return "", errNotAcceptable
|
||||
}
|
||||
|
||||
// parseAccept returns the q of each media range the Accept header accept
|
||||
// names, 1 where it gives none. A media range named more than once keeps its
|
||||
// lowest q, so a refusal is never overridden. A media range that does not
|
||||
// parse, or a q that is not a number from 0 to 1, is an error.
|
||||
func parseAccept(accept string) (map[string]float64, error) {
|
||||
qualities := make(map[string]float64)
|
||||
|
||||
for entry := range strings.SplitSeq(accept, ",") {
|
||||
// A header field list may hold empty entries
|
||||
if strings.TrimSpace(entry) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
mediaRange, params, err := mime.ParseMediaType(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %q: %w", errInvalidAccept, entry, err)
|
||||
}
|
||||
|
||||
quality := 1.0
|
||||
|
||||
if qParam, given := params["q"]; given {
|
||||
quality, err = strconv.ParseFloat(qParam, 64)
|
||||
inRange := quality >= 0 && quality <= 1
|
||||
|
||||
if err != nil || !inRange {
|
||||
return nil, fmt.Errorf("%w: %q: q is not a number from 0 to 1",
|
||||
errInvalidAccept, entry)
|
||||
}
|
||||
}
|
||||
|
||||
previous, named := qualities[mediaRange]
|
||||
if !named || quality < previous {
|
||||
qualities[mediaRange] = quality
|
||||
}
|
||||
}
|
||||
|
||||
return qualities, nil
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// The content types the tests below expect.
|
||||
const (
|
||||
avifType = "image/avif"
|
||||
webpType = "image/webp"
|
||||
jpegType = "image/jpeg"
|
||||
jsonType = "application/json"
|
||||
)
|
||||
|
||||
// TestFormatForAccept verifies the format chosen for the format auto from each
|
||||
// Accept header below, and the error for one that allows none of AVIF, WebP
|
||||
// and JPEG or is not valid.
|
||||
func TestFormatForAccept(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept string
|
||||
want imgcache.ImageFormat
|
||||
wantErr error
|
||||
}{
|
||||
{"AVIF-capable browser",
|
||||
"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
||||
imgcache.FormatAVIF, nil},
|
||||
{"WebP-capable browser",
|
||||
"image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5",
|
||||
imgcache.FormatWebP, nil},
|
||||
{"WebP only", webpType, imgcache.FormatWebP, nil},
|
||||
{"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imgcache.FormatJPEG, nil},
|
||||
{"wildcard only", "*/*", imgcache.FormatJPEG, nil},
|
||||
{"image wildcard only", "image/*", imgcache.FormatJPEG, nil},
|
||||
{"absent", "", imgcache.FormatJPEG, nil},
|
||||
{"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imgcache.FormatWebP, nil},
|
||||
{"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*",
|
||||
imgcache.FormatJPEG, nil},
|
||||
{"upper case and spaces", " Image/AVIF ; Q=0.5 ", imgcache.FormatAVIF, nil},
|
||||
{"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable},
|
||||
{"q=0 on everything", "*/*;q=0", "", errNotAcceptable},
|
||||
{"PNG only", "image/png", "", errNotAcceptable},
|
||||
{"malformed media range", "image/", "", errInvalidAccept},
|
||||
{"q not a number", "image/avif;q=high", "", errInvalidAccept},
|
||||
{"q above 1", "image/avif;q=2", "", errInvalidAccept},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := formatForAccept(tt.accept)
|
||||
t.Logf("Accept %q: %q, %v", tt.accept, got, err)
|
||||
|
||||
if got != tt.want || !errors.Is(err, tt.wantErr) {
|
||||
t.Errorf("formatForAccept(%q) = %q, %v, want %q, %v",
|
||||
tt.accept, got, err, tt.want, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL,
|
||||
// both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in
|
||||
// the format auto, made with h's image service and generator.
|
||||
func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Size: imgcache.Size{Width: 50, Height: 50},
|
||||
Format: imgcache.FormatAuto,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
token, err := h.encGen.Generate(&encurl.Payload{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
Format: imgcache.FormatAuto,
|
||||
ExpiresAt: time.Now().Add(time.Minute).Unix(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
return signedURL, "/v1/e/" + token + "/img.jpg"
|
||||
}
|
||||
|
||||
// requestImage sends method for target to srv with an Accept header line for
|
||||
// each of accept, and returns the response.
|
||||
func requestImage(
|
||||
t *testing.T, srv http.Handler, method, target string, accept ...string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), method, target, nil)
|
||||
|
||||
for _, value := range accept {
|
||||
req.Header.Add("Accept", value)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
t.Logf("%s %s with Accept %q: %d, Content-Type %s, Vary %v, X-Pixa-Cache %s",
|
||||
method, target, accept, rec.Code, rec.Header().Get("Content-Type"),
|
||||
rec.Header().Values("Vary"), rec.Header().Get("X-Pixa-Cache"))
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// TestFormatAuto_ChosenFromAccept requests an auto URL on each image route
|
||||
// with each Accept below, and checks the answer and that it carries
|
||||
// Vary: Accept. The signed URL is signed for auto, so it is valid whatever
|
||||
// Accept chooses.
|
||||
func TestFormatAuto_ChosenFromAccept(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept []string
|
||||
wantStatus int
|
||||
wantType string
|
||||
}{
|
||||
{"AVIF accepted", []string{"image/avif,image/webp,*/*;q=0.8"},
|
||||
http.StatusOK, avifType},
|
||||
{"WebP accepted", []string{"image/webp,*/*;q=0.8"}, http.StatusOK, webpType},
|
||||
{"no Accept", nil, http.StatusOK, jpegType},
|
||||
{"two Accept lines", []string{"image/png", webpType}, http.StatusOK, webpType},
|
||||
{"none of the three", []string{"image/gif"},
|
||||
http.StatusNotAcceptable, jsonType},
|
||||
{"not valid", []string{"image/avif;q=high"}, http.StatusBadRequest, jsonType},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, target := range []string{signedURL, encryptedURL} {
|
||||
rec := requestImage(t, srv, http.MethodGet, target, tt.accept...)
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
|
||||
if rec.Code != tt.wantStatus || gotType != tt.wantType {
|
||||
t.Errorf("%s: %d %s, want %d %s; body %s", target,
|
||||
rec.Code, gotType, tt.wantStatus, tt.wantType, rec.Body)
|
||||
}
|
||||
|
||||
if !slices.Contains(rec.Header().Values("Vary"), "Accept") {
|
||||
t.Errorf("%s: Vary = %v, want Accept in it",
|
||||
target, rec.Header().Values("Vary"))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_SignatureCoversAuto verifies that a /v1/image/ URL with the
|
||||
// format auto is checked against a signature for auto, not for the format
|
||||
// chosen: a URL signed for avif, with auto put in its path, is refused for a
|
||||
// client whose Accept chooses AVIF.
|
||||
func TestFormatAuto_SignatureCoversAuto(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Size: imgcache.Size{Width: 50, Height: 50},
|
||||
Format: imgcache.FormatAVIF,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1)
|
||||
if target == signedForAVIF {
|
||||
t.Fatalf("no /50x50.avif? in %s", signedForAVIF)
|
||||
}
|
||||
|
||||
rec := requestImage(t, srv, http.MethodGet, target, avifType)
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("status = %d, want %d", rec.Code, http.StatusUnauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_CachesEachFormatApart requests an auto URL for AVIF, then
|
||||
// JPEG, then both again. Each format is processed once and then served from
|
||||
// the cache, with an ETag of its own, so a client never gets the other format
|
||||
// from the cache.
|
||||
func TestFormatAuto_CachesEachFormatApart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, _ := autoPhotoURLs(t, h)
|
||||
|
||||
steps := []struct {
|
||||
wantType string
|
||||
wantCache string
|
||||
}{
|
||||
{avifType, "MISS"},
|
||||
{jpegType, "MISS"},
|
||||
{avifType, "HIT"},
|
||||
{jpegType, "HIT"},
|
||||
}
|
||||
|
||||
etags := make(map[string]string)
|
||||
|
||||
for _, step := range steps {
|
||||
rec := requestImage(t, srv, http.MethodGet, signedURL, step.wantType)
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
gotCache := rec.Header().Get("X-Pixa-Cache")
|
||||
|
||||
if rec.Code != http.StatusOK || gotType != step.wantType ||
|
||||
gotCache != step.wantCache {
|
||||
t.Fatalf("Accept %s: %d %s %s, want 200 %s %s", step.wantType,
|
||||
rec.Code, gotType, gotCache, step.wantType, step.wantCache)
|
||||
}
|
||||
|
||||
etag := rec.Header().Get("ETag")
|
||||
if previous, seen := etags[gotType]; seen && previous != etag {
|
||||
t.Errorf("%s ETag changed from %s to %s", gotType, previous, etag)
|
||||
}
|
||||
|
||||
etags[gotType] = etag
|
||||
}
|
||||
|
||||
if etags[avifType] == etags[jpegType] {
|
||||
t.Errorf("AVIF and JPEG have the same ETag %s", etags[avifType])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_Vary verifies that on each image route a HEAD answer and a
|
||||
// 304 for an auto URL carry Vary: Accept, and that the answer for a URL with
|
||||
// a fixed format does not.
|
||||
func TestFormatAuto_Vary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
srv := chi.NewRouter()
|
||||
srv.Get("/v1/image/*", h.HandleImage())
|
||||
srv.Head("/v1/image/*", h.HandleImage())
|
||||
srv.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
srv.Head("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
for _, urls := range [][2]string{
|
||||
{signedURL, signedPhotoURL(t, h)},
|
||||
{encryptedURL, encPhotoURL(t, h)},
|
||||
} {
|
||||
autoURL, fixedURL := urls[0], urls[1]
|
||||
|
||||
head := requestImage(t, srv, http.MethodHead, autoURL, webpType)
|
||||
checkVaryAccept(t, head, http.StatusOK, true)
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
autoURL, nil)
|
||||
req.Header.Set("Accept", webpType)
|
||||
req.Header.Set("If-None-Match", head.Header().Get("ETag"))
|
||||
|
||||
notModified := httptest.NewRecorder()
|
||||
srv.ServeHTTP(notModified, req)
|
||||
checkVaryAccept(t, notModified, http.StatusNotModified, true)
|
||||
|
||||
fixed := requestImage(t, srv, http.MethodGet, fixedURL)
|
||||
checkVaryAccept(t, fixed, http.StatusOK, false)
|
||||
}
|
||||
}
|
||||
|
||||
// checkVaryAccept fails the test unless rec answered wantStatus and, as
|
||||
// wantVary says, has or has not Accept in its Vary header.
|
||||
func checkVaryAccept(
|
||||
t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantVary bool,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
vary := rec.Header().Values("Vary")
|
||||
t.Logf("status %d, Vary %v", rec.Code, vary)
|
||||
|
||||
if rec.Code != wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, wantStatus)
|
||||
}
|
||||
|
||||
if slices.Contains(vary, "Accept") != wantVary {
|
||||
t.Errorf("Vary = %v, want Accept in it: %v", vary, wantVary)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package handlers
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
@@ -28,6 +29,11 @@ type Params struct {
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Database *database.Database
|
||||
Config *config.Config
|
||||
|
||||
// Fetcher, when provided, fetches upstream images in place of the
|
||||
// fetcher the handlers build from the config. Only tests provide one;
|
||||
// pixad does not.
|
||||
Fetcher httpfetcher.Fetcher `optional:"true"`
|
||||
}
|
||||
|
||||
// Handlers provides HTTP request handlers.
|
||||
@@ -36,6 +42,7 @@ type Handlers struct {
|
||||
hc *healthcheck.Healthcheck
|
||||
db *database.Database
|
||||
config *config.Config
|
||||
fetcher httpfetcher.Fetcher
|
||||
imgSvc *imgcache.Service
|
||||
imgCache *imgcache.Cache
|
||||
sessMgr *session.Manager
|
||||
@@ -59,21 +66,27 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
fetcher: params.Fetcher,
|
||||
csrfProtect: csrfProtect,
|
||||
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
//nolint:contextcheck // the eviction loop outlives OnStart; OnStop cancels it
|
||||
//nolint:contextcheck // the cache's goroutines outlive OnStart; OnStop stops them
|
||||
OnStart: func(_ context.Context) error {
|
||||
return s.initImageService()
|
||||
},
|
||||
// The pending counts are written here, before the database's stop
|
||||
// hook closes it.
|
||||
OnStop: func(ctx context.Context) error {
|
||||
if s.imgCache == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.imgCache.StopEviction(ctx)
|
||||
return errors.Join(
|
||||
s.imgCache.StopEviction(ctx),
|
||||
s.imgCache.StopPendingCountWrites(ctx),
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
@@ -115,6 +128,9 @@ func (s *Handlers) initImageService() error {
|
||||
// write-pressure passes. No-op when the disk cache is disabled.
|
||||
cache.StartEviction(imgcache.DefaultEvictionInterval)
|
||||
|
||||
// Writes the counts requests could not write by their deadline
|
||||
cache.StartPendingCountWrites()
|
||||
|
||||
// Create the fetcher config
|
||||
fetcherCfg := httpfetcher.DefaultConfig()
|
||||
fetcherCfg.AllowHTTP = s.config.AllowHTTP
|
||||
@@ -128,10 +144,12 @@ func (s *Handlers) initImageService() error {
|
||||
fetcherCfg.MaxConnections = s.config.UpstreamConnections
|
||||
fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
|
||||
|
||||
// Create the service
|
||||
// Create the service. With no fetcher provided, it builds its own from
|
||||
// fetcherCfg.
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
FetcherConfig: fetcherCfg,
|
||||
Fetcher: s.fetcher,
|
||||
SigningKey: s.config.SigningKey,
|
||||
Allowlist: s.config.AllowlistHosts,
|
||||
MaxConcurrentProcessing: s.config.MaxConcurrentProcessing,
|
||||
|
||||
@@ -18,7 +18,8 @@ import (
|
||||
)
|
||||
|
||||
// HandleImage handles the main image proxy route:
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>, or with no format
|
||||
// /v1/image/<host>/<path>/<width>x<height>
|
||||
func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
@@ -43,6 +44,11 @@ func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// The signature covers the format auto, not the format chosen
|
||||
if !s.chooseAutoFormat(w, r, req) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get cache key for logging
|
||||
cacheKey := imgcache.CacheKey(req)
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
start := time.Now()
|
||||
|
||||
req, ok := s.parseImageEncRequest(w, r)
|
||||
if !ok {
|
||||
if !ok || !s.chooseAutoFormat(w, r, req) {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/signature"
|
||||
)
|
||||
|
||||
// requireJPEGXL requires that rec answers 200 with a JPEG XL image.
|
||||
func requireJPEGXL(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body %q",
|
||||
rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
if got := rec.Header().Get("Content-Type"); got != jxlType {
|
||||
t.Errorf("Content-Type = %q, want %s", got, jxlType)
|
||||
}
|
||||
|
||||
if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
|
||||
t.Errorf("body is %s, want jxl", vips.ImageTypes[got])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageWithoutFormat_ServesJPEGXL verifies that a /v1/image/ URL whose
|
||||
// last segment is a size with no format, 50x50 or orig, answers JPEG XL.
|
||||
func TestImageWithoutFormat_ServesJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := newImageRoute(t, newPhotoFetcher(t, allowlistedHost))
|
||||
|
||||
for _, size := range []string{"50x50", "orig"} {
|
||||
target := "/v1/image/" + allowlistedHost + photoPath + "/" + size
|
||||
requireJPEGXL(t, sendGet(t, route, target))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageWithoutFormat_SignedAsJXL verifies that a /v1/image/ URL with no
|
||||
// format is signed as jxl: the signature made for the URL ending in .jxl is
|
||||
// accepted for the same URL without .jxl.
|
||||
func TestImageWithoutFormat_SignedAsJXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := newImageRoute(t, newPhotoFetcher(t, signedHost))
|
||||
expires := time.Now().Add(time.Hour)
|
||||
|
||||
sig := signature.New(testSigningKey).Sign(&signature.Request{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
Format: string(imgcache.FormatJXL),
|
||||
Quality: encurl.DefaultQuality,
|
||||
FitMode: string(imgcache.FitCover),
|
||||
Expires: expires,
|
||||
})
|
||||
query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix())
|
||||
|
||||
for _, size := range []string{"50x50.jxl", "50x50"} {
|
||||
target := "/v1/image/" + signedHost + photoPath + "/" + size + query
|
||||
requireJPEGXL(t, sendGet(t, route, target))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageEncWithoutFormat_ServesJPEGXL verifies that an encrypted URL whose
|
||||
// token holds no format answers JPEG XL.
|
||||
func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
token, err := h.encGen.Generate(&encurl.Payload{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
requireJPEGXL(t, getEncToken(srv, token))
|
||||
}
|
||||
|
||||
// TestGeneratorPage_SelectsJPEGXL verifies that the generator page's format
|
||||
// choice is JPEG XL until another is chosen.
|
||||
func TestGeneratorPage_SelectsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newCSRFTestRouter(t)
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(newSessionCookie(t, h))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if !strings.Contains(rec.Body.String(), `<option value="jxl" selected>`) {
|
||||
t.Errorf("generator page does not select JPEG XL: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_NoFormat_MakesJPEGXLURL verifies that the generator form
|
||||
// sent with an empty format field, or with none, makes a URL whose name ends
|
||||
// in .jxl and which answers JPEG XL.
|
||||
func TestGeneratePost_NoFormat_MakesJPEGXLURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photo := url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
}
|
||||
|
||||
emptyFormat := maps.Clone(photo)
|
||||
emptyFormat.Set(formatField, "")
|
||||
|
||||
for name, form := range map[string]url.Values{
|
||||
"empty format field": emptyFormat,
|
||||
"no format field": photo,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, form)
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %d %s",
|
||||
rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Logf("generated URL path: %s", match[1])
|
||||
|
||||
if !strings.HasSuffix(match[1], "/img.jxl") {
|
||||
t.Errorf("generated URL %s does not end in /img.jxl", match[1])
|
||||
}
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireJPEGXL(t, imageRec)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// jxlType is the content type of JPEG XL.
|
||||
const jxlType = "image/jxl"
|
||||
|
||||
// TestFormatForAccept_JPEGXL verifies that the format auto chooses JPEG XL
|
||||
// when Accept names image/jxl, ahead of AVIF whatever their q, and AVIF when
|
||||
// Accept refuses JPEG XL with q=0.
|
||||
func TestFormatForAccept_JPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept string
|
||||
want imgcache.ImageFormat
|
||||
}{
|
||||
{"JPEG XL-capable browser",
|
||||
"image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imgcache.FormatJXL},
|
||||
{"JPEG XL only", jxlType, imgcache.FormatJXL},
|
||||
{"JPEG XL with a lower q than AVIF", "image/avif,image/jxl;q=0.5",
|
||||
imgcache.FormatJXL},
|
||||
{"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imgcache.FormatAVIF},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := formatForAccept(tt.accept)
|
||||
if got != tt.want || err != nil {
|
||||
t.Errorf("formatForAccept(%q) = %q, %v, want %q",
|
||||
tt.accept, got, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_JPEGXL requests an auto URL on each image route with an
|
||||
// Accept header that names image/jxl, and checks that the answer is a JPEG XL
|
||||
// image.
|
||||
func TestFormatAuto_JPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
for _, target := range []string{signedURL, encryptedURL} {
|
||||
rec := requestImage(t, srv, http.MethodGet, target,
|
||||
"image/jxl,image/avif,image/webp,*/*;q=0.8")
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
|
||||
if rec.Code != http.StatusOK || gotType != jxlType {
|
||||
t.Errorf("%s: %d %s, want 200 %s; body %s",
|
||||
target, rec.Code, gotType, jxlType, rec.Body)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
|
||||
t.Errorf("%s: body is %s, want jxl", target, vips.ImageTypes[got])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package httpfetcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built
|
||||
// without DialContext, as pixa builds it, refuses to connect to a local
|
||||
// server.
|
||||
func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
transport := transportOf(t, New(DefaultConfig()))
|
||||
|
||||
addr := srv.Listener.Addr().String()
|
||||
|
||||
_, err := transport.DialContext(testContext(t), "tcp", addr)
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Fatalf("DialContext(%s) error = %v, want ErrSSRFBlocked", addr, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDialContextReplacesOnlyTheDialer checks that a fetcher built with
|
||||
// DialContext connects through it, while the URL check still refuses a
|
||||
// loopback URL and the redirect check a redirect to a link-local address.
|
||||
func TestDialContextReplacesOnlyTheDialer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
dialer := &recordingDialer{target: srv.Listener.Addr().String()}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.AllowHTTP = true
|
||||
cfg.DialContext = dialer.dialContext
|
||||
f := New(cfg)
|
||||
|
||||
if body := fetchBody(t, f, "/image"); body != imagePayload {
|
||||
t.Errorf("body = %q, want %q", body, imagePayload)
|
||||
}
|
||||
|
||||
_, err := f.Fetch(testContext(t), "http://127.0.0.1/image")
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err)
|
||||
}
|
||||
|
||||
_, err = f.Fetch(testContext(t), upstreamURL("/redirect/private"))
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err)
|
||||
}
|
||||
|
||||
// The upstream server is reached through DialContext, and nothing else
|
||||
// is asked of it.
|
||||
dialed := dialer.dialedAddrs()
|
||||
if len(dialed) == 0 {
|
||||
t.Error("DialContext was never called")
|
||||
}
|
||||
|
||||
for _, addr := range dialed {
|
||||
if addr != net.JoinHostPort(testPublicHost, "80") {
|
||||
t.Errorf("DialContext was asked to connect to %s", addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ const (
|
||||
contentTypeGIF = "image/gif"
|
||||
contentTypeWebP = "image/webp"
|
||||
contentTypeAVIF = "image/avif"
|
||||
contentTypeJXL = "image/jxl"
|
||||
contentTypeSVG = "image/svg+xml"
|
||||
contentTypeOctetStream = "application/octet-stream"
|
||||
)
|
||||
@@ -137,6 +138,11 @@ type Config struct {
|
||||
// BlockedNetworks are operator-supplied CIDR ranges refused by the
|
||||
// dialer, in addition to the always-enforced built-in ranges.
|
||||
BlockedNetworks []netip.Prefix
|
||||
// DialContext, when set, makes the fetcher's connections in place of
|
||||
// the dialer that refuses internal addresses; the URL and redirect
|
||||
// checks still run. Only tests set it, to reach a local server; the
|
||||
// config file and the environment cannot.
|
||||
DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// DefaultConfig returns a Config with sensible defaults.
|
||||
@@ -151,6 +157,7 @@ func DefaultConfig() *Config {
|
||||
contentTypeGIF,
|
||||
contentTypeWebP,
|
||||
contentTypeAVIF,
|
||||
contentTypeJXL,
|
||||
contentTypeSVG,
|
||||
},
|
||||
AllowHTTP: false,
|
||||
@@ -190,13 +197,19 @@ func New(config *Config) *HTTPFetcher {
|
||||
config = DefaultConfig()
|
||||
}
|
||||
|
||||
// Create transport with SSRF-safe dialer. The dialer re-resolves and
|
||||
// re-checks at connect time (closing the DNS-rebinding window) against
|
||||
// both the built-in ranges and the operator-supplied blocklist.
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
// Unless config.DialContext replaces it, the transport connects with
|
||||
// the SSRF-safe dialer, which re-resolves and re-checks at connect time
|
||||
// (closing the DNS-rebinding window) against both the built-in ranges
|
||||
// and the operator-supplied blocklist.
|
||||
dialContext := config.DialContext
|
||||
if dialContext == nil {
|
||||
dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialSSRFSafe(ctx, network, addr, config.BlockedNetworks)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
DialContext: dialContext,
|
||||
TLSHandshakeTimeout: DefaultTLSTimeout,
|
||||
MaxIdleConns: DefaultMaxIdleConns,
|
||||
IdleConnTimeout: DefaultIdleConnTimeout,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package httpfetcher
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestJPEGXLContentType verifies that the fetcher accepts an upstream answer
|
||||
// of type image/jxl by default, and that the mock fetcher serves a .jxl file
|
||||
// as image/jxl.
|
||||
func TestJPEGXLContentType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const jxlType = "image/jxl"
|
||||
|
||||
if !New(DefaultConfig()).isAllowedContentType(jxlType) {
|
||||
t.Errorf("isAllowedContentType(%q) = false, want true", jxlType)
|
||||
}
|
||||
|
||||
if got := detectContentTypeFromPath("images/photo.jxl"); got != jxlType {
|
||||
t.Errorf("detectContentTypeFromPath(photo.jxl) = %q, want %q", got, jxlType)
|
||||
}
|
||||
}
|
||||
@@ -109,6 +109,8 @@ func detectContentTypeFromPath(path string) string {
|
||||
return contentTypeWebP
|
||||
case strings.HasSuffix(path, ".avif"):
|
||||
return contentTypeAVIF
|
||||
case strings.HasSuffix(path, ".jxl"):
|
||||
return contentTypeJXL
|
||||
case strings.HasSuffix(path, ".svg"):
|
||||
return contentTypeSVG
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestImageProcessor_EmptyFormatRefused verifies that a request with no format
|
||||
// is refused, as both image routes give every request a format before it is
|
||||
// processed.
|
||||
func TestImageProcessor_EmptyFormatRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)), &Request{},
|
||||
)
|
||||
if !errors.Is(err, ErrUnsupportedOutputFormat) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// processedSize runs input through Process and returns the output's size in
|
||||
// bytes.
|
||||
func processedSize(t *testing.T, input []byte, req *Request) int64 {
|
||||
t.Helper()
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
_ = result.Content.Close()
|
||||
|
||||
return result.ContentLength
|
||||
}
|
||||
|
||||
// encodePNG encodes img as PNG with Go's encoder.
|
||||
func encodePNG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := png.Encode(&buf, img)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encode test PNG: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// decode decodes input with vips, as Process does.
|
||||
func decode(t *testing.T, input []byte) *vips.ImageRef {
|
||||
t.Helper()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(input)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode input: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(img.Close)
|
||||
|
||||
return img
|
||||
}
|
||||
|
||||
// TestImageProcessor_PNGIsCompressed verifies that a PNG output is
|
||||
// compressed: a flat 200x150 image, 90,000 bytes of raw pixels, comes out at
|
||||
// a small fraction of that.
|
||||
func TestImageProcessor_PNGIsCompressed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const width, height = 200, 150
|
||||
|
||||
flat := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
flat.Set(x, y, color.RGBA{R: 40, G: 120, B: 200, A: 255})
|
||||
}
|
||||
}
|
||||
|
||||
size := processedSize(t, encodePNG(t, flat), &Request{Format: FormatPNG})
|
||||
|
||||
const rawBytes = width * height * 3
|
||||
if size > rawBytes/10 {
|
||||
t.Errorf("PNG output is %d bytes, want under a tenth of its %d raw",
|
||||
size, rawBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_WebPAtDefaultEffort verifies that WebP is saved at
|
||||
// libvips' default effort, 4: the output is the size of the same image saved
|
||||
// at that effort.
|
||||
func TestImageProcessor_WebPAtDefaultEffort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
|
||||
size := processedSize(t, input, &Request{Format: FormatWebP, Quality: 85})
|
||||
|
||||
want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: 85,
|
||||
ReductionEffort: 4,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportWebp() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("WebP output is %d bytes, want %d, the size at effort 4",
|
||||
size, len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_AVIFAtEffort1 verifies that AVIF is saved at effort 1
|
||||
// with 8 bits per sample: the output is the size of the same image saved
|
||||
// with those settings. The source is 16-bit, which libvips saves with 12
|
||||
// bits when it is not given a bit depth, and 640x480: on a small image,
|
||||
// such as 64x48, efforts 1 and 2 give the same output.
|
||||
func TestImageProcessor_AVIFAtEffort1(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const width, height = 640, 480
|
||||
|
||||
source := image.NewRGBA64(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
source.Set(x, y, color.RGBA64{
|
||||
R: uint16((x * 65535 / width) & 0xffff),
|
||||
G: uint16((y * 65535 / height) & 0xffff),
|
||||
B: 32768,
|
||||
A: 65535,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
input := encodePNG(t, source)
|
||||
|
||||
size := processedSize(t, input, &Request{Format: FormatAVIF, Quality: 85})
|
||||
|
||||
want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: 85,
|
||||
Effort: 1,
|
||||
Bitdepth: 8,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAvif() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("AVIF output is %d bytes, want %d, the size at effort 1 "+
|
||||
"and 8 bits", size, len(want))
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,24 @@ func initVips() {
|
||||
})
|
||||
}
|
||||
|
||||
// errNoJPEGXL is returned by CheckJPEGXLSupport.
|
||||
var errNoJPEGXL = errors.New("libvips lacks JPEG XL support: install " +
|
||||
"vips-jxl on Alpine, or use a libvips built with libjxl")
|
||||
|
||||
// CheckJPEGXLSupport returns an error, naming the fix, when libvips
|
||||
// cannot load and save JPEG XL.
|
||||
func CheckJPEGXLSupport() error {
|
||||
initVips()
|
||||
|
||||
// govips counts a format as supported when libvips has its loader;
|
||||
// libvips builds the JPEG XL loader and saver together.
|
||||
if !vips.IsTypeSupported(vips.ImageTypeJXL) {
|
||||
return errNoJPEGXL
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Format represents supported output image formats.
|
||||
type Format string
|
||||
|
||||
@@ -47,6 +65,7 @@ const (
|
||||
FormatPNG Format = "png"
|
||||
FormatWebP Format = "webp"
|
||||
FormatAVIF Format = "avif"
|
||||
FormatJXL Format = "jxl"
|
||||
FormatGIF Format = "gif"
|
||||
)
|
||||
|
||||
@@ -237,9 +256,9 @@ func (p *ImageProcessor) Process(
|
||||
}
|
||||
}
|
||||
|
||||
// Determine output format
|
||||
// orig is the source's own format; encode refuses an empty format
|
||||
outputFormat := req.Format
|
||||
if outputFormat == FormatOriginal || outputFormat == "" {
|
||||
if outputFormat == FormatOriginal {
|
||||
outputFormat = p.formatFromString(inputFormat)
|
||||
}
|
||||
|
||||
@@ -287,6 +306,7 @@ const (
|
||||
mimeGIF = "image/gif"
|
||||
mimeWebP = "image/webp"
|
||||
mimeAVIF = "image/avif"
|
||||
mimeJXL = "image/jxl"
|
||||
)
|
||||
|
||||
// SupportedInputFormats returns MIME types this processor can read.
|
||||
@@ -297,6 +317,7 @@ func (p *ImageProcessor) SupportedInputFormats() []string {
|
||||
mimeGIF,
|
||||
mimeWebP,
|
||||
mimeAVIF,
|
||||
mimeJXL,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,6 +329,7 @@ func (p *ImageProcessor) SupportedOutputFormats() []Format {
|
||||
FormatGIF,
|
||||
FormatWebP,
|
||||
FormatAVIF,
|
||||
FormatJXL,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,6 +346,8 @@ func FormatToMIME(format Format) string {
|
||||
return mimeGIF
|
||||
case FormatAVIF:
|
||||
return mimeAVIF
|
||||
case FormatJXL:
|
||||
return mimeJXL
|
||||
case FormatOriginal:
|
||||
return "application/octet-stream"
|
||||
default:
|
||||
@@ -393,9 +417,11 @@ func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string {
|
||||
return "webp"
|
||||
case vips.ImageTypeAVIF, vips.ImageTypeHEIF:
|
||||
return string(FormatAVIF)
|
||||
case vips.ImageTypeJXL:
|
||||
return string(FormatJXL)
|
||||
case vips.ImageTypeUnknown, vips.ImageTypeMagick, vips.ImageTypePDF,
|
||||
vips.ImageTypeSVG, vips.ImageTypeTIFF, vips.ImageTypeBMP,
|
||||
vips.ImageTypeJP2K, vips.ImageTypeJXL:
|
||||
vips.ImageTypeJP2K:
|
||||
return "unknown"
|
||||
default:
|
||||
return "unknown"
|
||||
@@ -467,44 +493,6 @@ func (p *ImageProcessor) encode(
|
||||
quality = defaultQuality
|
||||
}
|
||||
|
||||
var params vips.ExportParams
|
||||
|
||||
switch format {
|
||||
case FormatJPEG:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeJPEG,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatPNG:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypePNG,
|
||||
}
|
||||
|
||||
case FormatGIF:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeGIF,
|
||||
}
|
||||
|
||||
case FormatWebP:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeWEBP,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatAVIF:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeAVIF,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatOriginal:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
}
|
||||
|
||||
// Stripping drops the ICC profile as well, and clients show an image
|
||||
// with no profile as sRGB, so convert to sRGB first. "srgb" names
|
||||
// libvips' built-in profile; govips' own sRGB path variable is set on
|
||||
@@ -516,11 +504,166 @@ func (p *ImageProcessor) encode(
|
||||
}
|
||||
}
|
||||
|
||||
// Drop EXIF, XMP, IPTC and the ICC profile. govips ignores this for
|
||||
// GIF, which carries none of them.
|
||||
params.StripMetadata = true
|
||||
switch format {
|
||||
case FormatJPEG:
|
||||
return exportJPEG(img, quality)
|
||||
|
||||
output, _, err := img.Export(¶ms)
|
||||
case FormatPNG:
|
||||
return exportPNG(img)
|
||||
|
||||
case FormatGIF:
|
||||
return exportGIF(img)
|
||||
|
||||
case FormatWebP:
|
||||
return exportWebP(img, quality)
|
||||
|
||||
case FormatAVIF:
|
||||
return exportAVIF(img, quality)
|
||||
|
||||
case FormatJXL:
|
||||
return exportJXL(img, quality)
|
||||
|
||||
case FormatOriginal:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
}
|
||||
}
|
||||
|
||||
// govips sends libvips Go's zero value for some settings an export leaves
|
||||
// out, such as no compression at all for PNG, so each export below sets
|
||||
// every setting whose zero value is not what pixa wants. Stripping metadata
|
||||
// drops EXIF, XMP, IPTC and the ICC profile.
|
||||
|
||||
// exportJPEG encodes img as JPEG at quality, without metadata. The settings
|
||||
// it leaves out are at libvips' defaults.
|
||||
func exportJPEG(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportJpeg(&vips.JpegExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// pngCompression is libvips' default PNG compression, from 0 (none) to 9.
|
||||
const pngCompression = 6
|
||||
|
||||
// exportPNG encodes img as PNG at libvips' default compression and row
|
||||
// filter, without metadata.
|
||||
func exportPNG(img *vips.ImageRef) ([]byte, error) {
|
||||
output, _, err := img.ExportPng(&vips.PngExportParams{
|
||||
StripMetadata: true,
|
||||
Compression: pngCompression,
|
||||
Filter: vips.PngFilterNone,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// gifEffort is libvips' default GIF effort, from 1 to 10.
|
||||
const gifEffort = 7
|
||||
|
||||
// exportGIF encodes img as GIF at libvips' default effort. govips cannot
|
||||
// have libvips strip metadata from GIF, which carries none.
|
||||
func exportGIF(img *vips.ImageRef) ([]byte, error) {
|
||||
output, _, err := img.ExportGIF(&vips.GifExportParams{Effort: gifEffort})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// webpEffort is libvips' default WebP effort, from 0 (fastest) to 6.
|
||||
const webpEffort = 4
|
||||
|
||||
// exportWebP encodes img as lossy WebP at quality and libvips' default
|
||||
// effort, without metadata.
|
||||
func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportWebp(&vips.WebpExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
ReductionEffort: webpEffort,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest
|
||||
// govips can set. With one thread, as pixad runs libvips, 1 takes about 51
|
||||
// seconds to save an 8192x8192 image of random pixels, the worst case,
|
||||
// against the default downstream_timeout of 60 seconds. On an image of
|
||||
// milder noise, which 1 saves in about 12 seconds, 2 takes nearly a minute
|
||||
// and libvips' default, 4, takes minutes.
|
||||
const avifEffort = 1
|
||||
|
||||
// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image.
|
||||
// libvips would save a 16-bit image with 12, but at avifEffort that takes
|
||||
// about 54 seconds for a 16-bit 8192x8192 image of milder noise, nearly all
|
||||
// of the default downstream_timeout, and about 12 seconds with 8.
|
||||
const avifBitdepth = 8
|
||||
|
||||
// exportAVIF encodes img as lossy AVIF at quality, avifEffort and
|
||||
// avifBitdepth, without metadata.
|
||||
func exportAVIF(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportAvif(&vips.AvifExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
Effort: avifEffort,
|
||||
Bitdepth: avifBitdepth,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// jxlResolution is the resolution every JPEG XL image is saved with, in
|
||||
// pixels per millimetre as libvips counts it: 72 dpi, what libvips gives a
|
||||
// JPEG that names none.
|
||||
const jxlResolution = 72 / 25.4
|
||||
|
||||
// exportJXL encodes img as JPEG XL at quality, with libvips' default effort
|
||||
// and without metadata. govips sends libvips a distance, the JPEG XL
|
||||
// encoder's own measure of quality, along with the quality, and libvips then
|
||||
// uses the distance alone, so the quality is also given as a distance.
|
||||
func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
// libvips converts a CMYK image to sRGB before it saves WebP, AVIF or
|
||||
// PNG, but cannot save one as JPEG XL. encode has already converted
|
||||
// any image with an ICC profile to sRGB, so this is CMYK with none.
|
||||
if img.Interpretation() == vips.InterpretationCMYK {
|
||||
err := img.ToColorSpace(vips.InterpretationSRGB)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert CMYK to sRGB: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// govips cannot make libvips strip metadata from JPEG XL, so it is
|
||||
// removed from the image itself. RemoveMetadata removes EXIF, XMP and
|
||||
// IPTC but keeps the ICC profile.
|
||||
err := img.RemoveMetadata()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = img.RemoveICCProfile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// libvips 8.16 and later still write an EXIF block of their own, from
|
||||
// the image's size, orientation and resolution, and fixed values. The
|
||||
// image is upright, so its orientation is 1, but its resolution is still
|
||||
// the source's.
|
||||
toSave, err := img.CopyChangingResolution(jxlResolution, jxlResolution)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer toSave.Close()
|
||||
|
||||
params := vips.NewJxlExportParams()
|
||||
params.Quality = quality
|
||||
params.Distance = jxlDistance(quality)
|
||||
|
||||
output, _, err := toSave.ExportJxl(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -528,6 +671,22 @@ func (p *ImageProcessor) encode(
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// jxlDistance turns a quality from 1 to 100 into the JPEG XL encoder's
|
||||
// distance, with the formula libvips and libjxl use for their own quality
|
||||
// setting, except that 100 stays lossy (distance 0.1) where libjxl makes it
|
||||
// lossless.
|
||||
//
|
||||
//nolint:mnd // the constants of that formula
|
||||
func jxlDistance(quality int) float64 {
|
||||
q := float64(quality)
|
||||
|
||||
if quality >= 30 {
|
||||
return 0.1 + (100-q)*0.09
|
||||
}
|
||||
|
||||
return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0
|
||||
}
|
||||
|
||||
// formatFromString converts a format string to Format.
|
||||
func (p *ImageProcessor) formatFromString(format string) Format {
|
||||
switch format {
|
||||
@@ -541,6 +700,8 @@ func (p *ImageProcessor) formatFromString(format string) Format {
|
||||
return FormatWebP
|
||||
case string(FormatAVIF):
|
||||
return FormatAVIF
|
||||
case string(FormatJXL):
|
||||
return FormatJXL
|
||||
default:
|
||||
return FormatJPEG
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestCheckJPEGXLSupport fails when libvips lacks JPEG XL support, as on
|
||||
// Alpine without the vips-jxl package.
|
||||
func TestCheckJPEGXLSupport(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := CheckJPEGXLSupport()
|
||||
if err != nil {
|
||||
t.Fatalf("CheckJPEGXLSupport() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and
|
||||
// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine
|
||||
// without the vips-jxl package.
|
||||
func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load test JPEG: %v", err)
|
||||
}
|
||||
|
||||
defer img.Close()
|
||||
|
||||
jxl, _, err := img.ExportJxl(vips.NewJxlExportParams())
|
||||
if err != nil {
|
||||
t.Fatalf("ExportJxl() error = %v", err)
|
||||
}
|
||||
|
||||
loaded, err := vips.NewImageFromBuffer(jxl)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the JPEG XL image: %v", err)
|
||||
}
|
||||
|
||||
defer loaded.Close()
|
||||
|
||||
if loaded.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()])
|
||||
}
|
||||
|
||||
if loaded.Width() != 64 || loaded.Height() != 48 {
|
||||
t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestImageProcessor_EncodeJPEGXL converts a JPEG to a smaller JPEG XL and
|
||||
// checks the content type, and the format and size the output loads as.
|
||||
func TestImageProcessor_EncodeJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := &Request{Size: Size{Width: 100, Height: 75}, Format: FormatJXL}
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), req,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
if result.ContentType != "image/jxl" {
|
||||
t.Errorf("ContentType = %q, want image/jxl", result.ContentType)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(result.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read result: %v", err)
|
||||
}
|
||||
|
||||
output, err := vips.NewImageFromBuffer(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the output: %v", err)
|
||||
}
|
||||
|
||||
defer output.Close()
|
||||
|
||||
if output.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("output format = %s, want jxl", vips.ImageTypes[output.Format()])
|
||||
}
|
||||
|
||||
if output.Width() != 100 || output.Height() != 75 {
|
||||
t.Errorf("output size = %dx%d, want 100x75", output.Width(), output.Height())
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLQuality verifies that the quality reaches the JPEG
|
||||
// XL encoder: the same image comes out smaller at quality 30 than at 90.
|
||||
func TestImageProcessor_JPEGXLQuality(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 400, 300)
|
||||
outputBytes := make(map[int]int64)
|
||||
|
||||
for _, quality := range []int{30, 90} {
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input), &Request{Format: FormatJXL, Quality: quality})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() at quality %d error = %v", quality, err)
|
||||
}
|
||||
|
||||
_ = result.Content.Close()
|
||||
outputBytes[quality] = result.ContentLength
|
||||
}
|
||||
|
||||
if outputBytes[30] >= outputBytes[90] {
|
||||
t.Errorf("%d bytes at quality 30, %d at 90, want fewer at 30",
|
||||
outputBytes[30], outputBytes[90])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLDropsSourceEXIF verifies that none of the source's
|
||||
// EXIF reaches a JPEG XL output. libvips 8.16 and later write an EXIF block of
|
||||
// their own into JPEG XL (orientation, resolution, size, colour space and
|
||||
// fixed defaults), and govips cannot ask them to leave it out, so the test
|
||||
// looks for the source's fields rather than for no EXIF at all.
|
||||
func TestImageProcessor_JPEGXLDropsSourceEXIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input, err := os.ReadFile("testdata/gps-exif.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
exif := processAndDecode(t, input, &Request{Format: FormatJXL}).GetExif()
|
||||
|
||||
for _, field := range []string{
|
||||
"exif-ifd0-Make", "exif-ifd0-Model", "exif-ifd2-BodySerialNumber",
|
||||
"exif-ifd2-DateTimeOriginal", "exif-ifd3-GPSLatitude",
|
||||
"exif-ifd3-GPSLongitude",
|
||||
} {
|
||||
if value, found := exif[field]; found {
|
||||
t.Errorf("output has %s: %s", field, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLDropsSourceResolution verifies that the source's
|
||||
// resolution does not reach the EXIF block libvips 8.16 and later write into
|
||||
// JPEG XL. A JPEG XL image holds its resolution in that block alone, so the
|
||||
// resolution the output loads with is the block's.
|
||||
func TestImageProcessor_JPEGXLDropsSourceResolution(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// dpi-300.jpg is a flat 8x8 grey image with a resolution of 300 dpi.
|
||||
input, err := os.ReadFile("testdata/dpi-300.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
// libvips gives the resolution in pixels per millimetre.
|
||||
xDPI := math.Round(output.ResX() * 25.4)
|
||||
yDPI := math.Round(output.ResY() * 25.4)
|
||||
|
||||
if xDPI == 300 || yDPI == 300 {
|
||||
t.Errorf("output resolution = %vx%v dpi, the source's", xDPI, yDPI)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLAppliesEXIFOrientation verifies that a JPEG XL
|
||||
// output is turned upright, as TestImageProcessor_AppliesEXIFOrientation does
|
||||
// for PNG.
|
||||
func TestImageProcessor_JPEGXLAppliesEXIFOrientation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// orientation-6.jpg is stored 16x8, red on the left and blue on the
|
||||
// right, with EXIF orientation 6 (turn 90 degrees clockwise to view).
|
||||
// Upright it is 8x16, red on top and blue below.
|
||||
input, err := os.ReadFile("testdata/orientation-6.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
if output.Width() != 8 || output.Height() != 16 {
|
||||
t.Fatalf("output is %dx%d, want 8x16", output.Width(), output.Height())
|
||||
}
|
||||
|
||||
top, err := output.GetPoint(4, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
bottom, err := output.GetPoint(4, 15)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
if top[0] <= top[2] || bottom[2] <= bottom[0] {
|
||||
t.Errorf("top pixel = %v, bottom pixel = %v, want red above blue",
|
||||
top, bottom)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLConvertsWideGamutToSRGB verifies that a JPEG XL
|
||||
// output is converted to sRGB, as TestImageProcessor_ConvertsWideGamutToSRGB
|
||||
// does for PNG. It does not check for an ICC profile, as libvips reports one
|
||||
// for every JPEG XL image it loads.
|
||||
func TestImageProcessor_JPEGXLConvertsWideGamutToSRGB(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// display-p3.jpg is a flat 8x8 image with the Display P3 profile
|
||||
// embedded, filled with Display P3 (234, 51, 35), which is sRGB red.
|
||||
input, err := os.ReadFile("testdata/display-p3.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
pixel, err := output.GetPoint(4, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
want := []float64{255, 0, 0}
|
||||
for i := range want {
|
||||
if math.Abs(pixel[i]-want[i]) > 5 {
|
||||
t.Fatalf("pixel = %v, want within 5 of %v", pixel, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLFromCMYK verifies that a CMYK JPEG with no ICC
|
||||
// profile can be served as JPEG XL, in sRGB.
|
||||
func TestImageProcessor_JPEGXLFromCMYK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// cmyk.jpg is a flat 8x8 CMYK image with no ICC profile, filled with
|
||||
// full cyan and no magenta, yellow or black.
|
||||
input, err := os.ReadFile("testdata/cmyk.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input, &Request{Format: FormatJXL})
|
||||
|
||||
if output.Bands() != 3 {
|
||||
t.Fatalf("output has %d bands, want 3", output.Bands())
|
||||
}
|
||||
|
||||
pixel, err := output.GetPoint(4, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
// Cyan in sRGB: little red, much green and blue.
|
||||
if pixel[0] > 50 || pixel[1] < 100 || pixel[2] < 200 {
|
||||
t.Errorf("pixel = %v, want cyan", pixel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJXLDistance verifies the JPEG XL distance for a few qualities: 90 is
|
||||
// distance 1, the encoder's own default, and 100 stays lossy.
|
||||
func TestJXLDistance(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
quality int
|
||||
want float64
|
||||
}{
|
||||
{quality: 100, want: 0.1},
|
||||
{quality: 90, want: 1},
|
||||
{quality: 30, want: 6.4},
|
||||
{quality: 20, want: 9.0667},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := jxlDistance(tt.quality)
|
||||
if math.Abs(got-tt.want) > 0.0001 {
|
||||
t.Errorf("jxlDistance(%d) = %v, want %v", tt.quality, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 352 B |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 799 B |
+88
-11
@@ -11,6 +11,8 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
@@ -79,6 +81,26 @@ type Cache struct {
|
||||
evictionDone chan struct{}
|
||||
evictionCancel context.CancelFunc
|
||||
|
||||
// The pending counts: hits, misses, upstream fetches and transforms
|
||||
// whose write to the database missed its request's deadline, kept
|
||||
// here until they are written by the goroutine that
|
||||
// StartPendingCountWrites starts. As for eviction, the channels are
|
||||
// created in NewCache: pendingCountsAdded wakes that goroutine, and
|
||||
// pendingCountsDone is closed when it returns. pendingCountsCancel,
|
||||
// set by StartPendingCountWrites, cancels its context, which tells it
|
||||
// to finish.
|
||||
pendingHits atomic.Int64
|
||||
pendingMisses atomic.Int64
|
||||
pendingUpstreamFetches atomic.Int64
|
||||
pendingUpstreamFetchBytes atomic.Int64
|
||||
pendingTransforms atomic.Int64
|
||||
pendingCountsAdded chan struct{}
|
||||
pendingCountsDone chan struct{}
|
||||
pendingCountsCancel context.CancelFunc
|
||||
|
||||
// Held through each write of the pending counts, and by Stats to read them
|
||||
pendingCountsWriteMutex sync.Mutex
|
||||
|
||||
// metaCache holds the content types of the variants most recently
|
||||
// stored or served, so a hit does not read the variant's .meta file.
|
||||
// It never stands in for the variant file, which is always opened.
|
||||
@@ -96,6 +118,17 @@ type Cache struct {
|
||||
// deterministically pause inside that window to exercise
|
||||
// concurrent stores against it; production code leaves it nil.
|
||||
evictSourceBlobTestHook func(ContentHash)
|
||||
|
||||
// reconciliationPageSize is the most rows one read of a content table
|
||||
// returns in the reconciliation pass and in Stats. newCache sets it to
|
||||
// defaultReconciliationPageSize; tests set it smaller.
|
||||
reconciliationPageSize int
|
||||
|
||||
// reconciliationReadTestHook, when set, is called after each of those
|
||||
// reads with the number of rows the read covered, so tests can check
|
||||
// that no read covers more than one page; production code leaves it
|
||||
// nil.
|
||||
reconciliationReadTestHook func(rows int)
|
||||
}
|
||||
|
||||
// NewCache creates a new cache instance.
|
||||
@@ -127,6 +160,11 @@ func newCache(
|
||||
evictionDone: make(chan struct{}),
|
||||
metaCache: metaCache,
|
||||
contentLocks: newContentLock(),
|
||||
|
||||
pendingCountsAdded: make(chan struct{}, 1),
|
||||
pendingCountsDone: make(chan struct{}),
|
||||
|
||||
reconciliationPageSize: defaultReconciliationPageSize,
|
||||
}
|
||||
|
||||
if c.disabled {
|
||||
@@ -461,18 +499,28 @@ func (c *Cache) CleanExpired(ctx context.Context) error {
|
||||
func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
var stats CacheStats
|
||||
|
||||
// So that no write of the pending counts falls between the two reads
|
||||
c.pendingCountsWriteMutex.Lock()
|
||||
|
||||
// Fetch hit/miss counts from the stats table
|
||||
err := c.db.QueryRowContext(ctx, `
|
||||
SELECT hit_count, miss_count
|
||||
FROM cache_stats WHERE id = 1
|
||||
`).Scan(&stats.HitCount, &stats.MissCount)
|
||||
|
||||
// Hits and misses not yet written count too (see StartPendingCountWrites)
|
||||
stats.HitCount += c.pendingHits.Load()
|
||||
stats.MissCount += c.pendingMisses.Load()
|
||||
|
||||
c.pendingCountsWriteMutex.Unlock()
|
||||
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, fmt.Errorf("failed to get cache stats: %w", err)
|
||||
}
|
||||
|
||||
// Count and size the cached source images and processed variants. A
|
||||
// disabled cache holds none, whatever rows an earlier run left.
|
||||
// Count and size the cached source images and processed variants from
|
||||
// their tables. A disabled cache holds none, whatever rows an earlier
|
||||
// run left.
|
||||
if !c.disabled {
|
||||
err = c.db.QueryRowContext(ctx, `
|
||||
SELECT (SELECT COUNT(*) FROM source_content)
|
||||
@@ -482,7 +530,7 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
c.log.Warn("failed to count cache items for stats", "error", err)
|
||||
}
|
||||
|
||||
stats.TotalSizeBytes, err = c.UsageBytes(ctx)
|
||||
stats.TotalSizeBytes, err = c.sumContentSizeBytes(ctx)
|
||||
if err != nil {
|
||||
c.log.Warn("failed to sum cache size for stats", "error", err)
|
||||
}
|
||||
@@ -497,19 +545,30 @@ func (c *Cache) Stats(ctx context.Context) (*CacheStats, error) {
|
||||
}
|
||||
|
||||
// IncrementStats counts a cache hit or miss, and an upstream fetch that read
|
||||
// fetchBytes bytes, as IncrementUpstreamFetch does.
|
||||
// fetchBytes bytes, as IncrementUpstreamFetch does. Like the other Increment
|
||||
// methods, it writes to the database with ctx's deadline but not its
|
||||
// cancellation, so an ended request is still counted but does not wait for
|
||||
// the database past its deadline; a count not written by then becomes a
|
||||
// pending count (see StartPendingCountWrites).
|
||||
func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64) {
|
||||
countCtx, cancel := withoutCancelKeepingDeadline(ctx)
|
||||
defer cancel()
|
||||
|
||||
var err error
|
||||
|
||||
pendingCount := &c.pendingMisses
|
||||
|
||||
if hit {
|
||||
_, err = c.db.ExecContext(ctx, `
|
||||
pendingCount = &c.pendingHits
|
||||
|
||||
_, err = c.db.ExecContext(countCtx, `
|
||||
UPDATE cache_stats
|
||||
SET hit_count = hit_count + 1,
|
||||
last_updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = 1
|
||||
`)
|
||||
} else {
|
||||
_, err = c.db.ExecContext(ctx, `
|
||||
_, err = c.db.ExecContext(countCtx, `
|
||||
UPDATE cache_stats
|
||||
SET miss_count = miss_count + 1,
|
||||
last_updated_at = CURRENT_TIMESTAMP
|
||||
@@ -517,7 +576,10 @@ func (c *Cache) IncrementStats(ctx context.Context, hit bool, fetchBytes int64)
|
||||
`)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
c.addPendingCount(pendingCount, 1)
|
||||
case err != nil:
|
||||
c.log.Warn("failed to count cache hit or miss", "hit", hit, "error", err)
|
||||
}
|
||||
|
||||
@@ -531,14 +593,22 @@ func (c *Cache) IncrementUpstreamFetch(ctx context.Context, fetchBytes int64) {
|
||||
return
|
||||
}
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
countCtx, cancel := withoutCancelKeepingDeadline(ctx)
|
||||
defer cancel()
|
||||
|
||||
_, err := c.db.ExecContext(countCtx, `
|
||||
UPDATE cache_stats
|
||||
SET upstream_fetch_count = upstream_fetch_count + 1,
|
||||
upstream_fetch_bytes = upstream_fetch_bytes + ?,
|
||||
last_updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = 1
|
||||
`, fetchBytes)
|
||||
if err != nil {
|
||||
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
c.addPendingCount(&c.pendingUpstreamFetches, 1)
|
||||
c.addPendingCount(&c.pendingUpstreamFetchBytes, fetchBytes)
|
||||
case err != nil:
|
||||
c.log.Warn("failed to count upstream fetch",
|
||||
"fetch_bytes", fetchBytes, "error", err)
|
||||
}
|
||||
@@ -546,13 +616,20 @@ func (c *Cache) IncrementUpstreamFetch(ctx context.Context, fetchBytes int64) {
|
||||
|
||||
// IncrementTransformCount counts one image transcoded by the image processor.
|
||||
func (c *Cache) IncrementTransformCount(ctx context.Context) {
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
countCtx, cancel := withoutCancelKeepingDeadline(ctx)
|
||||
defer cancel()
|
||||
|
||||
_, err := c.db.ExecContext(countCtx, `
|
||||
UPDATE cache_stats
|
||||
SET transform_count = transform_count + 1,
|
||||
last_updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = 1
|
||||
`)
|
||||
if err != nil {
|
||||
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
c.addPendingCount(&c.pendingTransforms, 1)
|
||||
case err != nil:
|
||||
c.log.Warn("failed to count transform", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEvictionReadsTheUsageTotal adds, stores again, resizes and evicts
|
||||
// cache content, then drops both content tables. UsageBytes must still
|
||||
// report what the tables held, and an eviction pass under the limit must
|
||||
// still succeed: neither may sum the tables.
|
||||
func TestEvictionReadsTheUsageTotal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
kept := storeEvictionTestSource(t, cache, "total.example.com", "/kept.jpg",
|
||||
bytes.Repeat([]byte{0x71}, 1000))
|
||||
evicted := storeEvictionTestSource(t, cache, "total.example.com", "/evicted.jpg",
|
||||
bytes.Repeat([]byte{0x72}, 700))
|
||||
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x73}, 500))
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x74}, 300))
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyTwo,
|
||||
bytes.Repeat([]byte{0x75}, 200))
|
||||
|
||||
// pixa never changes a source's size, but a statement that does must
|
||||
// change the total too.
|
||||
_, err := cache.db.ExecContext(ctx,
|
||||
`UPDATE source_content SET size_bytes = 900 WHERE content_hash = ?`,
|
||||
string(kept))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to change the source's size: %v", err)
|
||||
}
|
||||
|
||||
err = cache.evictSourceBlob(ctx, evicted)
|
||||
if err != nil {
|
||||
t.Fatalf("evictSourceBlob failed: %v", err)
|
||||
}
|
||||
|
||||
err = cache.evictVariant(ctx, testVariantKeyTwo)
|
||||
if err != nil {
|
||||
t.Fatalf("evictVariant failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = cache.db.ExecContext(ctx,
|
||||
`DROP TABLE source_content; DROP TABLE variant_content`)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to drop the content tables: %v", err)
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
t.Logf("UsageBytes() = %d, error = %v", usage, err)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes() error = %v, want nil: it read a content table", err)
|
||||
}
|
||||
|
||||
if usage != 1200 {
|
||||
t.Errorf("UsageBytes() = %d, want 1200 (900 + 300)", usage)
|
||||
}
|
||||
|
||||
err = cache.EvictToLimit(ctx)
|
||||
if err != nil {
|
||||
t.Errorf("EvictToLimit() error = %v, want nil: it read a content table", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconciliationCorrectsTheUsageTotal sets the total cache usage to a
|
||||
// wrong value and checks that a reconciliation pass, which sums both
|
||||
// content tables, puts it right.
|
||||
func TestReconciliationCorrectsTheUsageTotal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
storeEvictionTestSource(t, cache, "total.example.com", "/a.jpg",
|
||||
bytes.Repeat([]byte{0x76}, 1000))
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x77}, 500))
|
||||
|
||||
_, err := cache.db.ExecContext(ctx,
|
||||
`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set a wrong total: %v", err)
|
||||
}
|
||||
|
||||
err = cache.reconcileAccounting(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileAccounting failed: %v", err)
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes failed: %v", err)
|
||||
}
|
||||
|
||||
if usage != 1500 {
|
||||
t.Errorf("UsageBytes() after reconciliation = %d, want 1500 (1000 + 500)",
|
||||
usage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUsageTotalNotCorrectedFromAnOlderSum stores a variant after the
|
||||
// change count was read, then offers a sum taken before the store as the
|
||||
// correction: the total must keep the stored variant, as that sum may
|
||||
// have missed it.
|
||||
func TestUsageTotalNotCorrectedFromAnOlderSum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
var changeCount int64
|
||||
|
||||
err := cache.db.QueryRowContext(ctx,
|
||||
`SELECT change_count FROM cache_usage WHERE id = 1`,
|
||||
).Scan(&changeCount)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the change count: %v", err)
|
||||
}
|
||||
|
||||
storeEvictionTestVariant(t, cache, testVariantKeyOne,
|
||||
bytes.Repeat([]byte{0x78}, 500))
|
||||
|
||||
corrected, err := cache.correctUsageTotal(ctx, 0, changeCount)
|
||||
if err != nil {
|
||||
t.Fatalf("correctUsageTotal failed: %v", err)
|
||||
}
|
||||
|
||||
if corrected {
|
||||
t.Error("correctUsageTotal() = true, want false: content changed since the sum")
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes failed: %v", err)
|
||||
}
|
||||
|
||||
if usage != 500 {
|
||||
t.Errorf("UsageBytes() = %d, want 500", usage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReconciliationReadsAPageAtATime stores five source images and five
|
||||
// variants, sets the page size to two rows and runs a reconciliation
|
||||
// pass. No read the pass makes of a content table, to check its rows or
|
||||
// to sum them, may cover more than two rows, so a request's query waits
|
||||
// for one page at most. Between them the reads must still cover every
|
||||
// row of both tables twice, once to check it and once to sum it, and the
|
||||
// sum must put a wrong total right.
|
||||
func TestReconciliationReadsAPageAtATime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
ctx := t.Context()
|
||||
|
||||
const pageSize = 2
|
||||
|
||||
cache.reconciliationPageSize = pageSize
|
||||
|
||||
// Sources of 100 bytes and variants of 10, five of each.
|
||||
for i := range 5 {
|
||||
content := []byte(strconv.Itoa(i))
|
||||
|
||||
storeEvictionTestSource(t, cache, "pages.example.com",
|
||||
"/"+strconv.Itoa(i)+".jpg", bytes.Repeat(content, 100))
|
||||
storeEvictionTestVariant(t, cache, VariantKey("aabbccdd000"+strconv.Itoa(i)),
|
||||
bytes.Repeat(content, 10))
|
||||
}
|
||||
|
||||
_, err := cache.db.ExecContext(ctx,
|
||||
`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set a wrong total: %v", err)
|
||||
}
|
||||
|
||||
var rowsPerRead []int
|
||||
|
||||
cache.reconciliationReadTestHook = func(rows int) {
|
||||
rowsPerRead = append(rowsPerRead, rows)
|
||||
}
|
||||
|
||||
err = cache.reconcileAccounting(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileAccounting failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("rows covered by each read, in order: %v", rowsPerRead)
|
||||
|
||||
coveredRows := 0
|
||||
|
||||
for _, rows := range rowsPerRead {
|
||||
if rows > pageSize {
|
||||
t.Errorf("a read covered %d rows, want at most %d", rows, pageSize)
|
||||
}
|
||||
|
||||
coveredRows += rows
|
||||
}
|
||||
|
||||
// Ten rows, each read once to check it and once to sum it.
|
||||
if coveredRows != 20 {
|
||||
t.Errorf("the reads covered %d rows in all, want 20", coveredRows)
|
||||
}
|
||||
|
||||
usage, err := cache.UsageBytes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("UsageBytes failed: %v", err)
|
||||
}
|
||||
|
||||
if usage != 550 {
|
||||
t.Errorf("UsageBytes() after reconciliation = %d, want 550 (5*100 + 5*10)",
|
||||
usage)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceEvictionCandidatesComeFromTheIndex checks that the query
|
||||
// choosing source images to evict reads source_content in last access
|
||||
// order from its index, instead of reading and sorting the whole table.
|
||||
func TestSourceEvictionCandidatesComeFromTheIndex(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
rows, err := cache.db.QueryContext(t.Context(),
|
||||
`EXPLAIN QUERY PLAN `+sourceCandidatesQuery, evictionBatchSize)
|
||||
if err != nil {
|
||||
t.Fatalf("EXPLAIN QUERY PLAN failed: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var plan []string
|
||||
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
|
||||
var detail string
|
||||
|
||||
err := rows.Scan(&id, &parent, &unused, &detail)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to scan the query plan: %v", err)
|
||||
}
|
||||
|
||||
plan = append(plan, detail)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
t.Fatalf("query plan iteration failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("query plan: %q", plan)
|
||||
|
||||
if !strings.Contains(strings.Join(plan, "\n"), "idx_source_content_last_accessed") {
|
||||
t.Errorf("the source candidate query does not use "+
|
||||
"idx_source_content_last_accessed; plan: %q", plan)
|
||||
}
|
||||
}
|
||||
@@ -72,7 +72,7 @@ func (c *Cache) computeDefaultMaxBytes(
|
||||
}
|
||||
|
||||
// Both terms are at most math.MaxInt64, so the sum cannot overflow.
|
||||
//nolint:gosec // G115: UsageBytes sums file sizes, never negative
|
||||
//nolint:gosec // G115: UsageBytes returns the total cache usage, never negative
|
||||
spaceBytes := min(freeBytes, math.MaxInt64) + uint64(usedBytes)
|
||||
|
||||
computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
|
||||
|
||||
+246
-72
@@ -21,6 +21,12 @@ const DefaultEvictionInterval = 5 * time.Minute
|
||||
// and source blobs) one eviction pass fetches from the database.
|
||||
const evictionBatchSize = 100
|
||||
|
||||
// defaultReconciliationPageSize is the most rows one read of the
|
||||
// reconciliation pass returns, unless a test sets
|
||||
// Cache.reconciliationPageSize smaller. Each read is a query of its own,
|
||||
// so a request waits for one page at most, however large the cache is.
|
||||
const defaultReconciliationPageSize = 1000
|
||||
|
||||
// staleTempFileAge is how old an orphaned temp file (left behind by a
|
||||
// crashed write) must be before reconciliation removes it. Fresh temp
|
||||
// files may still belong to an in-flight store.
|
||||
@@ -45,7 +51,9 @@ const fallbackContentType = "application/octet-stream"
|
||||
|
||||
// UsageBytes returns the total number of bytes of cache content
|
||||
// tracked in the database (source content blobs plus processed
|
||||
// variants). It never scans the cache directories.
|
||||
// variants). It reads the total the database keeps up to date as rows
|
||||
// are added and removed, so it neither scans the cache directories nor
|
||||
// sums the tables.
|
||||
func (c *Cache) UsageBytes(ctx context.Context) (int64, error) {
|
||||
if c.disabled {
|
||||
return 0, nil
|
||||
@@ -53,12 +61,11 @@ func (c *Cache) UsageBytes(ctx context.Context) (int64, error) {
|
||||
|
||||
var total int64
|
||||
|
||||
err := c.db.QueryRowContext(ctx, `
|
||||
SELECT (SELECT COALESCE(SUM(size_bytes), 0) FROM source_content)
|
||||
+ (SELECT COALESCE(SUM(size_bytes), 0) FROM variant_content)
|
||||
`).Scan(&total)
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT total_size_bytes FROM cache_usage WHERE id = 1`,
|
||||
).Scan(&total)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to compute cache usage: %w", err)
|
||||
return 0, fmt.Errorf("failed to read cache usage: %w", err)
|
||||
}
|
||||
|
||||
return total, nil
|
||||
@@ -236,16 +243,19 @@ func (c *Cache) variantCandidates(ctx context.Context) ([]evictionCandidate, err
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
// sourceCandidates returns the least recently used source blobs. Rows
|
||||
// written before the LRU column existed fall back to fetched_at.
|
||||
// sourceCandidatesQuery selects the least recently used source blobs. It
|
||||
// orders by the last_accessed_at column itself, not by an expression, so
|
||||
// SQLite reads the rows in order from that column's index instead of
|
||||
// sorting the whole table.
|
||||
const sourceCandidatesQuery = `
|
||||
SELECT content_hash, size_bytes, last_accessed_at
|
||||
FROM source_content
|
||||
ORDER BY last_accessed_at ASC, content_hash ASC
|
||||
LIMIT ?`
|
||||
|
||||
// sourceCandidates returns the least recently used source blobs.
|
||||
func (c *Cache) sourceCandidates(ctx context.Context) ([]evictionCandidate, error) {
|
||||
rows, err := c.db.QueryContext(ctx, `
|
||||
SELECT content_hash, size_bytes,
|
||||
COALESCE(last_accessed_at, fetched_at, '1970-01-01 00:00:00') AS lru
|
||||
FROM source_content
|
||||
ORDER BY lru ASC, content_hash ASC
|
||||
LIMIT ?
|
||||
`, evictionBatchSize)
|
||||
rows, err := c.db.QueryContext(ctx, sourceCandidatesQuery, evictionBatchSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query source eviction candidates: %w", err)
|
||||
}
|
||||
@@ -532,11 +542,13 @@ func (c *Cache) runReconciliationPass(ctx context.Context) {
|
||||
// (or whose accounting insert failed, e.g. StoreVariant's best-effort
|
||||
// insert under transient DB contention), drops accounting rows whose
|
||||
// files are missing, removes source blob files the database does not
|
||||
// know (and rows whose files are gone), and sweeps stale temp files
|
||||
// left behind by crashed writes. Running it periodically, not just
|
||||
// once, bounds how long such drift can accumulate unaccounted for on a
|
||||
// long-running process to one eviction interval. Once ctx is cancelled,
|
||||
// it stops at the next file or row and returns ctx's error.
|
||||
// know (and rows whose files are gone), sweeps stale temp files left
|
||||
// behind by crashed writes, and last checks the total cache usage
|
||||
// against the tables. It reads the tables a page at a time. Running it
|
||||
// periodically, not just once, bounds how long such drift can
|
||||
// accumulate unaccounted for on a long-running process to one eviction
|
||||
// interval. Once ctx is cancelled, it stops at the next file or row and
|
||||
// returns ctx's error.
|
||||
func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||
if c.disabled {
|
||||
return nil
|
||||
@@ -562,7 +574,7 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return c.reconcileUsageTotal(ctx)
|
||||
}
|
||||
|
||||
// reconcileVariantFiles walks the variant storage directory, adopting
|
||||
@@ -657,46 +669,63 @@ func (c *Cache) variantContentTypeFromSidecar(variantPath string) string {
|
||||
// reconcileVariantRows drops accounting rows whose variant files are
|
||||
// missing, so the database never references deleted content.
|
||||
func (c *Cache) reconcileVariantRows(ctx context.Context) error {
|
||||
keys, err := c.allVariantKeys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var after VariantKey
|
||||
|
||||
for _, key := range keys {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.variants.Exists(key) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, string(key))
|
||||
for {
|
||||
keys, err := c.variantKeysAfter(ctx, after)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to drop stale variant accounting row: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("dropped accounting row for missing variant file", "cache_key", key)
|
||||
if c.reconciliationReadTestHook != nil {
|
||||
c.reconciliationReadTestHook(len(keys))
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.variants.Exists(key) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, err := c.db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, string(key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to drop stale variant accounting row: %w", err)
|
||||
}
|
||||
|
||||
c.log.Info("dropped accounting row for missing variant file", "cache_key", key)
|
||||
}
|
||||
|
||||
after = keys[len(keys)-1]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// allVariantKeys returns every tracked variant cache key.
|
||||
func (c *Cache) allVariantKeys(ctx context.Context) ([]VariantKey, error) {
|
||||
return queryStringColumn[VariantKey](ctx, c.db,
|
||||
`SELECT cache_key FROM variant_content`, "variant keys", "variant key")
|
||||
// variantKeysAfter returns, in order, up to c.reconciliationPageSize
|
||||
// tracked variant cache keys that sort after the given one.
|
||||
func (c *Cache) variantKeysAfter(
|
||||
ctx context.Context, after VariantKey,
|
||||
) ([]VariantKey, error) {
|
||||
return queryStringColumn[VariantKey](ctx, c.db, `
|
||||
SELECT cache_key FROM variant_content
|
||||
WHERE cache_key > ? ORDER BY cache_key LIMIT ?
|
||||
`, "variant keys", "variant key", string(after), c.reconciliationPageSize)
|
||||
}
|
||||
|
||||
// queryStringColumn runs a single-column query and returns the column
|
||||
// values as T. plural names the set for the query and scan failure
|
||||
// messages; singular names one row for the scan and iteration failure
|
||||
// messages.
|
||||
// queryStringColumn runs a single-column query with args and returns the
|
||||
// column values as T. plural names the set for the query and scan
|
||||
// failure messages; singular names one row for the scan and iteration
|
||||
// failure messages.
|
||||
func queryStringColumn[T ~string](
|
||||
ctx context.Context, db *sql.DB, query, plural, singular string,
|
||||
ctx context.Context, db *sql.DB, query, plural, singular string, args ...any,
|
||||
) ([]T, error) {
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
rows, err := db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query %s: %w", plural, err)
|
||||
}
|
||||
@@ -791,38 +820,183 @@ func (c *Cache) removeUntrackedSourceFile(
|
||||
// reconcileSourceRows removes source_content rows (and their metadata
|
||||
// references and sidecars) whose blob files are missing on disk.
|
||||
func (c *Cache) reconcileSourceRows(ctx context.Context) error {
|
||||
hashes, err := c.allSourceContentHashes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var after ContentHash
|
||||
|
||||
for _, hash := range hashes {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.srcContent.Exists(hash) {
|
||||
continue
|
||||
}
|
||||
|
||||
// The blob file is already gone; evictSourceBlob removes the
|
||||
// rows and sidecars and tolerates the missing file.
|
||||
err := c.evictSourceBlob(ctx, hash)
|
||||
for {
|
||||
hashes, err := c.sourceContentHashesAfter(ctx, after)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("dropped rows for missing source content file", "content_hash", hash)
|
||||
if c.reconciliationReadTestHook != nil {
|
||||
c.reconciliationReadTestHook(len(hashes))
|
||||
}
|
||||
|
||||
if len(hashes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, hash := range hashes {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if c.srcContent.Exists(hash) {
|
||||
continue
|
||||
}
|
||||
|
||||
// The blob file is already gone; evictSourceBlob removes the
|
||||
// rows and sidecars and tolerates the missing file.
|
||||
err := c.evictSourceBlob(ctx, hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Info("dropped rows for missing source content file", "content_hash", hash)
|
||||
}
|
||||
|
||||
after = hashes[len(hashes)-1]
|
||||
}
|
||||
}
|
||||
|
||||
// sourceContentHashesAfter returns, in order, up to
|
||||
// c.reconciliationPageSize tracked source content hashes that sort after
|
||||
// the given one.
|
||||
func (c *Cache) sourceContentHashesAfter(
|
||||
ctx context.Context, after ContentHash,
|
||||
) ([]ContentHash, error) {
|
||||
return queryStringColumn[ContentHash](ctx, c.db, `
|
||||
SELECT content_hash FROM source_content
|
||||
WHERE content_hash > ? ORDER BY content_hash LIMIT ?
|
||||
`, "source content hashes", "content hash", string(after),
|
||||
c.reconciliationPageSize)
|
||||
}
|
||||
|
||||
// Each of these queries sums size_bytes over the next page of rows of
|
||||
// one content table, the rows that sort after a key, and returns the
|
||||
// page's last key, the sum and the number of rows in the page. Past the
|
||||
// last row the page has no rows and the key is NULL.
|
||||
const (
|
||||
sourceSizePageQuery = `
|
||||
SELECT MAX(content_hash), COALESCE(SUM(size_bytes), 0), COUNT(*)
|
||||
FROM (
|
||||
SELECT content_hash, size_bytes FROM source_content
|
||||
WHERE content_hash > ? ORDER BY content_hash LIMIT ?
|
||||
)`
|
||||
variantSizePageQuery = `
|
||||
SELECT MAX(cache_key), COALESCE(SUM(size_bytes), 0), COUNT(*)
|
||||
FROM (
|
||||
SELECT cache_key, size_bytes FROM variant_content
|
||||
WHERE cache_key > ? ORDER BY cache_key LIMIT ?
|
||||
)`
|
||||
)
|
||||
|
||||
// sumContentSizeBytes sums size_bytes over both content tables, a page
|
||||
// of rows per query.
|
||||
func (c *Cache) sumContentSizeBytes(ctx context.Context) (int64, error) {
|
||||
sourceBytes, err := c.sumSizeBytesInPages(ctx, sourceSizePageQuery)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
variantBytes, err := c.sumSizeBytesInPages(ctx, variantSizePageQuery)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return sourceBytes + variantBytes, nil
|
||||
}
|
||||
|
||||
// sumSizeBytesInPages runs pageQuery, one of the size page queries
|
||||
// above, from the first page to the last and adds up the page sums.
|
||||
func (c *Cache) sumSizeBytesInPages(
|
||||
ctx context.Context, pageQuery string,
|
||||
) (int64, error) {
|
||||
var total int64
|
||||
|
||||
after := ""
|
||||
|
||||
for {
|
||||
var lastKey sql.NullString
|
||||
|
||||
var pageBytes int64
|
||||
|
||||
var pageRows int
|
||||
|
||||
err := c.db.QueryRowContext(ctx, pageQuery, after, c.reconciliationPageSize).
|
||||
Scan(&lastKey, &pageBytes, &pageRows)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to sum cache content sizes: %w", err)
|
||||
}
|
||||
|
||||
if c.reconciliationReadTestHook != nil {
|
||||
c.reconciliationReadTestHook(pageRows)
|
||||
}
|
||||
|
||||
if pageRows == 0 {
|
||||
return total, nil
|
||||
}
|
||||
|
||||
total += pageBytes
|
||||
after = lastKey.String
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileUsageTotal checks the total cache usage the database keeps
|
||||
// against size_bytes summed over both content tables, and corrects the
|
||||
// total when they differ.
|
||||
func (c *Cache) reconcileUsageTotal(ctx context.Context) error {
|
||||
var totalBytes, changeCount int64
|
||||
|
||||
err := c.db.QueryRowContext(ctx,
|
||||
`SELECT total_size_bytes, change_count FROM cache_usage WHERE id = 1`,
|
||||
).Scan(&totalBytes, &changeCount)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read cache usage: %w", err)
|
||||
}
|
||||
|
||||
sumBytes, err := c.sumContentSizeBytes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if sumBytes == totalBytes {
|
||||
return nil
|
||||
}
|
||||
|
||||
corrected, err := c.correctUsageTotal(ctx, sumBytes, changeCount)
|
||||
if err != nil || !corrected {
|
||||
return err
|
||||
}
|
||||
|
||||
c.log.Warn("corrected total cache usage to the sum of the content tables",
|
||||
"previous_usage_bytes", totalBytes, "usage_bytes", sumBytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// allSourceContentHashes returns every tracked source content hash.
|
||||
func (c *Cache) allSourceContentHashes(ctx context.Context) ([]ContentHash, error) {
|
||||
return queryStringColumn[ContentHash](ctx, c.db,
|
||||
`SELECT content_hash FROM source_content`,
|
||||
"source content hashes", "content hash")
|
||||
// correctUsageTotal sets the total cache usage to sumBytes, a sum of the
|
||||
// content tables taken when the change count was changeCount, and
|
||||
// reports whether it did. If a row was added, removed or resized since,
|
||||
// the count has moved and the total is left alone: the sum may have
|
||||
// missed that change, and the next pass checks again.
|
||||
func (c *Cache) correctUsageTotal(
|
||||
ctx context.Context, sumBytes, changeCount int64,
|
||||
) (bool, error) {
|
||||
result, err := c.db.ExecContext(ctx, `
|
||||
UPDATE cache_usage SET total_size_bytes = ?
|
||||
WHERE id = 1 AND change_count = ?
|
||||
`, sumBytes, changeCount)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to correct cache usage: %w", err)
|
||||
}
|
||||
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to read cache usage correction: %w", err)
|
||||
}
|
||||
|
||||
return affected > 0, nil
|
||||
}
|
||||
|
||||
// sweepStaleTempFile removes a temp file left behind by a crashed
|
||||
|
||||
@@ -21,7 +21,13 @@ const (
|
||||
FormatPNG ImageFormat = "png"
|
||||
FormatWebP ImageFormat = "webp"
|
||||
FormatAVIF ImageFormat = "avif"
|
||||
FormatJXL ImageFormat = "jxl"
|
||||
FormatGIF ImageFormat = "gif"
|
||||
|
||||
// FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each
|
||||
// request from its Accept header once the URL's signature or token has
|
||||
// been checked; it is never processed or cached as itself.
|
||||
FormatAuto ImageFormat = "auto"
|
||||
)
|
||||
|
||||
// Size represents requested image dimensions
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// StartPendingCountWrites starts the goroutine that writes the pending
|
||||
// counts to the database whenever there are some, one UPDATE at a time:
|
||||
// however many requests pass their deadline before their counts are
|
||||
// written, at most this one write waits for the database. It is a no-op
|
||||
// when already started. The goroutine outlives the caller, so it runs with
|
||||
// its own context, which StopPendingCountWrites cancels to tell it to
|
||||
// finish.
|
||||
func (c *Cache) StartPendingCountWrites() {
|
||||
if c.pendingCountsCancel != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c.pendingCountsCancel = cancel
|
||||
|
||||
go func() {
|
||||
c.pendingCountWriteLoop(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// StopPendingCountWrites tells the goroutine StartPendingCountWrites
|
||||
// started, if it did, to finish, and waits for its write under way to end
|
||||
// and for it to return. It then writes the pending counts left, so that
|
||||
// they are written before the database closes. It waits for all of this at
|
||||
// most until ctx ends, then logs the counts not written and returns an
|
||||
// error.
|
||||
func (c *Cache) StopPendingCountWrites(ctx context.Context) error {
|
||||
var err error
|
||||
|
||||
if c.pendingCountsCancel != nil {
|
||||
c.pendingCountsCancel()
|
||||
|
||||
select {
|
||||
case <-c.pendingCountsDone:
|
||||
case <-ctx.Done():
|
||||
// The write under way is left to finish: the database's
|
||||
// close waits for a query under way.
|
||||
err = fmt.Errorf("pending counts still being written: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = c.writePendingCounts(ctx)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
c.log.Error("counts not written at shutdown",
|
||||
"hits", c.pendingHits.Load(),
|
||||
"misses", c.pendingMisses.Load(),
|
||||
"upstream_fetches", c.pendingUpstreamFetches.Load(),
|
||||
"upstream_fetch_bytes", c.pendingUpstreamFetchBytes.Load(),
|
||||
"transforms", c.pendingTransforms.Load(),
|
||||
"error", err,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// pendingCountWriteLoop is the body of the goroutine
|
||||
// StartPendingCountWrites starts. It returns when ctx is cancelled, but
|
||||
// does not cut off a write under way then. A write that fails leaves the
|
||||
// counts pending, for the next write.
|
||||
func (c *Cache) pendingCountWriteLoop(ctx context.Context) {
|
||||
defer close(c.pendingCountsDone)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-c.pendingCountsAdded:
|
||||
}
|
||||
|
||||
err := c.writePendingCounts(context.WithoutCancel(ctx))
|
||||
if err != nil {
|
||||
c.log.Warn("failed to write pending counts", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// addPendingCount adds n to pendingCount, one of the pending counts, and
|
||||
// wakes the goroutine that writes them. pendingCountsAdded has capacity one,
|
||||
// so a wakeup already waiting is enough.
|
||||
func (c *Cache) addPendingCount(pendingCount *atomic.Int64, n int64) {
|
||||
pendingCount.Add(n)
|
||||
|
||||
select {
|
||||
case c.pendingCountsAdded <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// writePendingCounts adds the pending counts to the cache_stats row in one
|
||||
// UPDATE, then takes what it wrote out of them, leaving any added
|
||||
// meanwhile. When the UPDATE fails, they all stay pending.
|
||||
func (c *Cache) writePendingCounts(ctx context.Context) error {
|
||||
c.pendingCountsWriteMutex.Lock()
|
||||
defer c.pendingCountsWriteMutex.Unlock()
|
||||
|
||||
hits := c.pendingHits.Load()
|
||||
misses := c.pendingMisses.Load()
|
||||
upstreamFetches := c.pendingUpstreamFetches.Load()
|
||||
upstreamFetchBytes := c.pendingUpstreamFetchBytes.Load()
|
||||
transforms := c.pendingTransforms.Load()
|
||||
|
||||
if hits+misses+upstreamFetches+upstreamFetchBytes+transforms == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := c.db.ExecContext(ctx, `
|
||||
UPDATE cache_stats
|
||||
SET hit_count = hit_count + ?,
|
||||
miss_count = miss_count + ?,
|
||||
upstream_fetch_count = upstream_fetch_count + ?,
|
||||
upstream_fetch_bytes = upstream_fetch_bytes + ?,
|
||||
transform_count = transform_count + ?,
|
||||
last_updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = 1
|
||||
`, hits, misses, upstreamFetches, upstreamFetchBytes, transforms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write pending counts: %w", err)
|
||||
}
|
||||
|
||||
c.pendingHits.Add(-hits)
|
||||
c.pendingMisses.Add(-misses)
|
||||
c.pendingUpstreamFetches.Add(-upstreamFetches)
|
||||
c.pendingUpstreamFetchBytes.Add(-upstreamFetchBytes)
|
||||
c.pendingTransforms.Add(-transforms)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// pendingCountsWait is how long a test waits for the database's connection,
|
||||
// for a count to reach the database, for a write to start waiting for the
|
||||
// connection, or for StopPendingCountWrites to return.
|
||||
const pendingCountsWait = 5 * time.Second
|
||||
|
||||
// holdDatabase takes the one connection of cache's database, so that every
|
||||
// other query waits for it, and returns the func that frees it.
|
||||
func holdDatabase(t *testing.T, cache *Cache) func() {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), pendingCountsWait)
|
||||
defer cancel()
|
||||
|
||||
conn, err := cache.db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to take the database connection: %v", err)
|
||||
}
|
||||
|
||||
return func() { _ = conn.Close() }
|
||||
}
|
||||
|
||||
// waitForConnectionWaits waits until db.Stats().WaitCount, the number of
|
||||
// times a caller has waited for a connection, reaches waits.
|
||||
func waitForConnectionWaits(t *testing.T, db *sql.DB, waits int64) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(pendingCountsWait)
|
||||
|
||||
for db.Stats().WaitCount < waits {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("callers waited for the database connection %d times, want %d",
|
||||
db.Stats().WaitCount, waits)
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForCounters waits until the cache_stats row holds want.
|
||||
func waitForCounters(t *testing.T, cache *Cache, want cacheStatsCounters) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(pendingCountsWait)
|
||||
|
||||
for {
|
||||
got := readCacheStatsCounters(t, cache)
|
||||
if got == want {
|
||||
return
|
||||
}
|
||||
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("counters = %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy holds the
|
||||
// database's one connection while a request whose fetch is held reaches its
|
||||
// deadline. The request must still return by its deadline with the
|
||||
// deadline's error, and its miss must reach the database once the
|
||||
// connection is free.
|
||||
func TestService_Get_ReturnsByItsDeadlineWhileTheDatabaseIsBusy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures, fetcher := setupHeldFetchService(t)
|
||||
|
||||
svc.cache.StartPendingCountWrites()
|
||||
defer func() { _ = svc.cache.StopPendingCountWrites(t.Context()) }()
|
||||
|
||||
// Room for the request to reach its fetch on a busy host
|
||||
const timeout = 2 * time.Second
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), timeout)
|
||||
defer cancel()
|
||||
|
||||
deadline, _ := ctx.Deadline()
|
||||
|
||||
results := startGet(ctx, svc, photoVariant(fixtures, 85, FitCover))
|
||||
|
||||
// The request has made its database reads by the time it fetches.
|
||||
select {
|
||||
case <-fetcher.started:
|
||||
case <-time.After(timeout):
|
||||
t.Fatal("request did not reach its fetch by its deadline")
|
||||
}
|
||||
|
||||
releaseDatabase := holdDatabase(t, svc.cache)
|
||||
defer releaseDatabase()
|
||||
|
||||
select {
|
||||
case got := <-results:
|
||||
t.Logf("Get() returned %v after its deadline, error = %v",
|
||||
time.Since(deadline), got.err)
|
||||
|
||||
if !errors.Is(got.err, context.DeadlineExceeded) {
|
||||
t.Errorf("Get() error = %v, want %v", got.err, context.DeadlineExceeded)
|
||||
}
|
||||
case <-time.After(time.Until(deadline) + time.Second):
|
||||
t.Fatal("request did not return by its deadline while the database was busy")
|
||||
}
|
||||
|
||||
releaseDatabase()
|
||||
|
||||
waitForCounters(t, svc.cache, cacheStatsCounters{missCount: 1})
|
||||
}
|
||||
|
||||
// TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting holds the
|
||||
// database's one connection while many misses are counted past their
|
||||
// deadline. At most one write may then wait for the connection, and every
|
||||
// miss must reach the database once it is free.
|
||||
func TestIncrementStats_ManyCountsPastTheirDeadlineLeaveOneWriteWaiting(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
cache.StartPendingCountWrites()
|
||||
defer func() { _ = cache.StopPendingCountWrites(t.Context()) }()
|
||||
|
||||
releaseDatabase := holdDatabase(t, cache)
|
||||
defer releaseDatabase()
|
||||
|
||||
waitsBefore := cache.db.Stats().WaitCount
|
||||
|
||||
// Writes given a context past its deadline fail at once.
|
||||
ended, cancel := context.WithDeadline(t.Context(), time.Now())
|
||||
defer cancel()
|
||||
|
||||
const misses = 20
|
||||
|
||||
for range misses {
|
||||
cache.IncrementStats(ended, false, 0)
|
||||
}
|
||||
|
||||
// Once one write waits for the connection, any others start waiting too.
|
||||
waitForConnectionWaits(t, cache.db, waitsBefore+1)
|
||||
time.Sleep(arrivalWait)
|
||||
|
||||
waits := cache.db.Stats().WaitCount - waitsBefore
|
||||
t.Logf("%d writes waited for the database", waits)
|
||||
|
||||
if waits > 1 {
|
||||
t.Errorf("%d writes waited for the database, want at most 1", waits)
|
||||
}
|
||||
|
||||
releaseDatabase()
|
||||
|
||||
waitForCounters(t, cache, cacheStatsCounters{missCount: misses})
|
||||
}
|
||||
|
||||
// TestStats_IncludesPendingCounts counts hits and misses whose writes miss
|
||||
// their deadline, and checks that Stats adds them to the database's counts
|
||||
// before they are written.
|
||||
func TestStats_IncludesPendingCounts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
_, err := cache.db.ExecContext(t.Context(),
|
||||
`UPDATE cache_stats SET hit_count = 75, miss_count = 25 WHERE id = 1`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Writes given a context past its deadline fail at once.
|
||||
ended, cancel := context.WithDeadline(t.Context(), time.Now())
|
||||
defer cancel()
|
||||
|
||||
cache.IncrementStats(ended, true, 0)
|
||||
cache.IncrementStats(ended, true, 0)
|
||||
cache.IncrementStats(ended, false, 0)
|
||||
|
||||
stats, err := cache.Stats(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("Stats() error = %v", err)
|
||||
}
|
||||
|
||||
if stats.HitCount != 77 || stats.MissCount != 26 {
|
||||
t.Errorf("HitCount = %d, MissCount = %d, want 77 and 26",
|
||||
stats.HitCount, stats.MissCount)
|
||||
}
|
||||
|
||||
want := cacheStatsCounters{hitCount: 75, missCount: 25}
|
||||
if got := readCacheStatsCounters(t, cache); got != want {
|
||||
t.Errorf("counters in the database = %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopPendingCountWrites_WritesPendingCounts holds the database's one
|
||||
// connection while the goroutine that writes the pending counts waits for it,
|
||||
// counts more, then stops that goroutine. The stop must leave the write under
|
||||
// way to finish rather than cut it off, and every count must reach the
|
||||
// database once the connection is free.
|
||||
func TestStopPendingCountWrites_WritesPendingCounts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
cache.StartPendingCountWrites()
|
||||
defer func() { _ = cache.StopPendingCountWrites(t.Context()) }()
|
||||
|
||||
releaseDatabase := holdDatabase(t, cache)
|
||||
defer releaseDatabase()
|
||||
|
||||
waitsBefore := cache.db.Stats().WaitCount
|
||||
|
||||
// Writes given a context past its deadline fail at once.
|
||||
ended, cancel := context.WithDeadline(t.Context(), time.Now())
|
||||
defer cancel()
|
||||
|
||||
cache.IncrementStats(ended, true, 0)
|
||||
|
||||
// The goroutine's write waits for the connection.
|
||||
waitForConnectionWaits(t, cache.db, waitsBefore+1)
|
||||
|
||||
// Counted after that write read the pending counts
|
||||
cache.IncrementStats(ended, false, 0)
|
||||
cache.IncrementUpstreamFetch(ended, 1024)
|
||||
cache.IncrementTransformCount(ended)
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
stopped <- cache.StopPendingCountWrites(t.Context())
|
||||
}()
|
||||
|
||||
// Had the stop cut off the write under way, its own write would wait for
|
||||
// the connection too.
|
||||
time.Sleep(arrivalWait)
|
||||
|
||||
waits := cache.db.Stats().WaitCount - waitsBefore
|
||||
t.Logf("%d writes waited for the database", waits)
|
||||
|
||||
if waits > 1 {
|
||||
t.Errorf("%d writes waited for the database, want 1: "+
|
||||
"the stop cut off the write under way", waits)
|
||||
}
|
||||
|
||||
releaseDatabase()
|
||||
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("StopPendingCountWrites() error = %v", err)
|
||||
}
|
||||
case <-time.After(pendingCountsWait):
|
||||
t.Fatal("StopPendingCountWrites() did not return once the database was free")
|
||||
}
|
||||
|
||||
want := cacheStatsCounters{1, 1, 1, 1024, 1}
|
||||
if got := readCacheStatsCounters(t, cache); got != want {
|
||||
t.Errorf("counters = %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopPendingCountWrites_ReturnsWhenItsContextEnds holds the database's
|
||||
// one connection while the goroutine that writes the pending counts waits for
|
||||
// it, then stops that goroutine with a context that ends first. The stop must
|
||||
// return the context's error, and the write under way must still reach the
|
||||
// database once the connection is free.
|
||||
func TestStopPendingCountWrites_ReturnsWhenItsContextEnds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
cache.StartPendingCountWrites()
|
||||
defer func() { _ = cache.StopPendingCountWrites(t.Context()) }()
|
||||
|
||||
releaseDatabase := holdDatabase(t, cache)
|
||||
defer releaseDatabase()
|
||||
|
||||
waitsBefore := cache.db.Stats().WaitCount
|
||||
|
||||
// Writes given a context past its deadline fail at once.
|
||||
ended, cancel := context.WithDeadline(t.Context(), time.Now())
|
||||
defer cancel()
|
||||
|
||||
cache.IncrementStats(ended, true, 0)
|
||||
|
||||
// The goroutine's write waits for the connection.
|
||||
waitForConnectionWaits(t, cache.db, waitsBefore+1)
|
||||
|
||||
stopCtx, cancelStop := context.WithCancel(t.Context())
|
||||
cancelStop()
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
stopped <- cache.StopPendingCountWrites(stopCtx)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("StopPendingCountWrites() error = %v, want %v",
|
||||
err, context.Canceled)
|
||||
}
|
||||
case <-time.After(pendingCountsWait):
|
||||
t.Fatal("StopPendingCountWrites() did not return once its context ended")
|
||||
}
|
||||
|
||||
releaseDatabase()
|
||||
|
||||
waitForCounters(t, cache, cacheStatsCounters{hitCount: 1})
|
||||
}
|
||||
@@ -42,7 +42,8 @@ type Service struct {
|
||||
type ServiceConfig struct {
|
||||
// Cache is the cache instance
|
||||
Cache *Cache
|
||||
// FetcherConfig configures the upstream fetcher (ignored if Fetcher is set)
|
||||
// FetcherConfig configures the upstream fetcher built when Fetcher is
|
||||
// not set. Its AllowHTTP and MaxResponseSize are used either way.
|
||||
FetcherConfig *httpfetcher.Config
|
||||
// Fetcher is an optional custom fetcher (for testing)
|
||||
Fetcher httpfetcher.Fetcher
|
||||
@@ -99,6 +100,13 @@ func NewService(cfg *ServiceConfig) (*Service, error) {
|
||||
allowHTTP = cfg.FetcherConfig.AllowHTTP
|
||||
}
|
||||
|
||||
// JPEG XL is the default output format, so pixad does not start
|
||||
// without it.
|
||||
err := imageprocessor.CheckJPEGXLSupport()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxResponseSize := fetcherCfg.MaxResponseSize
|
||||
processor := imageprocessor.New(imageprocessor.Params{
|
||||
MaxInputBytes: maxResponseSize,
|
||||
@@ -154,7 +162,7 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e
|
||||
// Fall through to re-process
|
||||
} else {
|
||||
// Counted also when the request context has ended meanwhile
|
||||
s.cache.IncrementStats(context.WithoutCancel(ctx), true, 0)
|
||||
s.cache.IncrementStats(ctx, true, 0)
|
||||
|
||||
return &ImageResponse{
|
||||
Content: reader,
|
||||
@@ -171,7 +179,7 @@ func (s *Service) Get(ctx context.Context, req *ImageRequest) (*ImageResponse, e
|
||||
// failed or the request context has ended meanwhile
|
||||
response, err := s.processOrWait(ctx, req)
|
||||
|
||||
s.cache.IncrementStats(context.WithoutCancel(ctx), false, 0)
|
||||
s.cache.IncrementStats(ctx, false, 0)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -292,14 +300,8 @@ func (s *Service) processOrWait(
|
||||
}
|
||||
}()
|
||||
|
||||
processingCtx := context.WithoutCancel(ctx)
|
||||
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
var cancel context.CancelFunc
|
||||
|
||||
processingCtx, cancel = context.WithDeadline(processingCtx, deadline)
|
||||
defer cancel()
|
||||
}
|
||||
processingCtx, cancel := withoutCancelKeepingDeadline(ctx)
|
||||
defer cancel()
|
||||
|
||||
return s.processFromSourceOrFetch(processingCtx, req, cacheKey)
|
||||
})
|
||||
@@ -336,6 +338,20 @@ func (s *Service) processOrWait(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// withoutCancelKeepingDeadline returns ctx without its cancellation but with
|
||||
// its deadline, if it has one, and the func that releases the returned
|
||||
// context.
|
||||
func withoutCancelKeepingDeadline(
|
||||
ctx context.Context,
|
||||
) (context.Context, context.CancelFunc) {
|
||||
deadline, hasDeadline := ctx.Deadline()
|
||||
if !hasDeadline {
|
||||
return context.WithoutCancel(ctx), func() {}
|
||||
}
|
||||
|
||||
return context.WithDeadline(context.WithoutCancel(ctx), deadline)
|
||||
}
|
||||
|
||||
// loadCachedSource opens source content from cache, without reading it, and
|
||||
// returns it with its size; nil if the cached data is unavailable, empty or
|
||||
// exceeds maxResponseSize.
|
||||
@@ -445,7 +461,7 @@ func (s *Service) fetchAndProcess(
|
||||
fetchBytes := int64(len(sourceData))
|
||||
|
||||
// Counted also when the request context has ended meanwhile
|
||||
s.cache.IncrementUpstreamFetch(context.WithoutCancel(ctx), fetchBytes)
|
||||
s.cache.IncrementUpstreamFetch(ctx, fetchBytes)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read upstream response: %w", err)
|
||||
@@ -527,7 +543,7 @@ func (s *Service) processAndStore(
|
||||
processDuration := time.Since(processStart)
|
||||
|
||||
// Counted also when the request context has ended meanwhile
|
||||
s.cache.IncrementTransformCount(context.WithoutCancel(ctx))
|
||||
s.cache.IncrementTransformCount(ctx)
|
||||
|
||||
// Read processed content
|
||||
processedData, err := io.ReadAll(processResult.Content)
|
||||
|
||||
@@ -38,8 +38,10 @@ func ValidateDimension(name string, value int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sizeFormatRegex matches patterns like "800x600.webp", "0x0.jpeg", "orig.png"
|
||||
var sizeFormatRegex = regexp.MustCompile(`^(\d+)x(\d+)\.(\w+)$|^(orig)\.(\w+)$`)
|
||||
// sizeFormatRegex matches patterns like "800x600.webp", "0x0.jpeg", "orig.png",
|
||||
// and a size with no format, such as "800x600" or "orig"
|
||||
var sizeFormatRegex = regexp.MustCompile(
|
||||
`^(\d+)x(\d+)(?:\.(\w+))?$|^(orig)(?:\.(\w+))?$`)
|
||||
|
||||
// ParsedURL contains the parsed components of an image proxy URL.
|
||||
type ParsedURL struct {
|
||||
@@ -56,12 +58,13 @@ type ParsedURL struct {
|
||||
}
|
||||
|
||||
// ParseImagePath parses the path captured by chi's wildcard:
|
||||
// <host>/<path>/<size>.<format>
|
||||
// <host>/<path>/<size>.<format>, or <host>/<path>/<size> for JPEG XL
|
||||
// This is the primary entry point when using chi routing.
|
||||
// Examples:
|
||||
// - cdn.example.com/photos/cat.jpg/800x600.webp
|
||||
// - cdn.example.com/photos/cat.jpg/0x0.jpeg
|
||||
// - cdn.example.com/photos/cat.jpg/orig.png
|
||||
// - cdn.example.com/photos/cat.jpg/800x600
|
||||
func ParseImagePath(path string) (*ParsedURL, error) {
|
||||
// Strip leading slash if present (chi may include it)
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
@@ -72,7 +75,8 @@ func ParseImagePath(path string) (*ParsedURL, error) {
|
||||
return parseImageComponents(path)
|
||||
}
|
||||
|
||||
// ParseImageURL parses a full URL path like /v1/image/<host>/<path>/<size>.<format>
|
||||
// ParseImageURL parses a full URL path like /v1/image/<host>/<path>/<size>.<format>,
|
||||
// or /v1/image/<host>/<path>/<size> for JPEG XL
|
||||
// Use ParseImagePath instead when working with chi's wildcard capture.
|
||||
func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
||||
// Remove the /v1/image/ prefix
|
||||
@@ -89,7 +93,8 @@ func ParseImageURL(urlPath string) (*ParsedURL, error) {
|
||||
return parseImageComponents(remainder)
|
||||
}
|
||||
|
||||
// parseImageComponents parses <host>/<path>/<size>.<format> structure.
|
||||
// parseImageComponents parses <host>/<path>/<size>.<format>, or
|
||||
// <host>/<path>/<size> for JPEG XL.
|
||||
func parseImageComponents(remainder string) (*ParsedURL, error) {
|
||||
// Check for path traversal before any other processing
|
||||
err := checkPathTraversal(remainder)
|
||||
@@ -97,7 +102,7 @@ func parseImageComponents(remainder string) (*ParsedURL, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find the last path segment which contains size.format
|
||||
// Find the last path segment, which holds "size" or "size.format"
|
||||
lastSlash := strings.LastIndex(remainder, "/")
|
||||
if lastSlash == -1 {
|
||||
return nil, ErrMissingSize
|
||||
@@ -212,7 +217,7 @@ func checkPathTraversal(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseSizeFormat parses strings like "800x600.webp" or "orig.png"
|
||||
// parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600"
|
||||
func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||
matches := sizeFormatRegex.FindStringSubmatch(s)
|
||||
if matches == nil {
|
||||
@@ -225,11 +230,11 @@ func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||
)
|
||||
|
||||
if matches[4] == "orig" {
|
||||
// "orig.format" pattern
|
||||
// "orig" or "orig.format" pattern
|
||||
size = Size{Width: 0, Height: 0}
|
||||
formatStr = matches[5]
|
||||
} else {
|
||||
// "WxH.format" pattern
|
||||
// "WxH" or "WxH.format" pattern
|
||||
width, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return Size{}, "", ErrInvalidSize
|
||||
@@ -254,6 +259,11 @@ func parseSizeFormat(s string) (Size, ImageFormat, error) {
|
||||
return Size{}, "", err
|
||||
}
|
||||
|
||||
// A URL that names no format is served, and signed, as JPEG XL
|
||||
if formatStr == "" {
|
||||
return size, FormatJXL, nil
|
||||
}
|
||||
|
||||
format, err := parseFormat(formatStr)
|
||||
if err != nil {
|
||||
return Size{}, "", err
|
||||
@@ -275,8 +285,12 @@ func parseFormat(s string) (ImageFormat, error) {
|
||||
return FormatWebP, nil
|
||||
case "avif":
|
||||
return FormatAVIF, nil
|
||||
case "jxl":
|
||||
return FormatJXL, nil
|
||||
case "gif":
|
||||
return FormatGIF, nil
|
||||
case "auto":
|
||||
return FormatAuto, nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %s", ErrInvalidFormat, s)
|
||||
}
|
||||
|
||||
@@ -111,6 +111,21 @@ func TestParseImageURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImageURL_AutoFormat verifies that the format auto in an image URL
|
||||
// parses as FormatAuto.
|
||||
func TestParseImageURL_AutoFormat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImageURL("/v1/image/example.com/photo.jpg/200x200.auto")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImageURL() error = %v", err)
|
||||
}
|
||||
|
||||
if got.Format != FormatAuto {
|
||||
t.Errorf("Format = %q, want %q", got.Format, FormatAuto)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImageURL_Errors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// Package loadtestorigin is the upstream host script/loadtest points pixad
|
||||
// at, run by cmd/loadtest-origin. It answers every request, whatever its path,
|
||||
// with the same generated JPEG, so each new path is a new source image for
|
||||
// pixad to fetch, and it logs one line per request, so its log counts pixad's
|
||||
// fetches.
|
||||
package loadtestorigin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
listenAddress = ":80"
|
||||
readHeaderTimeout = 10 * time.Second
|
||||
imageWidth = 1600
|
||||
imageHeight = 1200
|
||||
jpegQuality = 85
|
||||
)
|
||||
|
||||
// Run makes the image and serves it on port 80 until the server fails, then
|
||||
// exits the process with status 1.
|
||||
func Run() {
|
||||
photo, err := makeJPEG()
|
||||
if err != nil {
|
||||
slog.Error("cannot make the image", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: listenAddress,
|
||||
Handler: newHandler(photo),
|
||||
ReadHeaderTimeout: readHeaderTimeout,
|
||||
}
|
||||
|
||||
err = server.ListenAndServe()
|
||||
slog.Error("server stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// newHandler answers every request with photo and logs the request's path.
|
||||
func newHandler(photo []byte) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
slog.Info("request", "path", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(photo)
|
||||
})
|
||||
}
|
||||
|
||||
// makeJPEG draws colour gradients crossed with a fine pattern, so the image
|
||||
// has detail to decode and does not compress to almost nothing.
|
||||
func makeJPEG() ([]byte, error) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight))
|
||||
|
||||
// red and green count up from 0 to 255 and wrap around, along each row
|
||||
// and down the image.
|
||||
var green uint8
|
||||
|
||||
for y := range imageHeight {
|
||||
var red uint8
|
||||
|
||||
for x := range imageWidth {
|
||||
img.SetRGBA(x, y, color.RGBA{
|
||||
R: red, G: green, B: red ^ green, A: math.MaxUint8,
|
||||
})
|
||||
red++
|
||||
}
|
||||
|
||||
green++
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package loadtestorigin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEveryPathServesTheSameJPEG checks that the origin answers any path with
|
||||
// 200 and the same JPEG, so every new path script/loadtest asks pixad for is
|
||||
// a valid source image.
|
||||
func TestEveryPathServesTheSameJPEG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photo, err := makeJPEG()
|
||||
if err != nil {
|
||||
t.Fatalf("makeJPEG: %v", err)
|
||||
}
|
||||
|
||||
size, err := jpeg.DecodeConfig(bytes.NewReader(photo))
|
||||
if err != nil {
|
||||
t.Fatalf("the image does not decode as a JPEG: %v", err)
|
||||
}
|
||||
|
||||
if size.Width != imageWidth || size.Height != imageHeight {
|
||||
t.Errorf("the image is %dx%d, want %dx%d",
|
||||
size.Width, size.Height, imageWidth, imageHeight)
|
||||
}
|
||||
|
||||
handler := newHandler(photo)
|
||||
|
||||
for _, path := range []string{"/", "/miss/1.jpg", "/herd/2.jpg"} {
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, path, nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("%s: status = %d, want %d", path, rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "image/jpeg" {
|
||||
t.Errorf("%s: Content-Type = %q, want image/jpeg", path, ct)
|
||||
}
|
||||
|
||||
if !bytes.Equal(rec.Body.Bytes(), photo) {
|
||||
t.Errorf("%s: the body is not the image", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package magic
|
||||
|
||||
import "testing"
|
||||
|
||||
// testMIMEJXL is the MIME type of JPEG XL.
|
||||
const testMIMEJXL = "image/jxl"
|
||||
|
||||
// TestDetectFormatJPEGXL verifies that both JPEG XL signatures, that of a
|
||||
// bare codestream and that of the container, are detected as image/jxl.
|
||||
func TestDetectFormatJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
}{
|
||||
{"codestream", pad(0xFF, 0x0A)},
|
||||
{"container", pad(
|
||||
0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x4C, 0x20, 0x0D, 0x0A, 0x87, 0x0A,
|
||||
)},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := DetectFormat(tt.data)
|
||||
if err != nil || got != MIMETypeJXL {
|
||||
t.Errorf("DetectFormat() = %q, %v, want %q", got, err, testMIMEJXL)
|
||||
}
|
||||
|
||||
err = ValidateMagicBytes(tt.data, testMIMEJXL)
|
||||
if err != nil {
|
||||
t.Errorf("ValidateMagicBytes(%q) error = %v", testMIMEJXL, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestJPEGXLMIMEType verifies that image/jxl is a supported input type and
|
||||
// maps to and from the format jxl.
|
||||
func TestJPEGXLMIMEType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if !IsSupportedMIMEType(testMIMEJXL) {
|
||||
t.Errorf("IsSupportedMIMEType(%q) = false", testMIMEJXL)
|
||||
}
|
||||
|
||||
format, ok := MIMEToImageFormat(testMIMEJXL)
|
||||
if format != FormatJXL || !ok {
|
||||
t.Errorf("MIMEToImageFormat(%q) = %q, %v, want %q, true",
|
||||
testMIMEJXL, format, ok, FormatJXL)
|
||||
}
|
||||
|
||||
if got := ImageFormatToMIME(FormatJXL); got != testMIMEJXL {
|
||||
t.Errorf("ImageFormatToMIME(%q) = %q, want %q", FormatJXL, got, testMIMEJXL)
|
||||
}
|
||||
}
|
||||
+19
-1
@@ -26,6 +26,7 @@ const (
|
||||
MIMETypeWebP = MIMEType("image/webp")
|
||||
MIMETypeGIF = MIMEType("image/gif")
|
||||
MIMETypeAVIF = MIMEType("image/avif")
|
||||
MIMETypeJXL = MIMEType("image/jxl")
|
||||
MIMETypeSVG = MIMEType("image/svg+xml")
|
||||
)
|
||||
|
||||
@@ -40,6 +41,7 @@ const (
|
||||
FormatPNG ImageFormat = "png"
|
||||
FormatWebP ImageFormat = "webp"
|
||||
FormatAVIF ImageFormat = "avif"
|
||||
FormatJXL ImageFormat = "jxl"
|
||||
FormatGIF ImageFormat = "gif"
|
||||
)
|
||||
|
||||
@@ -62,6 +64,11 @@ var (
|
||||
// AVIF uses the ftyp box with brand "avif" or "avis"
|
||||
// Format: size(4 bytes) + "ftyp" + brand(4 bytes)
|
||||
magicFtyp = []byte{0x66, 0x74, 0x79, 0x70} // "ftyp"
|
||||
// JPEG XL is a bare codestream or a codestream in a container
|
||||
magicJXLCodestream = []byte{0xFF, 0x0A}
|
||||
magicJXLContainer = []byte{
|
||||
0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x4C, 0x20, 0x0D, 0x0A, 0x87, 0x0A,
|
||||
}
|
||||
)
|
||||
|
||||
// WebP identifier appears at offset 8 after RIFF header.
|
||||
@@ -115,6 +122,12 @@ func DetectFormat(data []byte) (MIMEType, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Check JPEG XL (FF0A, or the 12-byte container signature)
|
||||
if bytes.HasPrefix(data, magicJXLCodestream) ||
|
||||
bytes.HasPrefix(data, magicJXLContainer) {
|
||||
return MIMETypeJXL, nil
|
||||
}
|
||||
|
||||
// Check SVG - look for XML declaration or SVG tag
|
||||
if detectSVG(data) {
|
||||
return MIMETypeSVG, nil
|
||||
@@ -181,7 +194,8 @@ func normalizeMIMEType(mimeType string) string {
|
||||
func IsSupportedMIMEType(mimeType string) bool {
|
||||
normalized := normalizeMIMEType(mimeType)
|
||||
switch MIMEType(normalized) {
|
||||
case MIMETypeJPEG, MIMETypePNG, MIMETypeWebP, MIMETypeGIF, MIMETypeAVIF, MIMETypeSVG:
|
||||
case MIMETypeJPEG, MIMETypePNG, MIMETypeWebP, MIMETypeGIF, MIMETypeAVIF,
|
||||
MIMETypeJXL, MIMETypeSVG:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -226,6 +240,8 @@ func MIMEToImageFormat(mimeType string) (ImageFormat, bool) {
|
||||
return FormatGIF, true
|
||||
case MIMETypeAVIF:
|
||||
return FormatAVIF, true
|
||||
case MIMETypeJXL:
|
||||
return FormatJXL, true
|
||||
case MIMETypeSVG:
|
||||
// SVG has no corresponding output format.
|
||||
return "", false
|
||||
@@ -247,6 +263,8 @@ func ImageFormatToMIME(format ImageFormat) string {
|
||||
return string(MIMETypeGIF)
|
||||
case FormatAVIF:
|
||||
return string(MIMETypeAVIF)
|
||||
case FormatJXL:
|
||||
return string(MIMETypeJXL)
|
||||
case FormatOriginal:
|
||||
// Original format passes content through unchanged.
|
||||
return mimeOctetStream
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestFormatAutoVaryNextToOrigin requests an image URL with the format auto
|
||||
// through the server's routes and verifies that the answer carries
|
||||
// Vary: Accept next to the Vary: Origin the CORS middleware sends.
|
||||
func TestFormatAutoVaryNextToOrigin(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := encodeTestPNG(t, 64, 48)
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(source)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"/v1/image/"+upstreamHost+"/photo.png/32x24.auto", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Accept", "image/webp")
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, req)
|
||||
|
||||
vary := rec.Header().Values("Vary")
|
||||
t.Logf("status %d, Content-Type %s, Vary %v",
|
||||
rec.Code, rec.Header().Get("Content-Type"), vary)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
if want := []string{"Origin", "Accept"}; !slices.Equal(vary, want) {
|
||||
t.Errorf("Vary = %v, want %v", vary, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxtest"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
)
|
||||
|
||||
// upstreamHost is the upstream host of the image URLs below. It is a
|
||||
// documentation address (RFC 5737), which the fetcher's URL check accepts as
|
||||
// public; the fetcher's dial function connects it to the test upstream server.
|
||||
const upstreamHost = "192.0.2.10"
|
||||
|
||||
// TestImageProxyFlow requests images through pixa's router, handlers,
|
||||
// upstream fetcher, image processor, disk cache and database, with only the
|
||||
// upstream origin replaced by a local test server. The first request for a URL
|
||||
// is fetched and converted; the second is served from the cache without
|
||||
// another upstream request. The source and the converted image are then on
|
||||
// disk, with their rows in the database.
|
||||
func TestImageProxyFlow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := encodeTestPNG(t, 64, 48)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sizeFormat string // the <size>.<format> part of the image URL
|
||||
contentType string
|
||||
decodeConfig func(io.Reader) (image.Config, error)
|
||||
width, height int
|
||||
}{
|
||||
{"resize and convert to JPEG", "32x24.jpeg", "image/jpeg",
|
||||
jpeg.DecodeConfig, 32, 24},
|
||||
{"orig", "orig.orig", "image/png", png.DecodeConfig, 64, 48},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var upstreamRequests atomic.Int32
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
upstreamRequests.Add(1)
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(source)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, db, stateDir := startImageProxy(t, upstream)
|
||||
target := "/v1/image/" + upstreamHost + "/photo.png/" + tt.sizeFormat
|
||||
|
||||
first := getImage(t, s, target)
|
||||
|
||||
if got := first.Header().Get("X-Pixa-Cache"); got != "MISS" {
|
||||
t.Errorf("first X-Pixa-Cache = %q, want MISS", got)
|
||||
}
|
||||
|
||||
if got := first.Header().Get("Content-Type"); got != tt.contentType {
|
||||
t.Errorf("Content-Type = %q, want %q", got, tt.contentType)
|
||||
}
|
||||
|
||||
decoded, err := tt.decodeConfig(bytes.NewReader(first.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("decoding the image: %v", err)
|
||||
}
|
||||
|
||||
if decoded.Width != tt.width || decoded.Height != tt.height {
|
||||
t.Errorf("image is %dx%d, want %dx%d",
|
||||
decoded.Width, decoded.Height, tt.width, tt.height)
|
||||
}
|
||||
|
||||
second := getImage(t, s, target)
|
||||
|
||||
if got := second.Header().Get("X-Pixa-Cache"); got != "HIT" {
|
||||
t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
|
||||
}
|
||||
|
||||
if !bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
|
||||
t.Error("the second response is not the image the first served")
|
||||
}
|
||||
|
||||
if got := upstreamRequests.Load(); got != 1 {
|
||||
t.Errorf("upstream received %d requests, want 1", got)
|
||||
}
|
||||
|
||||
checkSourceCached(t, db, stateDir, source)
|
||||
checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// startImageProxy starts the components pixad's fx app builds, from a config
|
||||
// with a fresh state directory and upstreamHost on the allowlist, and with an
|
||||
// upstream fetcher that connects every upstream address to upstream. It
|
||||
// returns the server with its routes, the database and the state directory.
|
||||
func startImageProxy(
|
||||
t *testing.T, upstream *httptest.Server,
|
||||
) (*Server, *sql.DB, string) {
|
||||
t.Helper()
|
||||
|
||||
stateDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
SigningKey: testSigningKey,
|
||||
StateDir: stateDir,
|
||||
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||
AllowlistHosts: []string{upstreamHost},
|
||||
// The test upstream server has no TLS.
|
||||
AllowHTTP: true,
|
||||
// A limit of its own, so the cache does not size itself from the
|
||||
// host's free disk space.
|
||||
CacheMaxBytes: 64 << 20,
|
||||
CacheMaxBytesExplicit: true,
|
||||
UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize,
|
||||
DownstreamTimeout: config.DefaultDownstreamTimeout,
|
||||
}
|
||||
|
||||
fetcherCfg := httpfetcher.DefaultConfig()
|
||||
fetcherCfg.AllowHTTP = true
|
||||
fetcherCfg.DialContext = func(
|
||||
ctx context.Context, network, _ string,
|
||||
) (net.Conn, error) {
|
||||
var dialer net.Dialer
|
||||
|
||||
return dialer.DialContext(ctx, network, upstream.Listener.Addr().String())
|
||||
}
|
||||
fetcher := httpfetcher.New(fetcherCfg)
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
mw *middleware.Middleware
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
app := fxtest.New(t,
|
||||
fx.Supply(cfg),
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
database.New,
|
||||
healthcheck.New,
|
||||
handlers.New,
|
||||
middleware.New,
|
||||
func() httpfetcher.Fetcher { return fetcher },
|
||||
),
|
||||
fx.Populate(&h, &mw, &db),
|
||||
)
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// Requests go straight to the router, as in newTestServer; the server's
|
||||
// own start hook, which listens on a port, is left out.
|
||||
s := &Server{config: cfg, mw: mw, h: h}
|
||||
s.SetupRoutes()
|
||||
|
||||
return s, db.DB(), stateDir
|
||||
}
|
||||
|
||||
// getImage sends a GET for target to s and fails unless it answers 200.
|
||||
func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
t.Logf("GET %s: %d, X-Pixa-Cache %s",
|
||||
target, rec.Code, rec.Header().Get("X-Pixa-Cache"))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s status = %d, want %d; body %s",
|
||||
target, rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// checkSourceCached checks that source is stored under its SHA-256 in
|
||||
// cache/sources, recorded in source_content, and that the source URL's row in
|
||||
// source_metadata points at it.
|
||||
func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
|
||||
t.Helper()
|
||||
|
||||
sum := sha256.Sum256(source)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
|
||||
checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
|
||||
source)
|
||||
|
||||
var rows int
|
||||
|
||||
err := db.QueryRowContext(t.Context(),
|
||||
"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
|
||||
).Scan(&rows)
|
||||
if err != nil || rows != 1 {
|
||||
t.Errorf("source_content rows for the source = %d (error %v), want 1",
|
||||
rows, err)
|
||||
}
|
||||
|
||||
var metadataHash string
|
||||
|
||||
err = db.QueryRowContext(t.Context(),
|
||||
`SELECT content_hash FROM source_metadata
|
||||
WHERE source_host = ? AND source_path = ?`,
|
||||
upstreamHost, "/photo.png",
|
||||
).Scan(&metadataHash)
|
||||
if err != nil || metadataHash != hash {
|
||||
t.Errorf("source_metadata content_hash = %q (error %v), want %q",
|
||||
metadataHash, err, hash)
|
||||
}
|
||||
}
|
||||
|
||||
// checkVariantCached checks that the converted image served is recorded in
|
||||
// variant_content with contentType, and stored under its cache key in
|
||||
// cache/variants.
|
||||
func checkVariantCached(
|
||||
t *testing.T, db *sql.DB, stateDir string, served []byte, contentType string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
var cacheKey, storedType string
|
||||
|
||||
err := db.QueryRowContext(t.Context(),
|
||||
"SELECT cache_key, content_type FROM variant_content",
|
||||
).Scan(&cacheKey, &storedType)
|
||||
if err != nil {
|
||||
t.Fatalf("variant_content row: %v", err)
|
||||
}
|
||||
|
||||
if storedType != contentType {
|
||||
t.Errorf("variant_content content_type = %q, want %q",
|
||||
storedType, contentType)
|
||||
}
|
||||
|
||||
checkFile(t, filepath.Join(stateDir, "cache", "variants",
|
||||
cacheKey[0:2], cacheKey[2:4], cacheKey), served)
|
||||
}
|
||||
|
||||
// checkFile checks that the file at path holds want.
|
||||
func checkFile(t *testing.T, path string, want []byte) {
|
||||
t.Helper()
|
||||
|
||||
//nolint:gosec // G304: a path under the test's state directory
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Errorf("reading %s: %v", path, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("%s holds %d bytes that are not the %d expected",
|
||||
path, len(got), len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// encodeTestPNG returns an opaque width x height PNG of one color.
|
||||
func encodeTestPNG(t *testing.T, width, height int) []byte {
|
||||
t.Helper()
|
||||
|
||||
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
img.Set(x, y, color.RGBA{R: 200, G: 40, B: 40, A: 255})
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := png.Encode(&buf, img)
|
||||
if err != nil {
|
||||
t.Fatalf("encoding the test PNG: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestImageProxyFlowJPEGXL requests the JPEG XL testdata/red.jxl, 64x48 and
|
||||
// made with libvips, through pixa's router, upstream fetcher, image processor
|
||||
// and disk cache: resized as PNG, resized as JPEG XL, and at its own size and
|
||||
// format, which is JPEG XL. The second request for each URL is a cache hit.
|
||||
func TestImageProxyFlowJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source, err := os.ReadFile("testdata/red.jxl")
|
||||
if err != nil {
|
||||
t.Fatalf("reading the test image: %v", err)
|
||||
}
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jxl")
|
||||
_, _ = w.Write(source)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, _, _ := startImageProxy(t, upstream)
|
||||
|
||||
tests := []struct {
|
||||
sizeFormat string // the <size>.<format> part of the image URL
|
||||
contentType string
|
||||
imageType vips.ImageType
|
||||
width, height int
|
||||
}{
|
||||
{"32x24.png", "image/png", vips.ImageTypePNG, 32, 24},
|
||||
{"32x24.jxl", "image/jxl", vips.ImageTypeJXL, 32, 24},
|
||||
{"orig.orig", "image/jxl", vips.ImageTypeJXL, 64, 48},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
target := "/v1/image/" + upstreamHost + "/red.jxl/" + tt.sizeFormat
|
||||
|
||||
for _, wantCache := range []string{"MISS", "HIT"} {
|
||||
rec := getImage(t, s, target)
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
gotCache := rec.Header().Get("X-Pixa-Cache")
|
||||
|
||||
if gotType != tt.contentType || gotCache != wantCache {
|
||||
t.Errorf("%s: %s, X-Pixa-Cache %s, want %s, %s",
|
||||
target, gotType, gotCache, tt.contentType, wantCache)
|
||||
}
|
||||
|
||||
img, err := vips.NewImageFromBuffer(rec.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("%s: loading the image: %v", target, err)
|
||||
}
|
||||
|
||||
if img.Format() != tt.imageType ||
|
||||
img.Width() != tt.width || img.Height() != tt.height {
|
||||
t.Errorf("%s: %s %dx%d, want %s %dx%d", target,
|
||||
vips.ImageTypes[img.Format()], img.Width(), img.Height(),
|
||||
vips.ImageTypes[tt.imageType], tt.width, tt.height)
|
||||
}
|
||||
|
||||
img.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,8 @@ func (s *Server) SetupRoutes() {
|
||||
r.Use(s.refuseDuringMaintenance)
|
||||
|
||||
// Main image proxy route
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>, or with no
|
||||
// format /v1/image/<host>/<path>/<width>x<height>
|
||||
r.Get("/image/*", s.h.HandleImage())
|
||||
r.Head("/image/*", s.h.HandleImage())
|
||||
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -95,10 +95,12 @@
|
||||
</label>
|
||||
<select id="format" name="format">
|
||||
<option value="orig" {{if eq .FormFormat "orig"}}selected{{end}}>Original</option>
|
||||
<option value="auto" {{if eq .FormFormat "auto"}}selected{{end}}>Auto (JPEG XL, AVIF, WebP or JPEG)</option>
|
||||
<option value="jpeg" {{if eq .FormFormat "jpeg"}}selected{{end}}>JPEG</option>
|
||||
<option value="png" {{if eq .FormFormat "png"}}selected{{end}}>PNG</option>
|
||||
<option value="webp" {{if eq .FormFormat "webp"}}selected{{end}}>WebP</option>
|
||||
<option value="avif" {{if eq .FormFormat "avif"}}selected{{end}}>AVIF</option>
|
||||
<option value="jxl" {{if or (eq .FormFormat "jxl") (eq .FormFormat "")}}selected{{end}}>JPEG XL</option>
|
||||
<option value="gif" {{if eq .FormFormat "gif"}}selected{{end}}>GIF</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"license": "GPL-3.0",
|
||||
"devDependencies": {
|
||||
"prettier": "3.8.1"
|
||||
}
|
||||
}
|
||||
+119
-8
@@ -3,16 +3,34 @@
|
||||
# this repo. Idempotent: every install is guarded by a check so already
|
||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||
# make, or go). The linter is never installed on the host: golangci-lint
|
||||
# runs only inside a container, Dockerfile.lint or the Dockerfile lint
|
||||
# stage (see script/lint). A C compiler and the CGO image libraries
|
||||
# (pkg-config, vips, libheif) are installed for the govips bindings.
|
||||
# Both Dockerfiles run this script too, so their build dependencies are
|
||||
# the ones listed here.
|
||||
# make, or go). Node is used directly if installed; otherwise it is
|
||||
# installed at a pinned version via nvm (installing nvm itself first,
|
||||
# from a hash-verified release archive, never curl | sh). The linter is
|
||||
# never installed on the host: golangci-lint runs only in the lint phase
|
||||
# of the Dockerfile (see script/lint).
|
||||
#
|
||||
# script/bootstrap git, make, Go, and Node, Yarn and the
|
||||
# prettier in yarn.lock for script/fmt and
|
||||
# script/fmt-check: all the host needs, as
|
||||
# the checks compile pixa in Docker
|
||||
# script/bootstrap --cgo git, make, Go, and a C compiler and the
|
||||
# CGO image libraries (pkg-config, vips
|
||||
# with its JPEG XL support, libheif) for
|
||||
# the govips bindings instead of Node: to
|
||||
# compile pixa, in the Dockerfile's test
|
||||
# phase and build stage, which format
|
||||
# nothing
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Pinned versions, 2026-07-06
|
||||
NODE_VERSION="22.17.0"
|
||||
NVM_VERSION="0.40.3"
|
||||
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
||||
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
||||
YARN_VERSION="1.22.22"
|
||||
|
||||
PKGMGR=""
|
||||
SUDO=""
|
||||
|
||||
@@ -35,6 +53,10 @@ detect_pkgmgr() {
|
||||
if [ "$(id -u)" != "0" ]; then
|
||||
SUDO="sudo"
|
||||
fi
|
||||
# This runs before the first install only. A fresh image, such
|
||||
# as a CI runner's, has no package lists, and apt-get install
|
||||
# finds no package without them.
|
||||
$SUDO apt-get update
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -53,6 +75,69 @@ missing() {
|
||||
! command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# verify_sha256 <file> <expected-hash>
|
||||
verify_sha256() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual="$(sha256sum "$1" | cut -d' ' -f1)"
|
||||
else
|
||||
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
|
||||
fi
|
||||
if [ "$actual" != "$2" ]; then
|
||||
echo "bootstrap: sha256 mismatch for $1" >&2
|
||||
echo " expected: $2" >&2
|
||||
echo " actual: $actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# nvm is a bash script; run a command in a bash with nvm loaded
|
||||
nvm_sh() {
|
||||
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
|
||||
}
|
||||
|
||||
ensure_nvm() {
|
||||
[ -s "$HOME/.nvm/nvm.sh" ] && return 0
|
||||
# nvm prerequisites; nvm itself requires bash
|
||||
if missing bash; then pkg_install bash bash bash bash; fi
|
||||
if missing curl; then pkg_install curl curl curl curl; fi
|
||||
if missing git; then pkg_install git git git git; fi
|
||||
tmp="$(mktemp -d)"
|
||||
curl -fsSL -o "$tmp/nvm.tar.gz" \
|
||||
"https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz"
|
||||
verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256"
|
||||
mkdir -p "$HOME/.nvm"
|
||||
tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
ensure_node() {
|
||||
if ! missing node; then return 0; fi
|
||||
ensure_nvm
|
||||
nvm_sh "nvm install $NODE_VERSION"
|
||||
}
|
||||
|
||||
ensure_yarn() {
|
||||
if ! missing yarn; then return 0; fi
|
||||
if ! missing corepack; then
|
||||
corepack enable
|
||||
corepack prepare "yarn@$YARN_VERSION" --activate
|
||||
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
||||
corepack prepare yarn@$YARN_VERSION --activate"
|
||||
else
|
||||
npm install -g "yarn@$YARN_VERSION"
|
||||
fi
|
||||
}
|
||||
|
||||
install_js_deps() {
|
||||
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
|
||||
yarn install --frozen-lockfile"
|
||||
else
|
||||
yarn install --frozen-lockfile
|
||||
fi
|
||||
}
|
||||
|
||||
# CGO dependencies for govips (image processing)
|
||||
ensure_cgo_deps() {
|
||||
# cgo compiles with gcc on Linux; build-base and build-essential
|
||||
@@ -66,12 +151,31 @@ ensure_cgo_deps() {
|
||||
if ! pkg-config --exists vips; then
|
||||
pkg_install vips libvips-dev vips vips-dev
|
||||
fi
|
||||
# libvips' JPEG XL loader and saver are in the nix and brew vips
|
||||
# packages, and in apt's from Debian 12 and Ubuntu 24.04 on, but in
|
||||
# the package vips-jxl on Alpine. detect_pkgmgr is called only where
|
||||
# apk exists, as on apt it updates the package lists.
|
||||
if ! missing apk; then
|
||||
detect_pkgmgr
|
||||
fi
|
||||
if [ "$PKGMGR" = "apk" ] && ! apk info -e vips-jxl >/dev/null; then
|
||||
apk add --no-cache vips-jxl
|
||||
fi
|
||||
if ! pkg-config --exists libheif; then
|
||||
pkg_install libheif libheif-dev libheif libheif-dev
|
||||
fi
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "usage: script/bootstrap [--cgo]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
main() {
|
||||
case "$*" in
|
||||
"" | --cgo) ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
cd "$ROOT"
|
||||
|
||||
# Base tooling
|
||||
@@ -81,8 +185,15 @@ main() {
|
||||
# Go toolchain
|
||||
if missing go; then pkg_install go golang go go; fi
|
||||
|
||||
# CGO image libraries
|
||||
ensure_cgo_deps
|
||||
# CGO image libraries where pixa is compiled; elsewhere Node, Yarn
|
||||
# and prettier
|
||||
if [ "$*" = "--cgo" ]; then
|
||||
ensure_cgo_deps
|
||||
else
|
||||
ensure_node
|
||||
ensure_yarn
|
||||
install_js_deps
|
||||
fi
|
||||
|
||||
go mod download
|
||||
|
||||
|
||||
+3
-2
@@ -1,7 +1,8 @@
|
||||
#!/bin/sh
|
||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||
# extension to scripts-to-rule-them-all. Must not modify any files.
|
||||
# Generic: usually needs no adaptation.
|
||||
# extension to scripts-to-rule-them-all. test and lint are Docker
|
||||
# phases; fmt-check is native, because a formatter writes the working
|
||||
# tree. Must not modify any files.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
+20
-9
@@ -1,18 +1,29 @@
|
||||
#!/bin/sh
|
||||
# script/cibuild: run the CI build. The Dockerfile runs the checks
|
||||
# (make fmt-check, lint, test) as build steps. This script passes a new
|
||||
# CHECK_EPOCH on every run, so Docker runs those steps instead of
|
||||
# reusing cached results: a successful run means the checks ran and
|
||||
# passed on this tree. Generic: needs no adaptation. The Gitea workflow
|
||||
# runs this on push.
|
||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||
# the formatter on the host, which a pristine checkout cannot do.
|
||||
# --no-cache for the same reason as script/docker: the gate phases the
|
||||
# final stage depends on are RUN steps, and a cached one is a check that
|
||||
# did not run.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
epoch="$(date +%s)$$"
|
||||
docker build --build-arg CHECK_EPOCH="$epoch" .
|
||||
"$SCRIPT_DIR/bootstrap"
|
||||
"$SCRIPT_DIR/check"
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant. VERSION is computed here because .dockerignore
|
||||
# excludes .git, so `git describe` in a build stage yields an empty
|
||||
# version without failing.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$("$SCRIPT_DIR/projectname")" .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+12
-6
@@ -1,9 +1,8 @@
|
||||
#!/bin/sh
|
||||
# script/docker: build the Docker image tagged with the project name.
|
||||
# Identical in all repos; the tag comes from script/projectname. Like
|
||||
# script/cibuild, it passes a new CHECK_EPOCH, so the build runs the
|
||||
# checks instead of reusing cached results. Generic: needs no
|
||||
# adaptation.
|
||||
# Identical in all repos; the tag comes from script/projectname.
|
||||
# --no-cache because the gate phases the final stage depends on are RUN
|
||||
# steps, and a cached one is a check that did not run.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -11,8 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
epoch="$(date +%s)$$"
|
||||
docker build --build-arg CHECK_EPOCH="$epoch" \
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant. VERSION is computed here because .dockerignore
|
||||
# excludes .git, so `git describe` in a build stage yields an empty
|
||||
# version without failing.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$("$SCRIPT_DIR/projectname")" .
|
||||
}
|
||||
|
||||
|
||||
+20
@@ -4,11 +4,31 @@ set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
|
||||
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
echo "Formatting code..."
|
||||
# shellcheck disable=SC2046 # word splitting of file list is wanted
|
||||
gofmt -w $(find . -name '*.go' -not -path './vendor/*')
|
||||
run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -5,6 +5,25 @@ set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt-check: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
|
||||
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
echo "Checking formatting..."
|
||||
@@ -13,6 +32,7 @@ main() {
|
||||
gofmt -l . | grep -v '^vendor/'
|
||||
exit 1
|
||||
fi
|
||||
run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
#!/bin/sh
|
||||
# script/install-precommit: install the git pre-commit hook that runs
|
||||
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
||||
# Generic: needs no adaptation.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
hook=".git/hooks/pre-commit"
|
||||
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
|
||||
chmod +x .git/hooks/pre-commit
|
||||
echo "pre-commit hook installed: runs script/precommit"
|
||||
|
||||
+13
-27
@@ -1,37 +1,23 @@
|
||||
#!/bin/sh
|
||||
# script/lint: run golangci-lint over the whole tree. This is the only
|
||||
# way the linter is run, everywhere; it is never installed on the host.
|
||||
# script/lint: run the linter. Linting is a phase of the Dockerfile and
|
||||
# this builds that phase alone; the linter is never installed or run on
|
||||
# a developer host, where a shared result cache and a host-global lock
|
||||
# make its answer untrustworthy.
|
||||
#
|
||||
# Inside a container it runs the linter. Anywhere else it builds
|
||||
# Dockerfile.lint, whose last step runs this script again inside that
|
||||
# container.
|
||||
#
|
||||
# Dockerfile.lint and the Dockerfile lint stage set container=docker
|
||||
# (the systemd convention for marking a container) to say where we are.
|
||||
# /.dockerenv cannot: it is missing inside build steps, and present on
|
||||
# hosts that are themselves containers.
|
||||
# The phase is not the last stage in the file, so it is built only when
|
||||
# --target names it. --no-cache because a cached lint layer is a lint
|
||||
# that did not run. The tag makes each build replace the previous image
|
||||
# instead of leaving a dangling one behind.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
if [ "${container:-}" = docker ]; then
|
||||
# `golangci-lint config verify` is not run: it fetches its JSON
|
||||
# schema over an unpinned live HTTPS call, which REPO_POLICIES.md
|
||||
# forbids.
|
||||
echo "Running linter..."
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
else
|
||||
# A new CACHEBUST on every run means the lint step is never
|
||||
# served from cache (see Dockerfile.lint). The cacheonly output
|
||||
# leaves no image behind.
|
||||
docker build \
|
||||
--progress=plain \
|
||||
--build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--output=type=cacheonly \
|
||||
-f Dockerfile.lint .
|
||||
fi
|
||||
docker build --no-cache \
|
||||
--target lint \
|
||||
-t "$("$SCRIPT_DIR/projectname")-lint" .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/bin/sh
|
||||
# script/loadtest: measure pixad's throughput, latency and peak memory.
|
||||
#
|
||||
# script/loadtest [duration [clients]] (defaults: 10s and 4)
|
||||
#
|
||||
# A benchmark, not a check: script/check does not run it. It needs Docker
|
||||
# and Go. It builds the image with script/docker and builds vegeta, the
|
||||
# load tool, from a pinned commit. Each scenario then gets a new pixad
|
||||
# container and a new origin container (cmd/loadtest-origin, which answers
|
||||
# every path with the same JPEG), and vegeta sends requests for <duration>
|
||||
# from <clients> clients at once, each asking for an image resized to
|
||||
# 400x300 WebP:
|
||||
#
|
||||
# hit the same image every time, put in the cache first
|
||||
# miss a new source image every time
|
||||
# herd each new source image once per client in a row, so that all
|
||||
# clients ask for it at the same time
|
||||
#
|
||||
# For each, it prints vegeta's report, pixad's peak resident memory and
|
||||
# how many requests reached the origin. README.md says how to read them.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
# vegeta v12.13.0, 2026-10-04
|
||||
VEGETA_COMMIT=4b240c3089fa4aa10816542d64a74294d974211f
|
||||
|
||||
# pixad refuses upstream hosts with private or local addresses, so the
|
||||
# containers share a network in 203.0.113.0/24, a range set aside for
|
||||
# documentation (RFC 5737) that pixad does not refuse and that is never
|
||||
# routed on the internet.
|
||||
SUBNET=203.0.113.0/24
|
||||
|
||||
usage() {
|
||||
echo "usage: script/loadtest [duration [clients]]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
main() {
|
||||
duration="${1:-10s}"
|
||||
clients="${2:-4}"
|
||||
# The duration is a whole number, not zero (vegeta takes 0 to mean no
|
||||
# end), followed by ms, s, m or h.
|
||||
case "$duration" in
|
||||
*ms) number="${duration%ms}" ;;
|
||||
*s | *m | *h) number="${duration%?}" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
case "$number" in
|
||||
"" | *[!0-9]*) usage ;;
|
||||
esac
|
||||
[ "$number" -gt 0 ] || usage
|
||||
# The number of clients is a whole number that does not start with 0,
|
||||
# which also refuses zero: vegeta reads a leading 0 as octal.
|
||||
case "$clients" in
|
||||
*[!0-9]* | 0*) usage ;;
|
||||
esac
|
||||
|
||||
cd "$ROOT"
|
||||
run="pixa-loadtest-$$"
|
||||
tmp="$(mktemp -d)"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 1' HUP INT TERM
|
||||
|
||||
"$SCRIPT_DIR/docker"
|
||||
# The image's ID, so a build elsewhere that moves the tag does not
|
||||
# change what a later scenario starts.
|
||||
image="$(docker image inspect --format '{{.Id}}' \
|
||||
"$("$SCRIPT_DIR/projectname")")"
|
||||
|
||||
GOBIN="$tmp" go install "github.com/tsenart/vegeta/v12@$VEGETA_COMMIT"
|
||||
# The origin runs in a container, so it is built for the Docker host.
|
||||
CGO_ENABLED=0 GOOS=linux \
|
||||
GOARCH="$(docker version --format '{{.Server.Arch}}')" \
|
||||
go build -o "$tmp/loadtest-origin" ./cmd/loadtest-origin
|
||||
|
||||
docker network create --subnet "$SUBNET" "$run" >/dev/null
|
||||
|
||||
start_containers
|
||||
# Put the image the hit scenario asks for in the cache.
|
||||
docker exec "$run-pixad" wget -q -O /dev/null \
|
||||
"http://localhost:8080/v1/image/origin/hit.jpg/400x300.webp"
|
||||
attack hit hit_targets
|
||||
stop_containers
|
||||
|
||||
start_containers
|
||||
attack miss miss_targets
|
||||
stop_containers
|
||||
|
||||
start_containers
|
||||
attack herd herd_targets
|
||||
stop_containers
|
||||
}
|
||||
|
||||
# start_containers starts a new origin and a new pixad, and waits up to 30
|
||||
# seconds for pixad's health check to pass.
|
||||
start_containers() {
|
||||
docker run -d --name "$run-origin" \
|
||||
--network "$run" --network-alias origin \
|
||||
-v "$tmp/loadtest-origin:/usr/local/bin/loadtest-origin:ro" \
|
||||
--entrypoint /usr/local/bin/loadtest-origin "$image" >/dev/null
|
||||
docker run -d --name "$run-pixad" \
|
||||
--network "$run" -p 127.0.0.1::8080 --health-interval=1s \
|
||||
-e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \
|
||||
-e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \
|
||||
"$image" >/dev/null
|
||||
|
||||
waited=0
|
||||
until [ "$(docker inspect --format '{{.State.Health.Status}}' \
|
||||
"$run-pixad")" = healthy ]; do
|
||||
if [ "$waited" -ge 30 ]; then
|
||||
echo "loadtest: pixad not healthy after 30 seconds; its log:" >&2
|
||||
docker logs "$run-pixad" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
pixa="http://$(docker port "$run-pixad" 8080/tcp)"
|
||||
}
|
||||
|
||||
stop_containers() {
|
||||
docker rm -f "$run-pixad" "$run-origin" >/dev/null
|
||||
}
|
||||
|
||||
# attack <scenario> <targets>: send the requests <targets> prints and
|
||||
# report on them.
|
||||
attack() {
|
||||
echo
|
||||
echo "== $1: $clients clients for $duration"
|
||||
"$2" | "$tmp/vegeta" attack -lazy -rate 0 -workers "$clients" \
|
||||
-max-workers "$clients" -duration "$duration" -max-body 0 |
|
||||
"$tmp/vegeta" report
|
||||
# pixad is process 1 in its container: the entrypoint execs it.
|
||||
echo "pixad peak memory (VmHWM):" \
|
||||
"$(docker exec "$run-pixad" awk '/^VmHWM:/ { print $2, $3 }' \
|
||||
/proc/1/status)"
|
||||
echo "requests to the origin:" \
|
||||
"$(docker logs "$run-origin" 2>&1 | grep -c ' request ')"
|
||||
}
|
||||
|
||||
# The targets functions print vegeta targets until vegeta stops reading.
|
||||
|
||||
hit_targets() {
|
||||
while :; do
|
||||
echo "GET $pixa/v1/image/origin/hit.jpg/400x300.webp"
|
||||
done
|
||||
}
|
||||
|
||||
miss_targets() {
|
||||
i=0
|
||||
while :; do
|
||||
i=$((i + 1))
|
||||
echo "GET $pixa/v1/image/origin/miss/$i.jpg/400x300.webp"
|
||||
done
|
||||
}
|
||||
|
||||
herd_targets() {
|
||||
i=0
|
||||
while :; do
|
||||
i=$((i + 1))
|
||||
n=0
|
||||
while [ "$n" -lt "$clients" ]; do
|
||||
n=$((n + 1))
|
||||
echo "GET $pixa/v1/image/origin/herd/$i.jpg/400x300.webp"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$run-pixad" "$run-origin" >/dev/null 2>&1 || :
|
||||
docker network rm "$run" >/dev/null 2>&1 || :
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+1
-2
@@ -1,7 +1,6 @@
|
||||
#!/bin/sh
|
||||
# script/setup: set up the repo for development after a fresh clone:
|
||||
# installs dependencies (script/bootstrap) and the git pre-commit hook.
|
||||
# Add any repo-specific initialization (db init, .env template) here.
|
||||
# installs dependencies and the git pre-commit hook.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
+10
-18
@@ -1,27 +1,19 @@
|
||||
#!/bin/sh
|
||||
# script/test: run the test suite. CGO dependencies (pkg-config, vips,
|
||||
# libheif) come from nix-shell when not already available (e.g. inside
|
||||
# a Docker build or an existing nix-shell).
|
||||
# script/test: run the test suite. Testing is a phase of the Dockerfile
|
||||
# and this builds that phase alone, on the same terms as script/lint:
|
||||
# --target because a phase that is not the last stage is built only when
|
||||
# named, --no-cache because a cached test layer is a test that did not
|
||||
# run, and a tag so each build replaces the previous image.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
run_with_cgo_deps() {
|
||||
if command -v pkg-config >/dev/null 2>&1; then
|
||||
sh -c "$1"
|
||||
else
|
||||
nix-shell -p pkg-config vips libheif git --run "$1"
|
||||
fi
|
||||
}
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
echo "Running tests..."
|
||||
# Run without -v first for clean output on success; on failure rerun
|
||||
# with -v for full diagnostics, then exit non-zero (REPO_POLICIES.md
|
||||
# conditional-verbose-rerun pattern). The first run already proved the
|
||||
# tests broken, so the build fails even if the rerun happens to pass.
|
||||
run_with_cgo_deps "CGO_ENABLED=1 go test -timeout 30s -race -cover ./... || { echo '--- Rerunning with -v for details ---'; CGO_ENABLED=1 go test -timeout 30s -race -v ./...; exit 1; }"
|
||||
docker build --no-cache \
|
||||
--target test \
|
||||
-t "$("$SCRIPT_DIR/projectname")-test" .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
prettier@3.8.1:
|
||||
version "3.8.1"
|
||||
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173"
|
||||
integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==
|
||||
Reference in New Issue
Block a user