Compare commits
1
Commits
next
..
08f8c25349
| Author | SHA1 | Date | |
|---|---|---|---|
 clawbot
|
08f8c25349 |
+7
-81
@@ -1,82 +1,8 @@
|
||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
||||
# `certs/server.key` still ship while this file reads as solved. Only
|
||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
||||
# into .gitignore, where `**/` is wrong.
|
||||
#
|
||||
# Matching is case-sensitive, so secrets use character ranges rather
|
||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
||||
#
|
||||
# Extend with this repo's own host-built artifacts, written anchored:
|
||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
||||
# deletes the package directory from the context.
|
||||
|
||||
# .git is sent without its config. Without a VERSION build argument the
|
||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||
# does not need .git/config; that file can hold a credential, such as a
|
||||
# password in a remote URL or the token the CI checkout step stores there.
|
||||
# Each submodule keeps a config with the same exposure in its git directory
|
||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
||||
# its own .git directory when it keeps one.
|
||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
||||
# `**/.git/modules/**/config` also matches that segment's directory
|
||||
# under .git/modules/. Go's version stamping then fails the build;
|
||||
# nothing leaks. Name such a submodule without that segment:
|
||||
# `git submodule add --name`.
|
||||
**/.git/config
|
||||
**/.git/modules/**/config
|
||||
|
||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
||||
# Anchored because it occurs once where agents run at the repo root.
|
||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
||||
# `services/api/.claude/` and must add its own anchored entry.
|
||||
.git
|
||||
.gitignore
|
||||
.DS_Store
|
||||
.env*
|
||||
.claude
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Re-include a committed template with a negation if the
|
||||
# build needs one: `!docs/example.env`.
|
||||
**/*.[eE][nN][vV]
|
||||
**/.[eE][nN][vV].*
|
||||
**/.[eE][nN][vV][rR][cC]
|
||||
|
||||
# Private keys and the bundles carrying them. Public certificates
|
||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
||||
**/*.[pP][eE][mM]
|
||||
**/*.[kK][eE][yY]
|
||||
**/*.[pP]12
|
||||
**/*.[pP][fF][xX]
|
||||
**/[iI][dD]_[rR][sS][aA]
|
||||
**/[iI][dD]_[dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
**/[iI][dD]_[eE][dD]25519
|
||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# Dependencies: restored inside the image, never copied in.
|
||||
**/node_modules
|
||||
|
||||
# OS metadata.
|
||||
**/.DS_Store
|
||||
**/Thumbs.db
|
||||
|
||||
# Editor state: never a build input, and it churns COPY.
|
||||
**/*.swp
|
||||
**/*.swo
|
||||
**/*~
|
||||
**/*.bak
|
||||
**/.idea
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# pixa's own entries. On the host, `make build` writes bin/pixad, and the
|
||||
# example config keeps its state directory in data/.
|
||||
/bin
|
||||
/data
|
||||
|
||||
# Local config files, kept out of git because they can hold the signing key.
|
||||
**/[cC][oO][nN][fF][iI][gG].[yY][mM][lL]
|
||||
**/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL]
|
||||
**/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL]
|
||||
node_modules
|
||||
bin/
|
||||
data/
|
||||
|
||||
@@ -13,6 +13,3 @@ indent_style = tab
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
|
||||
# This repository's own sections, such as one for another language it
|
||||
# uses, go below this comment, and a re-vendor keeps them.
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
# Every PR adds an entry at the top of TODO.md's Completed Steps; union keeps
|
||||
# both sides instead of conflicting. Git never reports a conflict here: read
|
||||
# the merged entries after every merge or rebase.
|
||||
TODO.md merge=union
|
||||
@@ -1,20 +1,10 @@
|
||||
name: check
|
||||
on: [push]
|
||||
# Free the shared runner: a new push cancels only the same branch's older run.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Free the shared runner from a hung build.
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
# actions/checkout v4.2.2, 2026-02-22
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
# script/cibuild needs no token, so none is left in .git/config.
|
||||
with:
|
||||
persist-credentials: false
|
||||
# All history and tags, so git describe finds the version tag.
|
||||
fetch-depth: 0
|
||||
- run: script/cibuild
|
||||
- run: script/docker-smoke
|
||||
|
||||
+6
-44
@@ -11,52 +11,15 @@ Thumbs.db
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
||||
# entry and must not be given a `**/` prefix on the way into one.
|
||||
.claude/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
|
||||
# Secrets. Unanchored like every entry above, so each matches at every
|
||||
# depth. Matching is case-sensitive on Linux, so names use character
|
||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Only the templates `example.env` and `sample.env` are
|
||||
# re-included below. A repository that commits any other template adds
|
||||
# its own negation at the end of this file, for example `!.env.example`.
|
||||
*.[eE][nN][vV]
|
||||
.[eE][nN][vV].*
|
||||
.[eE][nN][vV][rR][cC]
|
||||
!example.env
|
||||
!sample.env
|
||||
|
||||
# Private keys and the bundles carrying them.
|
||||
*.[pP][eE][mM]
|
||||
*.[kK][eE][yY]
|
||||
*.[pP]12
|
||||
*.[pP][fF][xX]
|
||||
[iI][dD]_[rR][sS][aA]
|
||||
[iI][dD]_[dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
[iI][dD]_[eE][dD]25519
|
||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# This repository's own entries, such as its build outputs, go below
|
||||
# this comment, and a re-vendor keeps them. Anchor a binary built at the
|
||||
# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
|
||||
|
||||
# Go
|
||||
*.log
|
||||
*.out
|
||||
*.test
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
# Dependencies
|
||||
vendor/
|
||||
node_modules/
|
||||
|
||||
# Build output
|
||||
/bin/
|
||||
@@ -68,6 +31,5 @@ vendor/
|
||||
*.sqlite3
|
||||
|
||||
# Local dev configs
|
||||
config.yml
|
||||
config.yaml
|
||||
config.dev.yml
|
||||
|
||||
+2
-69
@@ -10,23 +10,14 @@ run:
|
||||
|
||||
linters:
|
||||
default: all
|
||||
enable:
|
||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
||||
# left to `default: all`, because it carries the module policy below.
|
||||
- gomodguard_v2
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
# Deprecated: the warning is attached to the old name, so it is
|
||||
# silenced by disabling that name, not by enabling the successor.
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||
# Misses findings at random in v2.14.0; back once a pinned release fixes it
|
||||
- canonicalheader
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
@@ -37,64 +28,6 @@ linters:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
depguard:
|
||||
# Test-support code must not be compiled into the shipped binary. A
|
||||
# test-support package exists to hand a test privileges the program
|
||||
# itself must never have, so a file that is not a test must not import
|
||||
# one. Test files, and the files inside a package whose directory name
|
||||
# ends in `test`, are where that code belongs, and are exempt.
|
||||
#
|
||||
# The deny list below is the one part of this file a repository is
|
||||
# expected to extend, and the only part it may. depguard matches an
|
||||
# import path against a list of prefixes, so it cannot be told "any path
|
||||
# whose last segment ends in test"; a repository's own test-support
|
||||
# packages have to be named here one at a time, by full import path,
|
||||
# under a module path that differs from repository to repository. Add
|
||||
# them; change nothing else.
|
||||
rules:
|
||||
test-support:
|
||||
list-mode: lax
|
||||
files:
|
||||
- "$all"
|
||||
- "!$test"
|
||||
- "!**/*test/**"
|
||||
deny:
|
||||
- pkg: net/http/httptest
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
# Only decisions already recorded in the Go package defaults are
|
||||
# listed here. Every entry matches the module path exactly.
|
||||
gomodguard_v2:
|
||||
blocked:
|
||||
- module: github.com/rs/zerolog
|
||||
recommendations:
|
||||
- log/slog
|
||||
reason: "Structured logging is stdlib log/slog."
|
||||
# One entry per pre-fork module path, because the later releases
|
||||
# are separate paths. A prefix match would be shorter but would
|
||||
# also reach github.com/go-redis/redismock, the test double for
|
||||
# the successor these entries recommend.
|
||||
- module: github.com/go-redis/redis
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v7
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v8
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/sergi/go-diff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "No unified diff output; use go-udiff."
|
||||
- module: github.com/hexops/gotextdiff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "Unmaintained fork; use go-udiff."
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
node_modules/
|
||||
yarn.lock
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"tabWidth": 4,
|
||||
"proseWrap": "always"
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
# Repository Rules
|
||||
|
||||
Last Updated 2026-01-08
|
||||
|
||||
These rules MUST be followed at all times, it is very important.
|
||||
|
||||
- Never use `git add -A` - add specific changes to a deliberate commit. A commit
|
||||
should contain one change. After each change, make a commit with a good
|
||||
one-line summary.
|
||||
|
||||
- NEVER modify the linter config without asking first.
|
||||
|
||||
- NEVER modify tests to exclude special cases or otherwise get them to pass
|
||||
without asking first. In almost all cases, the code should be changed, NOT the
|
||||
tests. If you think the test needs to be changed, make your case for that and
|
||||
ask for permission to proceed, then stop. You need explicit user approval to
|
||||
modify existing tests. A bug fix may change an existing test without asking
|
||||
when that test pins exactly the behavior the filed bug calls wrong, and the
|
||||
change moves it to the fixed behavior and nothing else; any other change to an
|
||||
existing test still needs explicit user approval. (You do not need user
|
||||
approval for writing NEW tests.)
|
||||
|
||||
- When linting, assume the linter config is CORRECT, and that each item output
|
||||
by the linter is something that legitimately needs fixing in the code.
|
||||
|
||||
- When running tests, use `make test`.
|
||||
|
||||
- Before commits, run `make check`. This runs `make lint` and `make test` and
|
||||
`make check-fmt`. Any issues discovered MUST be resolved before committing
|
||||
unless explicitly told otherwise.
|
||||
|
||||
- When fixing a bug, write a failing test for the bug FIRST. Add appropriate
|
||||
logging to the test to ensure it is written correctly. Commit that. Then go
|
||||
about fixing the bug until the test passes (without modifying the test
|
||||
further). Then commit that.
|
||||
|
||||
- When adding a new feature, do the same - implement a test first (TDD). It
|
||||
doesn't have to be super complex. Commit the test, then commit the feature.
|
||||
|
||||
- When adding a new feature, use a feature branch. When the feature is
|
||||
completely finished and the code is up to standards (passes `make check`) then
|
||||
and only then can the feature branch be merged into `main` and the branch
|
||||
deleted.
|
||||
|
||||
- Write godoc documentation comments for all exported types and functions as you
|
||||
go along.
|
||||
|
||||
- ALWAYS be consistent in naming. If you name something one thing in one place,
|
||||
name it the EXACT SAME THING in another place.
|
||||
|
||||
- Be descriptive and specific in naming. `wl` is bad; `SourceHostWhitelist` is
|
||||
good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good.
|
||||
|
||||
- This is not prototype or teaching code - this is designed for production. Any
|
||||
security issues (such as denial of service) or other web vulnerabilities are
|
||||
P1 bugs and must be added to TODO.md at the top.
|
||||
|
||||
- As this is production code, no stubbing of implementations unless specifically
|
||||
instructed. We need working implementations.
|
||||
|
||||
- NEVER silently fall back to a different setting when a user's parameter
|
||||
explicitly specifies a value. If a user requests format=webp and WebP encoding
|
||||
is not supported, return an error - do NOT silently output PNG instead. If a
|
||||
user specifies fit=invalid and that fit mode doesn't exist, return an error -
|
||||
do NOT silently default to "cover". Silent fallbacks violate the principle of
|
||||
least surprise and mask bugs. The only acceptable defaults are for OMITTED
|
||||
parameters, never for INVALID explicit values.
|
||||
|
||||
- Avoid vendoring deps unless specifically instructed to. NEVER commit the
|
||||
vendor directory, NEVER commit compiled binaries. If these directories or
|
||||
files exist, add them to .gitignore (and commit the .gitignore) if they are
|
||||
not already in there. Keep the entire git repository (with history) small -
|
||||
under 20MiB, unless you specifically must commit larger files (e.g. test
|
||||
fixture example media files). Only OUR source code and immediately supporting
|
||||
files (such as test examples) goes into the repo/history.
|
||||
@@ -0,0 +1,76 @@
|
||||
# Repository Rules
|
||||
|
||||
Last Updated 2026-01-08
|
||||
|
||||
These rules MUST be followed at all times, it is very important.
|
||||
|
||||
* Never use `git add -A` - add specific changes to a deliberate commit. A
|
||||
commit should contain one change. After each change, make a commit with a
|
||||
good one-line summary.
|
||||
|
||||
* NEVER modify the linter config without asking first.
|
||||
|
||||
* NEVER modify tests to exclude special cases or otherwise get them to pass
|
||||
without asking first. In almost all cases, the code should be changed,
|
||||
NOT the tests. If you think the test needs to be changed, make your case
|
||||
for that and ask for permission to proceed, then stop. You need explicit
|
||||
user approval to modify existing tests. (You do not need user approval
|
||||
for writing NEW tests.)
|
||||
|
||||
* When linting, assume the linter config is CORRECT, and that each item
|
||||
output by the linter is something that legitimately needs fixing in the
|
||||
code.
|
||||
|
||||
* When running tests, use `make test`.
|
||||
|
||||
* Before commits, run `make check`. This runs `make lint` and `make test`
|
||||
and `make check-fmt`. Any issues discovered MUST be resolved before
|
||||
committing unless explicitly told otherwise.
|
||||
|
||||
* When fixing a bug, write a failing test for the bug FIRST. Add
|
||||
appropriate logging to the test to ensure it is written correctly. Commit
|
||||
that. Then go about fixing the bug until the test passes (without
|
||||
modifying the test further). Then commit that.
|
||||
|
||||
* When adding a new feature, do the same - implement a test first (TDD). It
|
||||
doesn't have to be super complex. Commit the test, then commit the
|
||||
feature.
|
||||
|
||||
* When adding a new feature, use a feature branch. When the feature is
|
||||
completely finished and the code is up to standards (passes `make check`)
|
||||
then and only then can the feature branch be merged into `main` and the
|
||||
branch deleted.
|
||||
|
||||
* Write godoc documentation comments for all exported types and functions as
|
||||
you go along.
|
||||
|
||||
* ALWAYS be consistent in naming. If you name something one thing in one
|
||||
place, name it the EXACT SAME THING in another place.
|
||||
|
||||
* Be descriptive and specific in naming. `wl` is bad;
|
||||
`SourceHostWhitelist` is good. `ConnsPerHost` is bad;
|
||||
`MaxConnectionsPerHost` is good.
|
||||
|
||||
* This is not prototype or teaching code - this is designed for production.
|
||||
Any security issues (such as denial of service) or other web
|
||||
vulnerabilities are P1 bugs and must be added to TODO.md at the top.
|
||||
|
||||
* As this is production code, no stubbing of implementations unless
|
||||
specifically instructed. We need working implementations.
|
||||
|
||||
* NEVER silently fall back to a different setting when a user's parameter
|
||||
explicitly specifies a value. If a user requests format=webp and WebP
|
||||
encoding is not supported, return an error - do NOT silently output PNG
|
||||
instead. If a user specifies fit=invalid and that fit mode doesn't exist,
|
||||
return an error - do NOT silently default to "cover". Silent fallbacks
|
||||
violate the principle of least surprise and mask bugs. The only acceptable
|
||||
defaults are for OMITTED parameters, never for INVALID explicit values.
|
||||
|
||||
* Avoid vendoring deps unless specifically instructed to. NEVER commit
|
||||
the vendor directory, NEVER commit compiled binaries. If these
|
||||
directories or files exist, add them to .gitignore (and commit the
|
||||
.gitignore) if they are not already in there. Keep the entire git
|
||||
repository (with history) small - under 20MiB, unless you specifically
|
||||
must commit larger files (e.g. test fixture example media files). Only
|
||||
OUR source code and immediately supporting files (such as test examples)
|
||||
goes into the repo/history.
|
||||
+1267
File diff suppressed because it is too large
Load Diff
+49
-109
@@ -1,127 +1,67 @@
|
||||
# In every stage ARG CACHEBUST comes right after the steps that install
|
||||
# system packages and download Go's modules, or right after FROM in a
|
||||
# stage with neither, and every step that lints, tests or compiles
|
||||
# comes after it. script/ passes a new value on every build, which
|
||||
# reruns every RUN step after it. Docker keeps the layers above it
|
||||
# until the base image, their own lines or go.mod and go.sum change.
|
||||
# The steps that compile mount Go's build cache, and golangci-lint run
|
||||
# also mounts golangci-lint's; both survive from build to build.
|
||||
|
||||
# Lint phase. script/lint builds it alone. The linter is run directly:
|
||||
# `make lint` and script/lint are themselves a docker build.
|
||||
# golangci/golangci-lint:v2.14.0, 2026-10-15
|
||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||
|
||||
# The linter compiles every package, and govips needs the libvips
|
||||
# headers for that. REPO_POLICIES.md has the lint phase install them
|
||||
# itself; this image is Debian, so with apt-get rather than apk.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Lint stage
|
||||
# Same image as Dockerfile.lint: change both pins together.
|
||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# script/bootstrap installs the build dependencies and downloads the Go
|
||||
# modules. Only script/, go.mod and go.sum are copied first, so this
|
||||
# layer is reused until one of them changes.
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/root/.cache/golangci-lint \
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. script/test builds it alone. -race needs cgo and so a C
|
||||
# compiler, which the Debian Go image ships and the alpine one does not.
|
||||
# golang:1.25.4-trixie, 2026-10-15
|
||||
FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test
|
||||
|
||||
# govips compiles against the libvips and libheif headers, and the tests
|
||||
# need libvips' JPEG XL support; libvips-dev brings all three. The tests
|
||||
# also encode AVIF, which needs libheif's AVIF encoder,
|
||||
# libheif-plugin-aomenc: Debian only recommends it, so it is named here.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev libheif-plugin-aomenc \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
COPY . .
|
||||
|
||||
# Without -v first; on a failure, again with -v for the details, and
|
||||
# the step fails even if the second run passes. -count=1 because the
|
||||
# mounted cache also holds test results, which go test would otherwise
|
||||
# replay. -parallel 4: by default a package runs as many of its tests at
|
||||
# once as the host has CPUs, and on a busy host they then wait so long
|
||||
# to be scheduled that a test that times a request can see it return a
|
||||
# second late.
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from the two phases above: these copies
|
||||
# make BuildKit build them first, so nothing after them runs unless lint
|
||||
# and test passed.
|
||||
# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
|
||||
FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||
|
||||
# git for the version, and for govips a C compiler, pkg-config and the
|
||||
# libvips and libheif headers
|
||||
RUN apk add --no-cache git build-base pkgconfig vips-dev libheif-dev
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
RUN script/bootstrap
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context: the
|
||||
# tag on a tagged commit, tag-N-gHASH after one, the short commit when
|
||||
# no tag is reachable. With .git present, a version that is still empty,
|
||||
# dev or unknown fails the build: git is missing or could not read the
|
||||
# checkout. A context without .git, as from a source tarball, stamps an
|
||||
# empty version. CGO stays enabled for govips; -trimpath keeps build
|
||||
# paths out of the binary, and -s -w leave out the symbol table and
|
||||
# debug information.
|
||||
ARG VERSION
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /pixad ./cmd/pixad
|
||||
# Tells script/lint it is inside a container, so it runs the linter.
|
||||
ENV container=docker
|
||||
|
||||
# Runtime stage, and the last one: a plain `docker build .` builds this
|
||||
# stage and what it depends on, and nothing else. It must use the Alpine
|
||||
# release the golang image above is based on, so that pixad runs against
|
||||
# the libvips and musl it was built with.
|
||||
# alpine:3.22, 2026-10-08
|
||||
FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8
|
||||
# Run formatting check and linter
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
|
||||
# support, without which pixad does not start.
|
||||
# Build stage
|
||||
# golang:1.25.4-alpine, 2026-02-25
|
||||
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||
|
||||
# Depend on lint stage passing
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Build dependencies and Go modules, as in the lint stage
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN script/bootstrap
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Run tests
|
||||
RUN make test
|
||||
|
||||
# VERSION is declared here, not earlier: a new value reruns only the
|
||||
# build, not script/bootstrap or the tests. CGO stays enabled for
|
||||
# govips; -trimpath keeps build paths out of the binary, and -s -w
|
||||
# leave out the symbol table and debug information.
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \
|
||||
-ldflags "-s -w -X main.Version=${VERSION}" \
|
||||
-o /pixad ./cmd/pixad
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-02-25
|
||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||
|
||||
# Install runtime dependencies only
|
||||
RUN apk add --no-cache \
|
||||
vips \
|
||||
vips-jxl \
|
||||
libheif \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
su-exec
|
||||
ARG CACHEBUST
|
||||
|
||||
# Copy binary from builder
|
||||
COPY --from=builder /pixad /usr/local/bin/pixad
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# Dockerfile.lint: the container script/lint builds to run golangci-lint,
|
||||
# which is never installed on the host. Pinned to the same image as the
|
||||
# Dockerfile lint stage: change both pins together, or the two run
|
||||
# different linter versions.
|
||||
#
|
||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# pixa is CGO/libvips: the type-aware linters compile every package, so
|
||||
# this image needs the same C libraries the build does. script/bootstrap
|
||||
# installs them and downloads the Go modules. Only script/, go.mod and
|
||||
# go.sum are copied first; they settle this layer's result, so it may
|
||||
# safely be reused between runs.
|
||||
COPY script/ ./script/
|
||||
COPY go.mod go.sum ./
|
||||
RUN script/bootstrap
|
||||
|
||||
COPY . .
|
||||
|
||||
# Tells script/lint it is inside a container, so it runs the linter.
|
||||
ENV container=docker
|
||||
|
||||
# script/lint passes a different CACHEBUST on every run, and BuildKit
|
||||
# keys every RUN after this ARG on its value, so the lint step always
|
||||
# runs instead of returning a cached success that linted nothing.
|
||||
#
|
||||
# Go's and golangci-lint's caches (/root/.cache, hundreds of MB) go on a
|
||||
# tmpfs that is discarded after the step. Written into the layer, they
|
||||
# would pile up as build cache on every run, since no later run, with
|
||||
# its new CACHEBUST, can reuse that layer.
|
||||
ARG CACHEBUST
|
||||
RUN --mount=type=tmpfs,target=/root/.cache script/lint
|
||||
@@ -1,10 +1,10 @@
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest bench
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks
|
||||
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
LDFLAGS := -X main.Version=$(VERSION)
|
||||
|
||||
# Use nix-shell to provide CGO dependencies unless they are already available
|
||||
# (e.g. inside an existing nix-shell).
|
||||
# (e.g. inside a Docker build or an existing nix-shell).
|
||||
HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null)
|
||||
ifdef HAS_PKGCONFIG
|
||||
NIX_RUN_PREFIX =
|
||||
@@ -32,11 +32,11 @@ fmt-check:
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
# Run linter (the lint phase of the Dockerfile)
|
||||
# Run linter
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
# Run tests (the test phase of the Dockerfile)
|
||||
# Run tests (30-second timeout)
|
||||
test:
|
||||
@script/test
|
||||
|
||||
@@ -59,31 +59,20 @@ docker:
|
||||
docker-smoke:
|
||||
@script/docker-smoke
|
||||
|
||||
# Measure throughput, latency and peak memory with the default duration and
|
||||
# number of clients (needs Docker and Go; a benchmark, not part of check)
|
||||
loadtest:
|
||||
@script/loadtest
|
||||
|
||||
# Serve cache hits in parallel through the image handler and report hits per
|
||||
# second (needs Go on the host after `script/bootstrap --cgo`, not Docker; a
|
||||
# benchmark, not part of check)
|
||||
bench:
|
||||
@script/bench
|
||||
|
||||
# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION)
|
||||
# Build Docker image tagged pixad:$(VERSION) and pixad:latest
|
||||
docker-versioned:
|
||||
@script/docker
|
||||
docker tag pixa pixa:$(VERSION)
|
||||
docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
|
||||
|
||||
# Run tests in Docker, as `make test` does
|
||||
# Run tests in Docker (needed for CGO/libvips)
|
||||
docker-test:
|
||||
@script/test
|
||||
docker build --target builder --build-arg VERSION=$(VERSION) -t pixad-builder .
|
||||
docker run --rm pixad-builder sh -c "CGO_ENABLED=1 GOTOOLCHAIN=auto go test -v ./..."
|
||||
|
||||
# Run local dev server in Docker
|
||||
devserver: docker-versioned devserver-stop
|
||||
docker run -d --name pixad-dev -p 8080:8080 \
|
||||
-v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
|
||||
pixa:latest
|
||||
pixad:latest
|
||||
@echo "pixad running at http://localhost:8080"
|
||||
|
||||
# Stop dev server
|
||||
|
||||
+88
-469
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-10-18
|
||||
last_modified: 2026-07-06
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
@@ -54,37 +54,21 @@ style conventions are in separate documents:
|
||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
||||
(detected in that order; apt runs noninteractive). For node it uses the
|
||||
installed node only when its major version is the pinned one; otherwise (node
|
||||
missing, or another major, such as the node 24 the CI runner image ships) it
|
||||
installs the PINNED node version via nvm and installs yarn and the packages
|
||||
under it. nvm itself is installed first if missing, from a hash-verified
|
||||
GitHub release archive (never `curl | sh`), with bash installed as an explicit
|
||||
prerequisite since nvm requires bash. Only the major version is compared
|
||||
because the `Dockerfile` stages start from a node image whose exact version is
|
||||
not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
|
||||
via `corepack prepare yarn@<version> --activate`. Never install "latest" or
|
||||
"lts"; always exact versions. `script/cibuild` runs the CI build: it changes
|
||||
to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
|
||||
image with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
||||
pristine checkout with nothing installed the run dies there, after the
|
||||
containerised gates have passed. **The bootstrap alone is not enough**: when
|
||||
`script/bootstrap` installs node and yarn under nvm it leaves neither on the
|
||||
`PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
|
||||
under another node. The host entrypoints that need yarn — `script/fmt` and
|
||||
`script/fmt-check` — therefore use the same test as `script/bootstrap`: when
|
||||
the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
|
||||
and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
|
||||
pinned node. A runner carrying nothing but docker and git then gets through
|
||||
`script/check`. Four further scripts are our own extensions to the standard:
|
||||
`script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
|
||||
`script/precommit` is what the git pre-commit hook runs, and it calls
|
||||
`script/check`; `script/install-precommit` installs the git pre-commit hook
|
||||
(the `make hooks` target shims to it); and `script/projectname` (literally
|
||||
that filename) simply outputs the project's name. Scripts that need the name
|
||||
call `script/projectname` — e.g. `script/docker` assembles its image tag from
|
||||
it — so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||
installed node if present; otherwise it installs a PINNED node version via
|
||||
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
||||
release archive (never `curl | sh`), with bash installed as an explicit
|
||||
prerequisite since nvm requires bash. yarn is then pinned via
|
||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||
scripts are our own extensions to the standard: `script/check` runs
|
||||
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||
what the git pre-commit hook runs, and it calls `script/check`;
|
||||
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||
target shims to it); and `script/projectname` (literally that filename) simply
|
||||
outputs the project's name. Scripts that need the name call
|
||||
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||
`script/precommit`, not in the hook itself. Model scripts are at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||
@@ -105,290 +89,87 @@ style conventions are in separate documents:
|
||||
contributor should be able to understand the entire development workflow by
|
||||
reading the Makefile.
|
||||
|
||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
||||
image cannot be built unless they pass. For non-server repos the final stage
|
||||
brings up a development environment; for server repos it is the runtime image.
|
||||
The gate phases and the build stage start from their pinned base images and
|
||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
||||
repo's own `Dockerfile` does for its yarn packages. The development
|
||||
environment stage installs development prerequisites by running
|
||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||
as a build step so the build fails if the branch is not green. For non-server
|
||||
repos, the Dockerfile should bring up a development environment and run
|
||||
`make check`. For server repos, `make check` should run as an early build
|
||||
stage before the final image is assembled. Dockerfiles install development
|
||||
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||
layer stays cached until dependencies change.
|
||||
|
||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
||||
and nothing else:
|
||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||
repos use a multistage build where linting runs in an independent stage based
|
||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||
then declares an explicit dependency on the lint stage via
|
||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||
linting before proceeding to compilation and tests. This ensures lint failures
|
||||
surface in seconds rather than minutes, without blocking on dependency
|
||||
download or compilation in the build stage.
|
||||
|
||||
```sh
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--target lint --output type=cacheonly .
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--target test --output type=cacheonly .
|
||||
```
|
||||
|
||||
**A stage that is not the last one in the file is built only when the final
|
||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
||||
two gates are always invoked by name here, and why the final stage carries a
|
||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
||||
and tested nothing.
|
||||
|
||||
**The gate builds write no image.** With `--output type=cacheonly` the phase
|
||||
runs and a failing step fails the build, but the result is not exported.
|
||||
Nothing uses those images, and writing one out is slow: a Go test phase's
|
||||
image holds the toolchain. A build given neither `--output` nor `-t` writes
|
||||
an untagged image and leaves it dangling, on every developer host and every
|
||||
CI runner. `script/cibuild` and `script/docker` build the image that ships
|
||||
and tag it, so each build replaces the previous image; each assigns the tag
|
||||
on its own line before the build, so `set -e` stops it where
|
||||
`script/projectname` fails.
|
||||
|
||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
||||
themselves a `docker build` and would recurse into a daemon that does not
|
||||
exist in a build step. Formatting is the exception and stays on the host:
|
||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
||||
read-only twin.
|
||||
|
||||
**No lint verdict may come from a host invocation of the linter.** On a
|
||||
shared host golangci-lint reads a result cache keyed on file content rather
|
||||
than location, so a second checkout of the same content is served the first
|
||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||
org, in both directions. A container has its own `TMPDIR`, so the lock is
|
||||
out of reach. The lint phase does keep golangci-lint's cache between check
|
||||
builds (below), but it lints every checkout at `/src`, and a finding is
|
||||
taken from that cache only for the same files, settings and golangci-lint
|
||||
version.
|
||||
|
||||
- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
|
||||
invalidates a `COPY` layer only when the copied content changes, so on an
|
||||
unchanged tree the check `RUN` is served from cache, nothing executes, and the
|
||||
build still exits 0. Every `docker build` in `script/` therefore passes
|
||||
`--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
|
||||
id, so no two builds share a value, not even the three in one
|
||||
`script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
|
||||
`script/docker` are the four, and there is no fifth — `script/check` runs the
|
||||
two gate phases and `script/fmt-check`, and builds no image of its own. In
|
||||
every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
|
||||
that install system packages and download Go's modules, or right after `FROM`
|
||||
in a stage with neither, and every step that lints, tests or compiles comes
|
||||
after it. A step that installs anything else as well, such as a
|
||||
`script/bootstrap` that also installs node packages, comes after it too. A new
|
||||
value reruns every `RUN` step after that line; `COPY` steps are compared by
|
||||
content and need nothing. Docker keeps the layers above the line until the
|
||||
base image, their own lines or the files they copy change. A stage without the
|
||||
`ARG` line is served from cache on an unchanged tree, so a repository
|
||||
re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
|
||||
not pass `--no-cache`: it reruns every step too, but it also builds the layers
|
||||
above the line again and empties every cache mount, so Go's build cache and
|
||||
golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
|
||||
survive between check builds, would start empty every time. A bare
|
||||
`docker build .` is not evidence that anything ran: a sub-second build
|
||||
reporting success is a cache hit, not a result. Never invalidate by pruning —
|
||||
`docker builder prune` and friends destroy a build cache shared with every
|
||||
other build on the host. When a check is added or changed, prove it works by
|
||||
planting a defect it must catch and watching the run fail on it, then revert
|
||||
the defect. A green run alone shows neither that the check ran nor that it
|
||||
covers what it should.
|
||||
|
||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||
hash), so lint failures surface in seconds rather than after a full compile,
|
||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
||||
`Dockerfile`:
|
||||
The standard pattern for a Go repo Dockerfile is:
|
||||
|
||||
```dockerfile
|
||||
# In every stage ARG CACHEBUST comes right after the steps that install
|
||||
# system packages and download Go's modules, or right after FROM in a
|
||||
# stage with neither, and every step that lints, tests or compiles
|
||||
# comes after it. script/ passes a new value on every build, which
|
||||
# reruns every RUN step after it. Docker keeps the layers above it
|
||||
# until the base image, their own lines or go.mod and go.sum change.
|
||||
# The steps that compile mount Go's build cache, and golangci-lint run
|
||||
# also mounts golangci-lint's; both survive from build to build.
|
||||
|
||||
# Lint phase
|
||||
# Lint stage — fast feedback on formatting and lint issues
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/root/.cache/golangci-lint \
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not. -count=1 because the mounted
|
||||
# cache also holds test results, which go test would otherwise replay.
|
||||
# golang:1.x, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so nothing after them can
|
||||
# run unless lint and test passed.
|
||||
# Build stage
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
RUN apk add --no-cache git
|
||||
WORKDIR /src
|
||||
|
||||
# Force BuildKit to run the lint stage before proceeding
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
COPY . .
|
||||
RUN make test
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context. With
|
||||
# .git present, a version that is still empty, dev or unknown fails the
|
||||
# build: git is missing or could not read the checkout.
|
||||
ARG VERSION
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage, and the last one
|
||||
# Runtime stage
|
||||
FROM alpine@sha256:...
|
||||
ARG CACHEBUST
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
|
||||
Key points:
|
||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
||||
both Go and the linter), so nothing needs installing.
|
||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
||||
and a stage nothing depends on is not built at all, so without these two
|
||||
lines a red gate would not fail the build.
|
||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||
same two copies. A plain `docker build .` builds the last stage's chain
|
||||
and nothing else.
|
||||
- Check builds keep these, and none of them can hand back a stale result:
|
||||
- The layers above `ARG CACHEBUST`: the system packages and Go's
|
||||
modules. Docker installs the packages again when the base image or the
|
||||
install line changes, and downloads the modules again when `go.mod` or
|
||||
`go.sum` changes. Go checks every module against `go.sum` as it
|
||||
downloads it, and no later build can change a kept layer. The modules
|
||||
are kept in a layer rather than a cache mount because a cache mount is
|
||||
shared by every build on the host, and Go builds from whatever files
|
||||
it holds without checking them again (only `go mod verify` does), so a
|
||||
file changed there would go unnoticed.
|
||||
- Go's build cache (`/root/.cache/go-build`, where Go keeps it for
|
||||
root), mounted on the steps that compile: `golangci-lint run`, both
|
||||
`go test` runs and `go build`. Go reuses a compiled package only when
|
||||
all its inputs are unchanged. The cache also holds test results, which
|
||||
is why both `go test` runs carry `-count=1`.
|
||||
- golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
|
||||
image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
|
||||
a package's findings only when the package's files and those of every
|
||||
package it imports, `go.mod`, the linters and their settings in
|
||||
`.golangci.yml`, and the golangci-lint version are all unchanged.
|
||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||
includes both Go and the linter), so there is no need to install the
|
||||
linter separately.
|
||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||
this line, the build stage would not wait for lint to finish and a lint
|
||||
failure might not fail the overall build.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
- If the project requires CGO or system libraries for linting, install them
|
||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
||||
has no `apk`, so install with `apt-get` under the Debian package name
|
||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
||||
lists in the same `RUN`, so the layer does not keep them. The test phase,
|
||||
on the Debian Go image, installs the libraries its tests need the same
|
||||
way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
|
||||
Docker keeps it until the base image or the install line changes:
|
||||
|
||||
```dockerfile
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
ARG CACHEBUST
|
||||
```
|
||||
|
||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
||||
submodule keeping its own `.git` directory. `git describe` does not need
|
||||
them, and each can hold a credential: a password in a remote URL, or the
|
||||
token the CI checkout step stores there. A submodule whose name has a
|
||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
||||
then fails the build: give it a name without that segment
|
||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
||||
takes the version from the `VERSION` build argument when one is given,
|
||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
||||
tagged commit; on a later commit, the tag, the number of commits since it
|
||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
||||
tag is reachable. The stage that compiles also marks its working directory
|
||||
safe for git (`git config --system --add safe.directory /src`): a context
|
||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
||||
checkout owned by another user, so the version would come out empty.
|
||||
`ARG VERSION` has no default, and the build fails if the context carries
|
||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
||||
refuses an empty build argument drops that refusal and keeps the argument.
|
||||
A checkout whose `.git` is a file (a linked worktree, or a repository
|
||||
checked out as a submodule) is the exception: that file points to a git
|
||||
directory outside the build context, so the build cannot read the version
|
||||
and a plain `docker build .` fails; pass the version with
|
||||
`--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already
|
||||
do.
|
||||
The lint stage should not depend on the actual build output — it exists to
|
||||
fail fast.
|
||||
- If the project requires CGO or system libraries for linting (e.g.
|
||||
`vips-dev`), install them in the lint stage with `apk add`.
|
||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||
build stage, not the lint stage, because they may require compiled
|
||||
artifacts or heavier dependencies.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `script/cibuild` on push, and checks out the repo as its only other step,
|
||||
with `persist-credentials: false`: `script/cibuild` needs no token, and
|
||||
without it the checkout leaves the job's token in `.git/config` for every
|
||||
later step. The checkout step also sets `fetch-depth: 0`, which fetches the
|
||||
tags `git describe` needs: by default it clones shallow with no tags, and a
|
||||
tagged repository's CI build would stamp a bare short commit id. The
|
||||
workflow's `concurrency` block groups runs by workflow and branch
|
||||
(`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`,
|
||||
so a new push cancels the older run on the same branch, queued or running, and
|
||||
no other: runs for replaced commits do not hold up the shared runner.
|
||||
`script/cibuild` bootstraps, runs the gate phases, and then builds the image,
|
||||
so a successful run means every check passed; a bare `docker build .` does not
|
||||
carry the same guarantee, because its gate phases may come from the cache. The
|
||||
image build gets a `CACHEBUST` value of its own and so runs the gate phases a
|
||||
second time. That is the price of the rule above, and it is worth paying: the
|
||||
image that ships is built from a run of its own gates rather than from a cache
|
||||
entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
|
||||
shared runner after 20 minutes. That allows for the three Docker builds
|
||||
described above (the test phase, the lint phase, then the image), each held to
|
||||
the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
|
||||
limited to `main` by a `branches` list under `on: push` cannot be checked by
|
||||
review: to try a change to it, add the feature branch to that list and push,
|
||||
then remove the branch from the list again before merging. Keep any job in it
|
||||
that publishes behind `if: github.ref_name == 'main'`, so the run from the
|
||||
feature branch publishes nothing.
|
||||
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||
Dockerfile already runs `make check`, a successful build implies all checks
|
||||
pass.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
@@ -408,21 +189,14 @@ style conventions are in separate documents:
|
||||
module under test to verify it compiles/parses. There is no excuse for
|
||||
`make test` to be a no-op.
|
||||
|
||||
- `make test` must complete in under 60 seconds. That is the hard cap, and a
|
||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
||||
20 and 60 seconds is still green, but the overage must be filed as an
|
||||
improvement bug against that repo. Add a 90-second timeout to the test
|
||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
||||
one.
|
||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||
Makefile.
|
||||
|
||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
||||
on success (just package/suite summaries) while providing full diagnostic
|
||||
detail on failure (every test case, every assertion). The command lives in the
|
||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
||||
Makefile form below is the same pattern for any repo-local invocation:
|
||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||
show full output. This keeps CI logs and `docker build` output clean on
|
||||
success (just package/suite summaries) while providing full diagnostic detail
|
||||
on failure (every test case, every assertion). The general shell pattern:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@@ -435,25 +209,11 @@ style conventions are in separate documents:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
@go test -timeout 30s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
go test -timeout 30s -race -v ./...; exit 1; }
|
||||
```
|
||||
|
||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
||||
cache, so neither run can report a stored pass in place of running the
|
||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
||||
and no recompilation.
|
||||
|
||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
||||
run again on unchanged code it prints that result, marked `(cached)`,
|
||||
without running them. The directory lasts from one run to the next on a
|
||||
developer's machine, where this target runs, and in the `test` phase of the
|
||||
`Dockerfile`, which mounts it to keep compiled packages between check
|
||||
builds, so the phase's `go test` runs carry `-count=1` too. A new
|
||||
`CACHEBUST` value (above) is what makes that step run on an unchanged tree.
|
||||
|
||||
Python example:
|
||||
|
||||
```makefile
|
||||
@@ -479,89 +239,10 @@ style conventions are in separate documents:
|
||||
must be in `.gitignore`. No exceptions.
|
||||
|
||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
||||
`node_modules/`, and the repo's own build outputs. Fetch the standard
|
||||
`.gitignore` from
|
||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||
Fetch the standard `.gitignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||
a new repo. A repo's `.gitignore` is the standard file followed by the repo's
|
||||
own entries, such as its binaries; a re-vendor replaces the standard part and
|
||||
keeps those entries. These patterns are written to `.gitignore`'s own
|
||||
semantics, in which an unanchored pattern already matches at every depth; they
|
||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
||||
|
||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
||||
across unmodified leaves secrets in the build context.** Docker matches with
|
||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
||||
therefore excludes only the copies at the repository root, while `config/.env`
|
||||
and `certs/server.key` still reach the context and can land in an image layer
|
||||
— which is more dangerous than a short file with no secret patterns at all,
|
||||
because it reads as solved and stops anyone looking. Give every
|
||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
||||
also catches something the build needs, re-include it with a negation
|
||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
||||
other file it covers. Fetch the standard `.dockerignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
||||
it with the repo's own artifacts.
|
||||
|
||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
||||
additional checkout of the repo — so under `COPY . .` the build context
|
||||
inflates by a multiple of the repo and another session's unreviewed work can
|
||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
||||
prefix, because the prefixed form would also delete any nested directory of
|
||||
that name from the build. Anchoring carries a known gap that the canonical
|
||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
||||
file and not the tracker: the directory is created in the agent's working
|
||||
directory, so a repo running agents in subdirectories still ships
|
||||
`services/api/.claude/` and must add its own anchored entry there.
|
||||
|
||||
- **A plain `docker build .` of a clone stamps the version that
|
||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
||||
the version they compute on the host; it takes precedence. They do this
|
||||
byte-identically across repos:
|
||||
|
||||
```sh
|
||||
# The version and the tag each get their own line: a failing command
|
||||
# substitution inside an argument does not trip `set -e`, so the inline
|
||||
# form degrades to an empty constant.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
tag="$(script/projectname)"
|
||||
docker build --build-arg CACHEBUST="$(date +%s)-$$" \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$tag" .
|
||||
```
|
||||
|
||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
||||
fallback is applied — a live check that fires on a build from an export with
|
||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
||||
the scripts stay byte-identical. One consequence for CI: the standard
|
||||
checkout action clones shallow and fetches no tags, so the canonical
|
||||
`.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step.
|
||||
|
||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
||||
a probe image that does `COPY . .`, and list what actually landed
|
||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
||||
transfers only the delta from the previous build.
|
||||
a new repo.
|
||||
|
||||
- **No build artifacts in version control.** Code-derived data (compiled
|
||||
bundles, minified output, generated assets) must never be committed to the
|
||||
@@ -577,61 +258,9 @@ style conventions are in separate documents:
|
||||
- Make all changes on a feature branch. You can do whatever you want on a
|
||||
feature branch.
|
||||
|
||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
||||
_NEVER_ be modified by an agent: fetch it from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
||||
because it cannot be written once for every repo: the `deny` list of the
|
||||
`test-support` depguard rule, where a repo names its own test-support packages
|
||||
by full import path. A repo adds entries there and changes nothing else, and a
|
||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
||||
image
|
||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
||||
directive must not name a newer Go minor version than the one golangci-lint
|
||||
was built with, or golangci-lint refuses to lint it: this release lints
|
||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
||||
the two prompted the change: the canonical copy can name linters that an older
|
||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
||||
`default: all` switches on until the canonical copy disables them.
|
||||
|
||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
||||
disagree about what the tool even is. The canonical form:
|
||||
- compares the installed version against the pin over the **whole** version
|
||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
||||
running `2.12.2-rc1` and skips the install;
|
||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
||||
mismatch, so the failure direction is a redundant install and never a
|
||||
skipped one;
|
||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
||||
then through `PATH`, not through the directory the installer wrote to —
|
||||
and fails naming the resolved path, since an install that a shadowing
|
||||
binary hides succeeds while changing nothing any caller sees;
|
||||
- is actually called, and prints the version on both success paths: a
|
||||
function defined and never invoked has the same exit status and the same
|
||||
empty output as one that worked.
|
||||
|
||||
Node is the exception to the whole-token comparison: the canonical
|
||||
`script/bootstrap` compares only its major version, because the node 22
|
||||
alpine image the `Dockerfile` stages start from is not the exact pin, and
|
||||
nvm cannot install a prebuilt node on alpine.
|
||||
|
||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||
|
||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||
manually by the user. Fetch from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
@@ -742,21 +371,15 @@ style conventions are in separate documents:
|
||||
Never edit existing migrations after release.
|
||||
|
||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
||||
settings: the standard file from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets
|
||||
tabs for `Makefile` and Go files, followed by the repo's own sections, such as
|
||||
one for another language it uses. A re-vendor replaces the standard part and
|
||||
keeps those sections.
|
||||
settings.
|
||||
|
||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
||||
and language-specific config). Everything else goes in a subdirectory.
|
||||
Canonical subdirectory names:
|
||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||
subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
||||
`cmd/`
|
||||
- `cmd/` — Go command entrypoints
|
||||
- `configs/` — configuration templates and examples
|
||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||
- `docs/` — documentation and markdown (README.md stays in root)
|
||||
@@ -783,7 +406,3 @@ style conventions are in separate documents:
|
||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||
- Python: `pyproject.toml`
|
||||
|
||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
||||
is never committed under a file or directory named after one agent tool, such
|
||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
// Command loadtest-origin is the upstream host script/loadtest points pixad
|
||||
// at; internal/loadtestorigin says what it does.
|
||||
package main
|
||||
|
||||
import "sneak.berlin/go/pixa/internal/loadtestorigin"
|
||||
|
||||
func main() {
|
||||
loadtestorigin.Run()
|
||||
}
|
||||
+53
-2
@@ -1,10 +1,61 @@
|
||||
// Package main is the entry point for the pixad image proxy server.
|
||||
package main
|
||||
|
||||
import "sneak.berlin/go/pixa/internal/app"
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
"sneak.berlin/go/pixa/internal/server"
|
||||
)
|
||||
|
||||
var Version string //nolint:gochecknoglobals // set by ldflags
|
||||
|
||||
var configPath string //nolint:gochecknoglobals // cobra flag
|
||||
|
||||
func main() {
|
||||
app.Run(Version)
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "pixad",
|
||||
Short: "Pixa image caching proxy server",
|
||||
Run: run,
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(_ *cobra.Command, _ []string) {
|
||||
globals.Version = Version
|
||||
|
||||
// Set config path in environment if specified via flag
|
||||
if configPath != "" {
|
||||
_ = os.Setenv("PIXA_CONFIG_PATH", configPath)
|
||||
}
|
||||
|
||||
fx.New(
|
||||
fx.Provide(
|
||||
config.New,
|
||||
database.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
logger.New,
|
||||
server.New,
|
||||
middleware.New,
|
||||
healthcheck.New,
|
||||
),
|
||||
fx.Invoke(func(*server.Server) {}),
|
||||
).Run()
|
||||
}
|
||||
|
||||
@@ -12,63 +12,40 @@
|
||||
# Durations are Go duration strings such as 30s or 2m and must be
|
||||
# positive; a bare number has no unit and aborts startup. Sizes are a
|
||||
# whole number of bytes.
|
||||
#
|
||||
# A key left out takes the default its comment gives.
|
||||
|
||||
# Port to listen on (default: 8080)
|
||||
# Server settings
|
||||
port: 8080
|
||||
|
||||
# Debug logging and plain-HTTP local development (default: false)
|
||||
debug: false
|
||||
|
||||
# While true, the image routes (/v1/image/ and /v1/e/) answer every request
|
||||
# for an image with 503 and a Retry-After header. The health check keeps
|
||||
# answering 200 and reports maintenance_mode as true. It stays 200 because
|
||||
# the image's Docker HEALTHCHECK requests it: a 503 there would make the
|
||||
# container unhealthy, and upaas marks a deploy failed when its container is
|
||||
# unhealthy. (default: false)
|
||||
# with 503 and a Retry-After header. The health check keeps answering 200 and
|
||||
# reports maintenance_mode as true. It stays 200 because the image's Docker
|
||||
# HEALTHCHECK requests it: a 503 there would make the container unhealthy, and
|
||||
# upaas marks a deploy failed when its container is unhealthy.
|
||||
maintenance_mode: false
|
||||
|
||||
# Data directory for SQLite database and cache files
|
||||
# (default: /var/lib/pixa)
|
||||
state_dir: ./data
|
||||
|
||||
# SQLite database URL (default:
|
||||
# file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)). pixa adds
|
||||
# _pragma=busy_timeout(5000) to it. An empty value aborts startup; leave the
|
||||
# key out to use the default.
|
||||
# db_url: "file:./data/state.sqlite3?_pragma=journal_mode(WAL)"
|
||||
|
||||
# Image proxy settings
|
||||
# HMAC signing key for URL signatures (required, at least 32 characters)
|
||||
# Generate with: openssl rand -base64 32
|
||||
signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||
|
||||
# Hosts that don't require signatures (default: none)
|
||||
# Every image on an allowlisted host is served to anyone without a
|
||||
# signature, at any size and in any format, so list only hosts you
|
||||
# control, where nobody else can put images.
|
||||
# Hosts that don't require signatures
|
||||
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
||||
# An entry that is neither a host name nor an IP address (IPv6 without
|
||||
# brackets), such as one with a port or a "*." wildcard, aborts startup.
|
||||
# allowlist_hosts:
|
||||
# - images.example.com
|
||||
|
||||
# Hosts whose pages may not show pixa's images, written as for
|
||||
# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
|
||||
# names one of them is answered 403 before anything is fetched, even when
|
||||
# the image is cached. A request with no Referer, or one that does not
|
||||
# parse, is served, so a site whose pages send no Referer is not stopped.
|
||||
# The login and generator pages are not covered. (default: none)
|
||||
# referer_blocklist:
|
||||
# - leech.example
|
||||
# - .hotlinker.example
|
||||
allowlist_hosts:
|
||||
- s3.sneak.cloud
|
||||
- static.sneak.cloud
|
||||
- sneak.berlin
|
||||
- github.com
|
||||
- user-images.githubusercontent.com
|
||||
|
||||
# Additional CIDR ranges to refuse when fetching upstream, extending the
|
||||
# SSRF protection. These are added to the always-enforced built-in ranges
|
||||
# (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and
|
||||
# similar), never replacing them. Each entry must be a valid CIDR in IPv4
|
||||
# or IPv6 form; an invalid entry aborts startup. (default: none)
|
||||
# or IPv6 form; an invalid entry aborts startup.
|
||||
# blocked_networks:
|
||||
# - 100.64.0.0/10
|
||||
# - 2001:db8::/32
|
||||
@@ -95,7 +72,6 @@ signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||
# - 2001:db8::/32
|
||||
|
||||
# Allow HTTP upstream (only for testing, always use HTTPS in production)
|
||||
# (default: false)
|
||||
allow_http: false
|
||||
|
||||
# Maximum concurrent connections per upstream host (default: 20)
|
||||
@@ -108,21 +84,12 @@ upstream_connections_per_host: 20
|
||||
# downstream_timeout has ended first.
|
||||
upstream_connections: 64
|
||||
|
||||
# Maximum number of images decoded and encoded at once (default: half the
|
||||
# number of CPUs pixa can use, at least 1; the number of CPUs follows a
|
||||
# container's CPU limit). A request that finds none free waits up to 10
|
||||
# seconds for one, and if none frees up it is answered 503, unless
|
||||
# downstream_timeout has ended first.
|
||||
# Maximum number of images decoded and encoded at once (default: the
|
||||
# number of CPUs pixa can use, which follows a container's CPU limit). A
|
||||
# request that finds none free waits up to 10 seconds for one, and if none
|
||||
# frees up it is answered 503, unless downstream_timeout has ended first.
|
||||
# max_concurrent_processing: 4
|
||||
|
||||
# How many requests to the image routes (/v1/image/ and /v1/e/) one client
|
||||
# may send per minute; a request over the limit is answered 429 with a
|
||||
# Retry-After header. 0 turns the limit off. Clients are told apart as for
|
||||
# the login limit, so behind a CDN or proxy whose addresses are not in
|
||||
# trusted_proxies all requests arriving from one proxy or CDN address count
|
||||
# as one client: raise the limit there, or set 0. (default: 600)
|
||||
image_requests_per_minute: 600
|
||||
|
||||
# Time allowed for one fetch from an upstream host (default: 30s)
|
||||
upstream_fetch_timeout: 30s
|
||||
|
||||
@@ -136,9 +103,8 @@ upstream_max_response_size: 52428800
|
||||
# longer than upstream_fetch_timeout plus 20 seconds.
|
||||
downstream_timeout: 60s
|
||||
|
||||
# The origin a browser lets read the responses of the image routes,
|
||||
# /v1/image/ and /v1/e/, sent as the CORS Access-Control-Allow-Origin
|
||||
# header; no other route sends it. "*" (the default) is any site;
|
||||
# The origin a browser lets read pixa's responses, sent as the CORS
|
||||
# Access-Control-Allow-Origin header: "*" (the default) is any site;
|
||||
# otherwise one http or https origin such as https://example.com, whose
|
||||
# host is a lowercase host name (letters, digits, hyphens and dots, with a
|
||||
# letter in its last part) or an IP address (IPv6 in brackets, in its
|
||||
@@ -149,22 +115,15 @@ access_control_allow_origin: "*"
|
||||
|
||||
# Maximum disk cache size in bytes. Explicit values are used exactly as
|
||||
# given; 0 disables the disk cache entirely (every request fetches and
|
||||
# processes uncached). When omitted, the default is 75% of the sum of
|
||||
# the free space on the filesystem containing <state_dir>/cache/ and
|
||||
# the bytes of images the cache already holds, worked out at startup,
|
||||
# processes uncached). When omitted, the default is 75% of the free
|
||||
# space on the filesystem containing <state_dir>/cache/ at startup,
|
||||
# with a minimum of 500 MiB.
|
||||
# cache_max_bytes: 10737418240
|
||||
|
||||
# Sentry DSN for error reporting (default: empty, which turns it off)
|
||||
# Sentry error reporting (optional)
|
||||
sentry_dsn: ""
|
||||
|
||||
# Username and password for /metrics, set together (default: unset). Metrics
|
||||
# are measured and /metrics is served only when both are set.
|
||||
# Metrics endpoint authentication (optional)
|
||||
# metrics:
|
||||
# username: "admin"
|
||||
# password: "secret"
|
||||
|
||||
# Environment variables set while this file loads, as described at the top
|
||||
# (default: none)
|
||||
# env:
|
||||
# PIXA_DEBUG: "true"
|
||||
@@ -1,17 +0,0 @@
|
||||
# Example Caddy config for running pixa behind Caddy; see "Deployment" in
|
||||
# README.md. Replace images.example.com with pixa's public host name, and
|
||||
# 127.0.0.1:8080 with the address Caddy reaches pixa on.
|
||||
#
|
||||
# Caddy gets and renews the TLS certificate for the host name, passes the
|
||||
# Host, Origin and Referer headers on unchanged, sets X-Forwarded-For to the
|
||||
# client's address, and waits for pixa's answer with no time limit of its
|
||||
# own, so pixa's downstream_timeout is what ends a slow request.
|
||||
|
||||
images.example.com
|
||||
|
||||
# pixa asks for metrics.username and metrics.password on /metrics. This
|
||||
# line also keeps it off the public address, for a scraper that reaches
|
||||
# pixa directly; remove it to read /metrics through Caddy.
|
||||
respond /metrics 404
|
||||
|
||||
reverse_proxy 127.0.0.1:8080
|
||||
@@ -14,12 +14,12 @@ require (
|
||||
github.com/go-chi/httprate v0.16.0
|
||||
github.com/gorilla/csrf v1.7.3
|
||||
github.com/gorilla/securecookie v1.1.2
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/slok/go-http-metrics v0.13.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
go.uber.org/fx v1.24.0
|
||||
golang.org/x/crypto v0.41.0
|
||||
golang.org/x/sync v0.19.0
|
||||
modernc.org/sqlite v1.42.2
|
||||
)
|
||||
|
||||
@@ -134,6 +134,7 @@ require (
|
||||
golang.org/x/image v0.34.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
golang.org/x/term v0.34.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
|
||||
@@ -228,6 +228,8 @@ github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I=
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
// Package app reads the pixad command line and runs the server.
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
"sneak.berlin/go/pixa/internal/server"
|
||||
)
|
||||
|
||||
var configPath string //nolint:gochecknoglobals // cobra flag
|
||||
|
||||
// Run reads the command line and runs the server until it stops, with
|
||||
// version as the version pixad logs and reports. It exits the process
|
||||
// with status 1 when the command line is not valid.
|
||||
func Run(version string) {
|
||||
globals.Version = version
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "pixad",
|
||||
Short: "Pixa image caching proxy server",
|
||||
Run: run,
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(_ *cobra.Command, _ []string) {
|
||||
// Set config path in environment if specified via flag
|
||||
if configPath != "" {
|
||||
_ = os.Setenv("PIXA_CONFIG_PATH", configPath)
|
||||
}
|
||||
|
||||
// A write to a closed stdout or stderr must not end the process.
|
||||
signal.Ignore(syscall.SIGPIPE)
|
||||
|
||||
fx.New(
|
||||
fx.Provide(
|
||||
config.New,
|
||||
database.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
logger.New,
|
||||
server.New,
|
||||
middleware.New,
|
||||
healthcheck.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
func(log *logger.Logger) { log.Identify() },
|
||||
func(*server.Server) {},
|
||||
),
|
||||
).Run()
|
||||
}
|
||||
@@ -88,56 +88,3 @@ func TestBlockedNetworksInvalidAbortsStartup(t *testing.T) {
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestBlockedNetworksMappedEntryIsIPv4Range checks that an entry written in
|
||||
// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
|
||||
// since the fetcher matches an IPv4-mapped address as the IPv4 address it
|
||||
// holds. /96, the shortest such entry, covers every IPv4 address.
|
||||
func TestBlockedNetworksMappedEntryIsIPv4Range(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
yamlContent := signingKeyLine + `blocked_networks:
|
||||
- "::ffff:10.0.0.0/104"
|
||||
- "::ffff:0:0/96"
|
||||
`
|
||||
|
||||
c, err := configFromYAML(t, yamlContent)
|
||||
if err != nil {
|
||||
t.Fatalf("IPv4-mapped blocked_networks entries should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"10.0.0.0/8", "0.0.0.0/0"}
|
||||
if len(c.BlockedNetworks) != len(want) {
|
||||
t.Fatalf("BlockedNetworks = %v, want %d entries", c.BlockedNetworks, len(want))
|
||||
}
|
||||
|
||||
for i, w := range want {
|
||||
if got := c.BlockedNetworks[i].String(); got != w {
|
||||
t.Errorf("BlockedNetworks[%d] = %q, want %q", i, got, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockedNetworksMappedEntryShorterThan96AbortsStartup checks that an
|
||||
// entry in the IPv4-mapped form shorter than /96, which covers more than the
|
||||
// IPv4 addresses, aborts startup with an error naming the key and the value.
|
||||
func TestBlockedNetworksMappedEntryShorterThan96AbortsStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runAbortCases(t, []abortCase{
|
||||
{
|
||||
name: "mapped-shorter-than-96",
|
||||
yaml: signingKeyLine + `blocked_networks:
|
||||
- "::ffff:0.0.0.0/80"
|
||||
`,
|
||||
wantErrSubstrings: []string{keyBlockedNetworks, "::ffff:0.0.0.0/80"},
|
||||
},
|
||||
{
|
||||
name: "mapped-95",
|
||||
yaml: signingKeyLine + `blocked_networks:
|
||||
- "::ffff:0:0/95"
|
||||
`,
|
||||
wantErrSubstrings: []string{keyBlockedNetworks},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,10 +1,26 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Static errors returned by the stub free-space probes below.
|
||||
var (
|
||||
errTestStatfsFailed = errors.New("statfs failed")
|
||||
errTestProbeNotExpected = errors.New("probe must not be called")
|
||||
)
|
||||
|
||||
// discardLogger returns a logger that swallows all output, for tests
|
||||
// that exercise code paths which log.
|
||||
func discardLogger() *slog.Logger {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
|
||||
// TestCacheMaxBytesExplicitValueUsedWithoutFloor verifies that an
|
||||
// explicitly configured cache_max_bytes value is used exactly as
|
||||
// given: the 500 MiB floor applies only to the computed default, never
|
||||
@@ -135,30 +151,155 @@ func TestCacheMaxBytesInvalidValuesAbortStartup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheMaxBytesExplicitIsRecorded verifies that an omitted
|
||||
// cache_max_bytes is recorded as not explicit, so the cache works out
|
||||
// the default when it opens, and that an explicit zero is recorded as
|
||||
// explicit, so it disables the disk cache instead.
|
||||
func TestCacheMaxBytesExplicitIsRecorded(t *testing.T) {
|
||||
// TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace verifies the
|
||||
// computed default is 75% of the probed free space when that exceeds
|
||||
// the floor.
|
||||
func TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
signingKeyLine := "signing_key: " + validTestSigningKey + "\n"
|
||||
// 4 GiB free -> 3 GiB default.
|
||||
probe := func(string) (uint64, error) { return 4294967296, nil }
|
||||
|
||||
omitted, err := configFromYAML(t, signingKeyLine)
|
||||
got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe)
|
||||
if err != nil {
|
||||
t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err)
|
||||
}
|
||||
|
||||
if got != 3221225472 {
|
||||
t.Errorf("ComputeDefaultCacheMaxBytes = %d, want 3221225472 (75%% of 4 GiB)",
|
||||
got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault
|
||||
// verifies that when 75% of free space is below 500 MiB, the computed
|
||||
// default is floored at DefaultCacheMaxBytesFloor.
|
||||
func TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
freeBytes uint64
|
||||
}{
|
||||
{name: "100 MiB free", freeBytes: 104857600},
|
||||
{name: "zero free", freeBytes: 0},
|
||||
{name: "just below floor threshold", freeBytes: 699050665},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
probe := func(string) (uint64, error) { return tc.freeBytes, nil }
|
||||
|
||||
got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe)
|
||||
if err != nil {
|
||||
t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err)
|
||||
}
|
||||
|
||||
if got != DefaultCacheMaxBytesFloor {
|
||||
t.Errorf("ComputeDefaultCacheMaxBytes = %d, want floor %d",
|
||||
got, DefaultCacheMaxBytesFloor)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeDefaultCacheMaxBytesPropagatesProbeError verifies that a
|
||||
// failing free-space probe produces an error naming the config key,
|
||||
// instead of a silently wrong default.
|
||||
func TestComputeDefaultCacheMaxBytesPropagatesProbeError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
probe := func(string) (uint64, error) { return 0, errTestStatfsFailed }
|
||||
|
||||
_, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe)
|
||||
if err == nil {
|
||||
t.Fatal("probe failure must produce an error, got nil")
|
||||
}
|
||||
|
||||
t.Logf("got expected error: %v", err)
|
||||
|
||||
if !strings.Contains(err.Error(), keyCacheMaxBytes) {
|
||||
t.Errorf("error %q does not name the config key cache_max_bytes", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveCacheMaxBytesComputesDefaultWhenOmitted verifies that an
|
||||
// omitted cache_max_bytes key resolves to the computed default, that
|
||||
// the probe is pointed at <state_dir>/cache/ (which must be created
|
||||
// first so statfs measures the right filesystem), and that the result
|
||||
// lands on the Config.
|
||||
func TestResolveCacheMaxBytesComputesDefaultWhenOmitted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, "signing_key: "+validTestSigningKey+"\n")
|
||||
if err != nil {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
if omitted.CacheMaxBytesExplicit {
|
||||
t.Error("omitted cache_max_bytes recorded as explicit")
|
||||
c.StateDir = t.TempDir()
|
||||
wantCacheDir := filepath.Join(c.StateDir, "cache")
|
||||
|
||||
var probedPath string
|
||||
|
||||
// 4 GiB free -> 3 GiB default.
|
||||
probe := func(path string) (uint64, error) {
|
||||
probedPath = path
|
||||
|
||||
return 4294967296, nil
|
||||
}
|
||||
|
||||
zero, err := configFromYAML(t, signingKeyLine+"cache_max_bytes: 0\n")
|
||||
err = c.resolveCacheMaxBytes(discardLogger(), probe)
|
||||
if err != nil {
|
||||
t.Fatalf("cache_max_bytes: 0 must be accepted, got error: %v", err)
|
||||
t.Fatalf("resolveCacheMaxBytes returned error: %v", err)
|
||||
}
|
||||
|
||||
if !zero.CacheMaxBytesExplicit {
|
||||
t.Error("cache_max_bytes: 0 not recorded as explicit")
|
||||
if c.CacheMaxBytes != 3221225472 {
|
||||
t.Errorf("CacheMaxBytes = %d, want computed default 3221225472",
|
||||
c.CacheMaxBytes)
|
||||
}
|
||||
|
||||
if probedPath != wantCacheDir {
|
||||
t.Errorf("free space probed at %q, want cache directory %q",
|
||||
probedPath, wantCacheDir)
|
||||
}
|
||||
|
||||
info, err := os.Stat(wantCacheDir)
|
||||
if err != nil || !info.IsDir() {
|
||||
t.Errorf("cache directory %q was not created before probing: info=%v err=%v",
|
||||
wantCacheDir, info, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveCacheMaxBytesDoesNotOverrideExplicitValue verifies that
|
||||
// an explicitly configured value survives resolution untouched and
|
||||
// that the free-space probe is never consulted for it.
|
||||
func TestResolveCacheMaxBytesDoesNotOverrideExplicitValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 1024\n"
|
||||
|
||||
c, err := configFromYAML(t, yamlContent)
|
||||
if err != nil {
|
||||
t.Fatalf("explicit cache_max_bytes must be accepted, got error: %v", err)
|
||||
}
|
||||
|
||||
c.StateDir = t.TempDir()
|
||||
|
||||
probe := func(string) (uint64, error) {
|
||||
t.Error("free-space probe must not be consulted for explicit values")
|
||||
|
||||
return 0, errTestProbeNotExpected
|
||||
}
|
||||
|
||||
err = c.resolveCacheMaxBytes(discardLogger(), probe)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCacheMaxBytes returned error: %v", err)
|
||||
}
|
||||
|
||||
if c.CacheMaxBytes != 1024 {
|
||||
t.Errorf("CacheMaxBytes = %d, want explicit 1024 (no floor, no recompute)",
|
||||
c.CacheMaxBytes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// DefaultCacheMaxBytesFloor is the minimum computed default for the
|
||||
// cache_max_bytes setting: 500 MiB. The floor applies only to the
|
||||
// computed default (when the key is omitted from the configuration),
|
||||
// never to explicitly configured values.
|
||||
const DefaultCacheMaxBytesFloor int64 = 524288000
|
||||
|
||||
// cacheDirPerms is the permission mode for the cache directory created
|
||||
// before probing free space, matching the state directory permissions.
|
||||
const cacheDirPerms = 0o750
|
||||
|
||||
// freeSpaceFractionNumerator and freeSpaceFractionDenominator express
|
||||
// the 75% share of free space used for the computed default limit as
|
||||
// integer arithmetic (dividing before multiplying avoids overflow).
|
||||
const (
|
||||
freeSpaceFractionNumerator uint64 = 3
|
||||
freeSpaceFractionDenominator uint64 = 4
|
||||
)
|
||||
|
||||
// FreeSpaceProbeFunc reports the number of free bytes available on the
|
||||
// filesystem containing path. It is a function type so tests can
|
||||
// inject a fake probe instead of depending on the host disk.
|
||||
type FreeSpaceProbeFunc func(path string) (uint64, error)
|
||||
|
||||
// defaultFreeSpaceProbe reports free filesystem bytes via statfs on
|
||||
// the given path, as available to unprivileged processes.
|
||||
func defaultFreeSpaceProbe(path string) (uint64, error) {
|
||||
var stat syscall.Statfs_t
|
||||
|
||||
err := syscall.Statfs(path, &stat)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if stat.Bsize < 0 {
|
||||
return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path)
|
||||
}
|
||||
|
||||
blockSize := uint64(stat.Bsize)
|
||||
|
||||
return stat.Bavail * blockSize, nil
|
||||
}
|
||||
|
||||
// ComputeDefaultCacheMaxBytes returns the default cache size limit for
|
||||
// the filesystem containing cacheDir: 75% of the free bytes reported
|
||||
// by probe, with a floor of DefaultCacheMaxBytesFloor.
|
||||
func ComputeDefaultCacheMaxBytes(
|
||||
cacheDir string, probe FreeSpaceProbeFunc,
|
||||
) (int64, error) {
|
||||
freeBytes, err := probe(cacheDir)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("config key %q: cannot determine free space for %q: %w",
|
||||
"cache_max_bytes", cacheDir, err)
|
||||
}
|
||||
|
||||
computed := freeBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
|
||||
computed = min(computed, math.MaxInt64)
|
||||
|
||||
// gosec cannot see that min() above bounds computed, so it reads
|
||||
// this conversion as potentially overflowing. It cannot: computed is
|
||||
// at most math.MaxInt64 on every path here.
|
||||
//nolint:gosec // G115: clamped to MaxInt64 by min above
|
||||
limit := int64(computed)
|
||||
limit = max(limit, DefaultCacheMaxBytesFloor)
|
||||
|
||||
return limit, nil
|
||||
}
|
||||
|
||||
// resolveCacheMaxBytes finalizes CacheMaxBytes after state_dir
|
||||
// validation: an explicitly configured value is kept as-is (no floor
|
||||
// applies), while an omitted key receives the computed default based
|
||||
// on free space in <state_dir>/cache/. The cache directory is created
|
||||
// first so statfs measures the filesystem that will actually hold the
|
||||
// cache. The effective limit is logged either way.
|
||||
func (c *Config) resolveCacheMaxBytes(
|
||||
log *slog.Logger, probe FreeSpaceProbeFunc,
|
||||
) error {
|
||||
if !c.cacheMaxBytesExplicit {
|
||||
cacheDir := filepath.Join(c.StateDir, "cache")
|
||||
|
||||
err := os.MkdirAll(cacheDir, cacheDirPerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config key %q: cannot create cache directory %q: %w",
|
||||
keyCacheMaxBytes, cacheDir, err)
|
||||
}
|
||||
|
||||
limit, err := ComputeDefaultCacheMaxBytes(cacheDir, probe)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.CacheMaxBytes = limit
|
||||
|
||||
log.Info("computed default cache size limit from free space",
|
||||
"cache_max_bytes", limit,
|
||||
"cache_dir", cacheDir,
|
||||
)
|
||||
}
|
||||
|
||||
log.Info("effective cache size limit",
|
||||
"cache_max_bytes", c.CacheMaxBytes,
|
||||
"cache_disabled", c.CacheMaxBytes == 0,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
@@ -13,8 +12,8 @@ const (
|
||||
)
|
||||
|
||||
// TestOmittedConcurrencyLimitsUseDefaults checks that an omitted
|
||||
// max_concurrent_processing is half the CPUs Go uses, and at least 1, and
|
||||
// an omitted upstream_connections is 64.
|
||||
// max_concurrent_processing is the number of CPUs Go uses and an omitted
|
||||
// upstream_connections is 64.
|
||||
func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -23,9 +22,9 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
if c.MaxConcurrentProcessing != max(runtime.GOMAXPROCS(0)/2, 1) {
|
||||
t.Errorf("MaxConcurrentProcessing = %d, want %d, half the CPUs",
|
||||
c.MaxConcurrentProcessing, max(runtime.GOMAXPROCS(0)/2, 1))
|
||||
if c.MaxConcurrentProcessing != runtime.GOMAXPROCS(0) {
|
||||
t.Errorf("MaxConcurrentProcessing = %d, want %d, one per CPU",
|
||||
c.MaxConcurrentProcessing, runtime.GOMAXPROCS(0))
|
||||
}
|
||||
|
||||
if c.UpstreamConnections != 64 {
|
||||
@@ -33,36 +32,6 @@ func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne checks
|
||||
// that an omitted max_concurrent_processing is half the CPUs Go uses,
|
||||
// rounded down, and 1 when Go uses one CPU.
|
||||
//
|
||||
//nolint:paralleltest // it sets GOMAXPROCS, which other tests read
|
||||
func TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne(t *testing.T) {
|
||||
cpusBefore := runtime.GOMAXPROCS(0)
|
||||
|
||||
t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) })
|
||||
|
||||
for _, tc := range []struct{ cpus, want int }{
|
||||
{1, 1}, {2, 1}, {3, 1}, {8, 4},
|
||||
} {
|
||||
runtime.GOMAXPROCS(tc.cpus)
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine)
|
||||
if err != nil {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("GOMAXPROCS %d: MaxConcurrentProcessing %d",
|
||||
tc.cpus, c.MaxConcurrentProcessing)
|
||||
|
||||
if c.MaxConcurrentProcessing != tc.want {
|
||||
t.Errorf("GOMAXPROCS %d: MaxConcurrentProcessing = %d, want %d",
|
||||
tc.cpus, c.MaxConcurrentProcessing, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestExplicitConcurrencyLimitsAreUsed checks that valid values for the
|
||||
// two limits are used as given.
|
||||
func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) {
|
||||
@@ -191,19 +160,3 @@ func TestConcurrencyLimitsFromEnvironment(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup checks
|
||||
// that a PIXA_MAX_CONCURRENT_PROCESSING that is not a whole number of at
|
||||
// least 1, an empty one included, aborts startup naming the variable and
|
||||
// the key instead of taking the default.
|
||||
func TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup(t *testing.T) {
|
||||
for _, value := range []string{"", " ", "0.5", "half", "-1", "4 images"} {
|
||||
t.Run(fmt.Sprintf("value %q", value), func(t *testing.T) {
|
||||
t.Setenv(testMaxConcurrentProcessingVar, value)
|
||||
|
||||
_, err := configFromYAML(t, signingKeyLine)
|
||||
wantStartupError(t, err,
|
||||
testMaxConcurrentProcessingVar, keyMaxConcurrentProcessing)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+81
-225
@@ -4,19 +4,16 @@ package config
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
@@ -34,17 +31,8 @@ const (
|
||||
DefaultUpstreamFetchTimeout = 30 * time.Second
|
||||
DefaultUpstreamMaxResponseSize = 50 << 20 // 50 MiB
|
||||
DefaultDownstreamTimeout = 60 * time.Second
|
||||
DefaultImageRequestsPerMinute = 600
|
||||
)
|
||||
|
||||
// DefaultMaxConcurrentProcessing returns the most images processed at once
|
||||
// when max_concurrent_processing is omitted: half the CPUs Go uses
|
||||
// (runtime.GOMAXPROCS(0), which follows a container's CPU limit), rounded
|
||||
// down, and at least 1.
|
||||
func DefaultMaxConcurrentProcessing() int {
|
||||
return max(runtime.GOMAXPROCS(0)/2, 1) //nolint:mnd // half the CPUs
|
||||
}
|
||||
|
||||
// Configuration key names.
|
||||
const (
|
||||
keyDebug = "debug"
|
||||
@@ -58,7 +46,6 @@ const (
|
||||
keyMetricsPassword = "metrics.password"
|
||||
keySigningKey = "signing_key"
|
||||
keyAllowlistHosts = "allowlist_hosts"
|
||||
keyRefererBlocklist = "referer_blocklist"
|
||||
keyAllowHTTP = "allow_http"
|
||||
keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
|
||||
keyUpstreamConnections = "upstream_connections"
|
||||
@@ -70,11 +57,10 @@ const (
|
||||
keyUpstreamFetchTimeout = "upstream_fetch_timeout"
|
||||
keyUpstreamMaxResponseSize = "upstream_max_response_size"
|
||||
keyDownstreamTimeout = "downstream_timeout"
|
||||
keyImageRequestsPerMinute = "image_requests_per_minute"
|
||||
)
|
||||
|
||||
// placeholderSigningKey is the dummy signing_key shipped in
|
||||
// configs/config.example.yml. It is 45 characters, so it passes the length
|
||||
// config.example.yml. It is 45 characters, so it passes the length
|
||||
// check, but it is public in this repository and must be rejected at
|
||||
// startup so no deployment ever signs URLs with it.
|
||||
const placeholderSigningKey = "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||
@@ -91,7 +77,6 @@ var (
|
||||
errNotABoolean = errors.New("not a boolean")
|
||||
errNotAStringList = errors.New("not a list of strings")
|
||||
errNotAValidCIDR = errors.New("not a valid CIDR network")
|
||||
errMappedTooShort = errors.New("in the IPv4-mapped form but shorter than /96")
|
||||
errNotAMetricsMap = errors.New("not a map of metrics settings")
|
||||
errEmptyListEntry = errors.New("list contains an empty entry")
|
||||
errEmptyEntry = errors.New("contains an empty entry")
|
||||
@@ -101,20 +86,23 @@ var (
|
||||
errMustBeAtLeastOne = errors.New("must be at least 1")
|
||||
errValueTooShort = errors.New("value too short")
|
||||
errPlaceholderKey = errors.New(
|
||||
"is the placeholder from configs/config.example.yml; " +
|
||||
"is the placeholder from config.example.yml; " +
|
||||
"generate a real key with: openssl rand -base64 32")
|
||||
errMustBeSetTogether = errors.New("must be set together")
|
||||
errMustNotBeNegative = errors.New("must not be negative")
|
||||
errOverflowsInt64 = errors.New("overflows a 64-bit integer")
|
||||
errValueNull = errors.New(
|
||||
errNegativeBlockSize = errors.New(
|
||||
"statfs reported negative block size")
|
||||
errValueNull = errors.New(
|
||||
"value is null; omit the key entirely to use the default")
|
||||
errValuesNull = errors.New(
|
||||
"value is null; omit a key entirely to use its default")
|
||||
errNotAHost = errors.New("must be a host name such as " +
|
||||
"cdn.example.com or .example.com, or an IP address")
|
||||
errNotADuration = errors.New("not a duration such as 30s or 2m")
|
||||
errMustBePositive = errors.New("must be positive")
|
||||
errNotAnOrigin = errors.New(
|
||||
errNotBareHostname = errors.New(
|
||||
"must be a bare hostname without scheme, path, or whitespace")
|
||||
errNoHostnameLabels = errors.New("contains no hostname labels")
|
||||
errNotADuration = errors.New("not a duration such as 30s or 2m")
|
||||
errMustBePositive = errors.New("must be positive")
|
||||
errNotAnOrigin = errors.New(
|
||||
`not "*" or an origin such as https://example.com`)
|
||||
)
|
||||
|
||||
@@ -142,10 +130,6 @@ type Config struct {
|
||||
AllowHTTP bool // Allow non-TLS upstream (testing only)
|
||||
UpstreamConnectionsPerHost int // Max concurrent connections per upstream host
|
||||
|
||||
// RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
|
||||
// image routes refuse a request whose Referer names a matching host.
|
||||
RefererBlocklist []string
|
||||
|
||||
// UpstreamConnections is the most concurrent connections to all
|
||||
// upstream hosts together, on top of the per-host limit.
|
||||
// MaxConcurrentProcessing is the most images processed at once.
|
||||
@@ -169,11 +153,6 @@ type Config struct {
|
||||
// of the per-request timeout middleware.
|
||||
DownstreamTimeout time.Duration
|
||||
|
||||
// ImageRequestsPerMinute is how many requests one client may make to
|
||||
// the image routes per minute; the next is refused with 429 Too Many
|
||||
// Requests. 0 turns the limit off.
|
||||
ImageRequestsPerMinute int
|
||||
|
||||
// BlockedNetworks are operator-supplied CIDR ranges to refuse in
|
||||
// addition to the built-in SSRF blocklist. Enforced by the upstream
|
||||
// fetcher's dialer; the built-in ranges always apply.
|
||||
@@ -190,19 +169,18 @@ type Config struct {
|
||||
// address, and an explicit list replaces the default.
|
||||
TrustedProxies []netip.Prefix
|
||||
|
||||
// CacheMaxBytes is the disk cache size limit in bytes. Only an
|
||||
// explicit zero (CacheMaxBytesExplicit true) disables the disk
|
||||
// cache. Zero with CacheMaxBytesExplicit false means
|
||||
// cache_max_bytes was omitted, and the cache works out the default
|
||||
// limit when it opens.
|
||||
// CacheMaxBytes is the disk cache size limit in bytes. Zero
|
||||
// disables the disk cache entirely. When cache_max_bytes is
|
||||
// omitted from the configuration, this holds the computed default
|
||||
// (75% of free space on the filesystem containing
|
||||
// <state_dir>/cache/, floored at DefaultCacheMaxBytesFloor).
|
||||
CacheMaxBytes int64
|
||||
|
||||
// CacheMaxBytesExplicit records whether cache_max_bytes was
|
||||
// cacheMaxBytesExplicit records whether cache_max_bytes was
|
||||
// explicitly set, in the environment or the configuration file.
|
||||
// Explicit values are used exactly as given; for an omitted key the
|
||||
// cache works out the default limit when it opens (see
|
||||
// imgcache.CacheConfig.UseDefaultMaxBytes).
|
||||
CacheMaxBytesExplicit bool
|
||||
// Explicit values are used exactly as given; only an omitted key
|
||||
// gets the computed default (and its floor) in resolveCacheMaxBytes.
|
||||
cacheMaxBytesExplicit bool
|
||||
}
|
||||
|
||||
// New creates a new Config instance from the environment and the
|
||||
@@ -239,13 +217,9 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// An omitted cache_max_bytes is worked out and logged when the
|
||||
// cache opens.
|
||||
if c.CacheMaxBytesExplicit {
|
||||
log.Info("effective cache size limit",
|
||||
"cache_max_bytes", c.CacheMaxBytes,
|
||||
"cache_disabled", c.CacheMaxBytes == 0,
|
||||
)
|
||||
err = c.resolveCacheMaxBytes(log, defaultFreeSpaceProbe)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if c.Debug {
|
||||
@@ -291,6 +265,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
}
|
||||
|
||||
loader := &strictLoader{sc: sc}
|
||||
|
||||
c := &Config{
|
||||
Debug: loader.boolVal(keyDebug, false),
|
||||
MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
|
||||
@@ -306,8 +281,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost),
|
||||
UpstreamConnections: loader.intVal(
|
||||
keyUpstreamConnections, DefaultUpstreamConnections),
|
||||
// Decoding and encoding are CPU-bound, so the default is one image
|
||||
// per CPU Go uses, which follows a container's CPU limit.
|
||||
MaxConcurrentProcessing: loader.intVal(
|
||||
keyMaxConcurrentProcessing, DefaultMaxConcurrentProcessing()),
|
||||
keyMaxConcurrentProcessing, runtime.GOMAXPROCS(0)),
|
||||
UpstreamFetchTimeout: loader.durationVal(
|
||||
keyUpstreamFetchTimeout, DefaultUpstreamFetchTimeout),
|
||||
UpstreamMaxResponseSize: loader.int64Val(
|
||||
@@ -316,18 +293,17 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin),
|
||||
DownstreamTimeout: loader.durationVal(
|
||||
keyDownstreamTimeout, DefaultDownstreamTimeout),
|
||||
ImageRequestsPerMinute: loader.intVal(
|
||||
keyImageRequestsPerMinute, DefaultImageRequestsPerMinute),
|
||||
CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
|
||||
BlockedNetworks: blockedNetworks,
|
||||
TrustedProxies: trustedProxies,
|
||||
RefererBlocklist: loader.hostListVal(keyRefererBlocklist),
|
||||
CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
|
||||
BlockedNetworks: blockedNetworks,
|
||||
TrustedProxies: trustedProxies,
|
||||
}
|
||||
|
||||
// The default for an omitted cache_max_bytes is worked out when
|
||||
// the cache opens; here we only record whether the operator set
|
||||
// the key explicitly.
|
||||
_, c.CacheMaxBytesExplicit = lookupValue(sc, keyCacheMaxBytes)
|
||||
// The computed default for cache_max_bytes needs a validated
|
||||
// state_dir, so it is resolved later (resolveCacheMaxBytes); here
|
||||
// we only record whether the operator set the key explicitly.
|
||||
if _, present := lookupValue(sc, keyCacheMaxBytes); present {
|
||||
c.cacheMaxBytesExplicit = true
|
||||
}
|
||||
|
||||
// Build DBURL from StateDir if not explicitly set. The derived URL
|
||||
// is a default: it applies only when db_url is omitted, never to an
|
||||
@@ -339,7 +315,7 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
settingName(keyDBURL), errValueEmpty)
|
||||
}
|
||||
|
||||
c.DBURL = defaultDBURL(c.StateDir)
|
||||
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_journal_mode=WAL", c.StateDir)
|
||||
}
|
||||
|
||||
if loader.err != nil {
|
||||
@@ -354,30 +330,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// defaultDBURL returns the db_url used when db_url is omitted: the file
|
||||
// state.sqlite3 in stateDir, in WAL mode, which the driver sets only
|
||||
// through a _pragma parameter. String() percent-encodes any %, ? or # in
|
||||
// the directory name, which SQLite decodes back, so the URL names exactly
|
||||
// this file. OmitHost writes file:/var/lib/pixa/... instead of
|
||||
// file:///var/lib/pixa/..., and keeps a relative stateDir from being read
|
||||
// as a host name. The path written into the URL is stateDir followed by
|
||||
// /state.sqlite3; when it starts with //, as it does for a stateDir of /
|
||||
// or one starting with //, the empty host is kept, so the URL starts
|
||||
// file://// and SQLite does not read the first name in the path as a
|
||||
// host.
|
||||
func defaultDBURL(stateDir string) string {
|
||||
dbPath := stateDir + "/state.sqlite3"
|
||||
|
||||
dbURL := url.URL{
|
||||
Scheme: "file",
|
||||
OmitHost: !strings.HasPrefix(dbPath, "//"),
|
||||
Path: dbPath,
|
||||
RawQuery: "_pragma=journal_mode(WAL)",
|
||||
}
|
||||
|
||||
return dbURL.String()
|
||||
}
|
||||
|
||||
// validateKnownKeys rejects configuration files containing keys the
|
||||
// application does not understand, so typos fail at startup instead of
|
||||
// being silently ignored, and rejects keys that are explicitly set to
|
||||
@@ -462,8 +414,7 @@ func isKnownConfigKey(key string) bool {
|
||||
keyUpstreamConnectionsPerHost, keyUpstreamConnections,
|
||||
keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks,
|
||||
keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout,
|
||||
keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist,
|
||||
keyImageRequestsPerMinute, "env":
|
||||
keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env":
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -486,7 +437,6 @@ func envVarNames() map[string]string {
|
||||
keyMetricsPassword: "PIXA_METRICS_PASSWORD",
|
||||
keySigningKey: "PIXA_SIGNING_KEY",
|
||||
keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS",
|
||||
keyRefererBlocklist: "PIXA_REFERER_BLOCKLIST",
|
||||
keyAllowHTTP: "PIXA_ALLOW_HTTP",
|
||||
keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
|
||||
keyUpstreamConnections: "PIXA_UPSTREAM_CONNECTIONS",
|
||||
@@ -498,7 +448,6 @@ func envVarNames() map[string]string {
|
||||
keyUpstreamFetchTimeout: "PIXA_UPSTREAM_FETCH_TIMEOUT",
|
||||
keyUpstreamMaxResponseSize: "PIXA_UPSTREAM_MAX_RESPONSE_SIZE",
|
||||
keyDownstreamTimeout: "PIXA_DOWNSTREAM_TIMEOUT",
|
||||
keyImageRequestsPerMinute: "PIXA_IMAGE_REQUESTS_PER_MINUTE",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -599,8 +548,8 @@ func (c *Config) ensureStateDirWritable() error {
|
||||
}
|
||||
|
||||
// validateSigningKey checks that the signing key is present, long
|
||||
// enough, and not the public placeholder from configs/config.example.yml.
|
||||
// The key value itself is never echoed in error messages.
|
||||
// enough, and not the public placeholder from config.example.yml. The
|
||||
// key value itself is never echoed in error messages.
|
||||
func (c *Config) validateSigningKey() error {
|
||||
if c.SigningKey == "" {
|
||||
return fmt.Errorf("%s: %w", settingName(keySigningKey), errValueRequired)
|
||||
@@ -651,29 +600,24 @@ func (c *Config) validate() error {
|
||||
settingName(keyCacheMaxBytes), c.CacheMaxBytes, errMustNotBeNegative)
|
||||
}
|
||||
|
||||
// Zero is valid (it turns the limit off); only negative values are
|
||||
// rejected.
|
||||
if c.ImageRequestsPerMinute < 0 {
|
||||
return fmt.Errorf("%s: value %d %w",
|
||||
settingName(keyImageRequestsPerMinute), c.ImageRequestsPerMinute,
|
||||
errMustNotBeNegative)
|
||||
}
|
||||
|
||||
err = c.validateUpstreamMaxResponseSize()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, host := range c.AllowlistHosts {
|
||||
err := validateHostPattern(keyAllowlistHosts, host)
|
||||
err := validateAllowlistHost(host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = c.validateSentryDSN()
|
||||
if err != nil {
|
||||
return err
|
||||
if c.SentryDSN != "" {
|
||||
parsed, err := url.Parse(c.SentryDSN)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return fmt.Errorf("%s: value %q is %w",
|
||||
settingName(keySentryDSN), c.SentryDSN, errNotAValidURL)
|
||||
}
|
||||
}
|
||||
|
||||
if (c.MetricsUsername == "") != (c.MetricsPassword == "") {
|
||||
@@ -685,22 +629,6 @@ func (c *Config) validate() error {
|
||||
return c.validateAccessControlAllowOrigin()
|
||||
}
|
||||
|
||||
// validateSentryDSN checks that sentry_dsn, when set, is a URL with a scheme
|
||||
// and a host.
|
||||
func (c *Config) validateSentryDSN() error {
|
||||
if c.SentryDSN == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(c.SentryDSN)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return fmt.Errorf("%s: value %q is %w",
|
||||
settingName(keySentryDSN), c.SentryDSN, errNotAValidURL)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateUpstreamMaxResponseSize checks that upstream_max_response_size
|
||||
// is from 1 byte to 1 GiB. An upstream response is read whole into
|
||||
// memory, and the image processor reads one byte past this limit, which
|
||||
@@ -801,24 +729,25 @@ func (c *Config) validateConcurrencyLimits() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// hostNamePattern matches a host name: letters, digits, hyphens, underscores
|
||||
// and dots, optionally after one leading dot.
|
||||
var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
|
||||
|
||||
// validateHostPattern checks that an entry of the named key, allowlist_hosts
|
||||
// or referer_blocklist, is an IP address or a host name, the host name
|
||||
// optionally with one leading dot for suffix matching. Anything else, such as
|
||||
// a URL, a port or a "*." wildcard, can never match a host name that resolves,
|
||||
// so it is refused.
|
||||
// So is "." alone: the allowlist matcher would match it against any host
|
||||
// written with a trailing dot, which in allowlist_hosts disables URL signing.
|
||||
func validateHostPattern(key, host string) error {
|
||||
_, err := netip.ParseAddr(host)
|
||||
if err == nil || hostNamePattern.MatchString(host) {
|
||||
return nil
|
||||
// validateAllowlistHost checks that an allowlist_hosts entry is a bare
|
||||
// hostname, optionally with a leading dot for suffix matching. URLs,
|
||||
// paths, and whitespace indicate a misconfigured entry. An entry with
|
||||
// no hostname labels (such as ".") is rejected: the allowlist matcher
|
||||
// treats a leading dot as a suffix pattern, so a bare "." would match
|
||||
// any upstream host written in FQDN trailing-dot form and effectively
|
||||
// disable URL signing.
|
||||
func validateAllowlistHost(host string) error {
|
||||
if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
|
||||
return fmt.Errorf("%s: entry %q %w",
|
||||
settingName(keyAllowlistHosts), host, errNotBareHostname)
|
||||
}
|
||||
|
||||
return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
|
||||
if strings.Trim(host, ".") == "" {
|
||||
return fmt.Errorf("%s: entry %q %w",
|
||||
settingName(keyAllowlistHosts), host, errNoHostnameLabels)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
|
||||
@@ -849,28 +778,19 @@ func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, erro
|
||||
for _, path := range configPaths {
|
||||
cleanPath := filepath.Clean(path)
|
||||
|
||||
// Only a config file that does not exist is skipped, including
|
||||
// one whose path runs through a file, such as under a HOME of
|
||||
// /dev/null. One that cannot be read or does not parse is a
|
||||
// fatal startup error.
|
||||
//nolint:gosec // G703: a fixed config location; HOME is the operator's
|
||||
_, statErr := os.Stat(cleanPath)
|
||||
if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
|
||||
continue
|
||||
if statErr == nil {
|
||||
// A config file that exists but does not parse is a fatal
|
||||
// startup error, never something to skip over.
|
||||
sc, err := smartconfig.NewFromConfigPath(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
|
||||
}
|
||||
|
||||
log.Info("loaded config file", "path", path)
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
if statErr != nil {
|
||||
return nil, fmt.Errorf("failed to read config file %s: %w", path, statErr)
|
||||
}
|
||||
|
||||
sc, err := smartconfig.NewFromConfigPath(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
|
||||
}
|
||||
|
||||
log.Info("loaded config file", "path", path)
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
return nil, nil //nolint:nilnil // nil config is valid (use defaults)
|
||||
@@ -949,19 +869,6 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
|
||||
return val
|
||||
}
|
||||
|
||||
func (l *strictLoader) hostListVal(key string) []string {
|
||||
if l.err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
val, err := parseHostList(l.sc, key)
|
||||
if err != nil {
|
||||
l.err = err
|
||||
}
|
||||
|
||||
return val
|
||||
}
|
||||
|
||||
// getString returns the string value for key, or defaultVal if the key
|
||||
// is omitted. A present value that is not a string, or is explicitly
|
||||
// null, is an error.
|
||||
@@ -1249,13 +1156,6 @@ func defaultTrustedProxies() []netip.Prefix {
|
||||
// wrong type, an empty entry, a non-string entry, or an unparseable CIDR
|
||||
// aborts startup naming the key and the offending value; the default
|
||||
// (an empty list) applies only to an omitted key.
|
||||
//
|
||||
// An entry written in the IPv4-mapped form (::ffff:a.b.c.d) is kept as the
|
||||
// IPv4 range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. Both
|
||||
// blocked_networks and trusted_proxies are matched against an IPv4-mapped
|
||||
// address as the IPv4 address it holds, so the entry as written would match
|
||||
// nothing. Such an entry shorter than /96 covers more than the IPv4
|
||||
// addresses and aborts startup.
|
||||
func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
raw, ok := lookupValue(sc, key)
|
||||
if !ok {
|
||||
@@ -1266,15 +1166,11 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
return nil, errNullConfigValue(key)
|
||||
}
|
||||
|
||||
entries, err := listEntries(raw, key)
|
||||
entries, err := cidrListEntries(raw, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// An IPv4-mapped address is the 96 bits of ::ffff:0:0/96 followed by
|
||||
// the 32 bits of the IPv4 address it holds.
|
||||
const mappedPrefixBits = 96
|
||||
|
||||
prefixes := make([]netip.Prefix, 0, len(entries))
|
||||
|
||||
for _, entry := range entries {
|
||||
@@ -1284,57 +1180,17 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
settingName(key), entry, errNotAValidCIDR)
|
||||
}
|
||||
|
||||
if prefix.Addr().Is4In6() && prefix.Bits() < mappedPrefixBits {
|
||||
return nil, fmt.Errorf("%s: value %q is %w",
|
||||
settingName(key), entry, errMappedTooShort)
|
||||
}
|
||||
|
||||
if prefix.Addr().Is4In6() {
|
||||
prefix = netip.PrefixFrom(prefix.Addr().Unmap(),
|
||||
prefix.Bits()-mappedPrefixBits)
|
||||
}
|
||||
|
||||
prefixes = append(prefixes, prefix)
|
||||
}
|
||||
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
// parseHostList parses the value of the named config key into host patterns,
|
||||
// or returns nil if the key is omitted. It accepts a YAML list of strings or a
|
||||
// comma-separated string. An explicitly null value, a wrong type, an empty
|
||||
// entry, a non-string entry, or an entry validateHostPattern rejects aborts
|
||||
// startup naming the key and the offending value.
|
||||
func parseHostList(sc *smartconfig.Config, key string) ([]string, error) {
|
||||
raw, ok := lookupValue(sc, key)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if raw == nil {
|
||||
return nil, errNullConfigValue(key)
|
||||
}
|
||||
|
||||
entries, err := listEntries(raw, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
err := validateHostPattern(key, entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// listEntries extracts the raw entries of the named list key as trimmed,
|
||||
// non-empty strings, from either a YAML list of strings or a comma-separated
|
||||
// string; an empty string is an empty list, as for allowlist_hosts. Any other
|
||||
// shape is a configuration error.
|
||||
func listEntries(raw any, key string) ([]string, error) {
|
||||
// cidrListEntries extracts the raw entries of the named CIDR-list key as
|
||||
// trimmed, non-empty strings, from either a YAML list of strings or a
|
||||
// comma-separated string; an empty string is an empty list, as for
|
||||
// allowlist_hosts. Any other shape is a configuration error.
|
||||
func cidrListEntries(raw any, key string) ([]string, error) {
|
||||
switch val := raw.(type) {
|
||||
case []any:
|
||||
entries := make([]string, 0, len(val))
|
||||
|
||||
@@ -92,32 +92,6 @@ func TestGetStringSlice_Empty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestExampleConfigAllowlistsNoHost loads configs/config.example.yml
|
||||
// with a real signing key, as Getting Started in README.md has it copied
|
||||
// and edited, and checks that it allowlists no host: every image on an
|
||||
// allowlisted host is served to anyone without a signature.
|
||||
func TestExampleConfigAllowlistsNoHost(t *testing.T) {
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
|
||||
examplePath := filepath.Join("..", "..", "configs", "config.example.yml")
|
||||
|
||||
sc, err := loadTestConfig(examplePath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load %s: %v", examplePath, err)
|
||||
}
|
||||
|
||||
c, err := newFromSmartConfig(sc)
|
||||
if err != nil {
|
||||
t.Fatalf("%s is not a valid config: %v", examplePath, err)
|
||||
}
|
||||
|
||||
t.Logf("allowlist_hosts in %s: %v", examplePath, c.AllowlistHosts)
|
||||
|
||||
if len(c.AllowlistHosts) != 0 {
|
||||
t.Errorf("%s allowlists %v, want no host", examplePath, c.AllowlistHosts)
|
||||
}
|
||||
}
|
||||
|
||||
// loadTestConfig is a helper to load a config file for testing.
|
||||
func loadTestConfig(path string) (*smartconfig.Config, error) {
|
||||
return smartconfig.NewFromConfigPath(path)
|
||||
|
||||
@@ -1,18 +1,14 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
|
||||
_ "modernc.org/sqlite" // SQLite driver registration
|
||||
)
|
||||
|
||||
// validTestSigningKey is a 32-character signing key that satisfies the
|
||||
@@ -98,141 +94,12 @@ func TestOmittedValuesUseDefaults(t *testing.T) {
|
||||
t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts)
|
||||
}
|
||||
|
||||
wantDBURL := "file:" + DefaultStateDir +
|
||||
"/state.sqlite3?_pragma=journal_mode(WAL)"
|
||||
wantDBURL := "file:" + DefaultStateDir + "/state.sqlite3?_journal_mode=WAL"
|
||||
if c.DBURL != wantDBURL {
|
||||
t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultDBURLOpensTheDatabaseInWALMode opens the db_url derived from
|
||||
// state_dir with the SQLite driver pixad uses and checks that the database
|
||||
// is in WAL mode: the driver ignores any parameter it does not know.
|
||||
func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+t.TempDir()+"\n")
|
||||
if err != nil {
|
||||
t.Fatalf("config with only state_dir set should be valid, got: %v", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", c.DBURL)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open %q: %v", c.DBURL, err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
var journalMode string
|
||||
|
||||
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the journal mode of %q: %v", c.DBURL, err)
|
||||
}
|
||||
|
||||
if journalMode != "wal" {
|
||||
t.Errorf("journal mode of %q = %q, want wal", c.DBURL, journalMode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDerivedDBURLOpensStateDirHoldingURLCharacters opens the db_url
|
||||
// derived from a state_dir whose name holds %41, ? and #, and checks that
|
||||
// the database is state.sqlite3 in that directory, in WAL mode. Put into
|
||||
// a file: URL as it is, the name would lose its meaning: %41 is read as
|
||||
// A, and the path ends at the ?, which also loses the journal mode.
|
||||
func TestDerivedDBURLOpensStateDirHoldingURLCharacters(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stateDir := filepath.Join(t.TempDir(), "a%41b?c#d")
|
||||
|
||||
err := os.Mkdir(stateDir, 0o750)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create %q: %v", stateDir, err)
|
||||
}
|
||||
|
||||
// Quoted so YAML takes the whole name as it is.
|
||||
c, err := configFromYAML(t, signingKeyLine+"state_dir: '"+stateDir+"'\n")
|
||||
if err != nil {
|
||||
t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
|
||||
}
|
||||
|
||||
t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
|
||||
|
||||
checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(stateDir, "state.sqlite3"))
|
||||
}
|
||||
|
||||
// TestDerivedDBURLOpensStateDirStartingWithTwoSlashes opens the db_url
|
||||
// derived from a state_dir that starts with //, and checks that the
|
||||
// database is state.sqlite3 in that directory, in WAL mode. Written right
|
||||
// after file:, such a name gives a URL starting file://, and SQLite reads
|
||||
// the first directory name as a host and refuses to open it.
|
||||
func TestDerivedDBURLOpensStateDirStartingWithTwoSlashes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
stateDir := "/" + dir
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+stateDir+"\n")
|
||||
if err != nil {
|
||||
t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
|
||||
}
|
||||
|
||||
t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
|
||||
|
||||
checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(dir, "state.sqlite3"))
|
||||
}
|
||||
|
||||
// TestDerivedDBURLForRootStateDir checks the db_url derived from a
|
||||
// state_dir of /. The path written into the URL is then
|
||||
// //state.sqlite3, and written right after file: it gives a URL starting
|
||||
// file://, where SQLite reads state.sqlite3 as a host and refuses to open
|
||||
// it. The test only checks the URL, since it cannot write to /.
|
||||
func TestDerivedDBURLForRootStateDir(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+"state_dir: /\n")
|
||||
if err != nil {
|
||||
t.Fatalf("config with state_dir / should be valid, got: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("state_dir / gives db_url %q", c.DBURL)
|
||||
|
||||
wantDBURL := "file:////state.sqlite3?_pragma=journal_mode(WAL)"
|
||||
if c.DBURL != wantDBURL {
|
||||
t.Errorf("DBURL = %q, want %q", c.DBURL, wantDBURL)
|
||||
}
|
||||
}
|
||||
|
||||
// checkDBURLOpensFileInWALMode opens dbURL with the SQLite driver pixad
|
||||
// uses, and checks that the database is in WAL mode and is the file
|
||||
// dbFile.
|
||||
func checkDBURLOpensFileInWALMode(t *testing.T, dbURL, dbFile string) {
|
||||
t.Helper()
|
||||
|
||||
db, err := sql.Open("sqlite", dbURL)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open %q: %v", dbURL, err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
var journalMode string
|
||||
|
||||
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the journal mode of %q: %v", dbURL, err)
|
||||
}
|
||||
|
||||
if journalMode != "wal" {
|
||||
t.Errorf("journal mode of %q = %q, want wal", dbURL, journalMode)
|
||||
}
|
||||
|
||||
_, err = os.Stat(dbFile)
|
||||
if err != nil {
|
||||
t.Errorf("opening %q did not create %q: %v", dbURL, dbFile, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitValidValuesAreUsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -315,25 +182,6 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with
|
||||
// an underscore, which pixa can fetch from, is accepted as an entry.
|
||||
func TestAllowlistHostsAcceptsUnderscore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts:
|
||||
- my_bucket.example.com
|
||||
- .my_bucket.example.org
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("host names with an underscore should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"my_bucket.example.com", ".my_bucket.example.org"}
|
||||
if !slices.Equal(c.AllowlistHosts, want) {
|
||||
t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want)
|
||||
}
|
||||
}
|
||||
|
||||
// runAbortCases asserts that each case's config aborts startup with an
|
||||
// error message mentioning every expected substring.
|
||||
func runAbortCases(t *testing.T, cases []abortCase) {
|
||||
@@ -436,20 +284,6 @@ func invalidHostAndCredentialCases() []abortCase {
|
||||
keyAllowlistHosts, "example.com/images",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with wildcard",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyAllowlistHosts, "*.example.com",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with port",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyAllowlistHosts, "example.com:8443",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with whitespace",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
|
||||
@@ -730,116 +564,6 @@ func TestMalformedConfigFileAbortsStartup(t *testing.T) {
|
||||
t.Logf("got expected error: %v", err)
|
||||
}
|
||||
|
||||
// TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup checks that a
|
||||
// config file pixa cannot read because it may not enter its directory
|
||||
// aborts startup instead of being passed over.
|
||||
func TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root may enter any directory")
|
||||
}
|
||||
|
||||
home := t.TempDir()
|
||||
configDir := filepath.Join(home, ".config", "pixa-test-nonexistent-app")
|
||||
configPath := filepath.Join(configDir, "config.yml")
|
||||
|
||||
err := os.MkdirAll(configDir, 0o700)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create config directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(configPath, []byte(signingKeyLine), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
}
|
||||
|
||||
err = os.Chmod(configDir, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to remove the config directory's permissions: %v", err)
|
||||
}
|
||||
|
||||
// Give the directory back its permissions so t.TempDir can remove it.
|
||||
t.Cleanup(func() {
|
||||
//nolint:gosec // G302: a directory needs its execute bit to be removed
|
||||
_ = os.Chmod(configDir, 0o700)
|
||||
})
|
||||
|
||||
// The ~/.config candidate is the only one that exists: the appname
|
||||
// rules out /etc, and the working directory is empty.
|
||||
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||
t.Setenv("HOME", home)
|
||||
t.Chdir(t.TempDir())
|
||||
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||
if err == nil {
|
||||
t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
|
||||
sc)
|
||||
}
|
||||
|
||||
t.Logf("got expected error: %v", err)
|
||||
|
||||
if !strings.Contains(err.Error(), configPath) {
|
||||
t.Errorf("error %q does not name the config file %s", err.Error(), configPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigFileLinkingToItselfAbortsStartup checks that a config file
|
||||
// pixa cannot read for a reason other than not existing aborts startup,
|
||||
// as root too: a symbolic link to itself fails with "too many levels of
|
||||
// symbolic links".
|
||||
func TestConfigFileLinkingToItselfAbortsStartup(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
|
||||
err := os.Symlink("config.yml", filepath.Join(workDir, "config.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create symbolic link: %v", err)
|
||||
}
|
||||
|
||||
// Only the working directory's config.yml is there: the appname rules
|
||||
// out /etc, and HOME is empty.
|
||||
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||
t.Setenv("HOME", t.TempDir())
|
||||
t.Chdir(workDir)
|
||||
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||
if err == nil {
|
||||
t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
|
||||
sc)
|
||||
}
|
||||
|
||||
t.Logf("got expected error: %v", err)
|
||||
|
||||
if !strings.Contains(err.Error(), "config.yml") {
|
||||
t.Errorf("error %q does not name the config file config.yml", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigPathThroughFileIsPassedOver checks that a config file path
|
||||
// that runs through a file, such as one under a HOME of /dev/null, is
|
||||
// passed over like one that does not exist, since no file can be there.
|
||||
func TestConfigPathThroughFileIsPassedOver(t *testing.T) {
|
||||
// No config file is there: the appname rules out /etc, HOME is
|
||||
// /dev/null, and the working directory is empty.
|
||||
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||
t.Setenv("HOME", os.DevNull)
|
||||
t.Chdir(t.TempDir())
|
||||
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||
if err != nil {
|
||||
t.Fatalf("a config path through a file must be passed over, got error: %v",
|
||||
err)
|
||||
}
|
||||
|
||||
if sc != nil {
|
||||
t.Errorf("expected no config file, got config: %v", sc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureStateDirCreatesDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -65,12 +65,10 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
|
||||
t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example")
|
||||
t.Setenv("PIXA_ALLOW_HTTP", "true")
|
||||
t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
|
||||
t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
|
||||
t.Setenv("PIXA_MAX_CONCURRENT_PROCESSING", "3")
|
||||
t.Setenv("PIXA_IMAGE_REQUESTS_PER_MINUTE", "30")
|
||||
t.Setenv("PIXA_CACHE_MAX_BYTES", "1024")
|
||||
t.Setenv("PIXA_BLOCKED_NETWORKS", "203.0.113.0/24")
|
||||
t.Setenv("PIXA_TRUSTED_PROXIES", "192.0.2.0/24")
|
||||
@@ -95,14 +93,12 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
|
||||
MetricsPassword: "metricspass",
|
||||
SigningKey: validTestSigningKey,
|
||||
AllowlistHosts: []string{testHostS3, ".example.com"},
|
||||
RefererBlocklist: []string{"hotlinker.example", ".leech.example"},
|
||||
AllowHTTP: true,
|
||||
UpstreamConnectionsPerHost: 5,
|
||||
UpstreamConnections: 10,
|
||||
MaxConcurrentProcessing: 3,
|
||||
ImageRequestsPerMinute: 30,
|
||||
CacheMaxBytes: 1024,
|
||||
CacheMaxBytesExplicit: true,
|
||||
cacheMaxBytesExplicit: true,
|
||||
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")},
|
||||
TrustedProxies: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
|
||||
AccessControlAllowOrigin: "https://app.example.com",
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
// testImageRequestsPerMinuteVar is the variable that sets
|
||||
// image_requests_per_minute.
|
||||
const testImageRequestsPerMinuteVar = "PIXA_IMAGE_REQUESTS_PER_MINUTE"
|
||||
|
||||
// TestImageRequestsPerMinuteValues checks that an omitted
|
||||
// image_requests_per_minute is 600, and that 0, which turns the limit off,
|
||||
// and any larger whole number are used as given.
|
||||
func TestImageRequestsPerMinuteValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine)
|
||||
if err != nil {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
if c.ImageRequestsPerMinute != 600 {
|
||||
t.Errorf("ImageRequestsPerMinute = %d, want 600", c.ImageRequestsPerMinute)
|
||||
}
|
||||
|
||||
explicit := []struct {
|
||||
value string
|
||||
want int
|
||||
}{
|
||||
{"0", 0},
|
||||
{"30", 30},
|
||||
}
|
||||
|
||||
for _, tc := range explicit {
|
||||
c, err := configFromYAML(t,
|
||||
signingKeyLine+"image_requests_per_minute: "+tc.value+"\n")
|
||||
if err != nil {
|
||||
t.Fatalf("image_requests_per_minute: %s should load, got error: %v",
|
||||
tc.value, err)
|
||||
}
|
||||
|
||||
if c.ImageRequestsPerMinute != tc.want {
|
||||
t.Errorf("ImageRequestsPerMinute = %d, want %d",
|
||||
c.ImageRequestsPerMinute, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvalidImageRequestsPerMinuteAbortsStartup checks that a value that is
|
||||
// negative, not a whole number, or null aborts startup naming the key and
|
||||
// the value, and the variable too where the value could have come from it.
|
||||
func TestInvalidImageRequestsPerMinuteAbortsStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := keyImageRequestsPerMinute
|
||||
|
||||
runAbortCases(t, []abortCase{
|
||||
{
|
||||
name: "negative",
|
||||
yaml: signingKeyLine + key + ": -1\n",
|
||||
wantErrSubstrings: []string{
|
||||
key, testImageRequestsPerMinuteVar, "value -1",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "not a number",
|
||||
yaml: signingKeyLine + key + ": plenty\n",
|
||||
wantErrSubstrings: []string{
|
||||
key, testImageRequestsPerMinuteVar, "plenty",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fractional",
|
||||
yaml: signingKeyLine + key + ": 2.5\n",
|
||||
wantErrSubstrings: []string{key, "2.5"},
|
||||
},
|
||||
{
|
||||
name: "explicit null",
|
||||
yaml: signingKeyLine + key + ": null\n",
|
||||
wantErrSubstrings: []string{key, nullValueText},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestImageRequestsPerMinuteFromEnvironment checks that the variable sets the
|
||||
// limit over the config file, and that an invalid value in it aborts startup
|
||||
// naming the variable and the value.
|
||||
func TestImageRequestsPerMinuteFromEnvironment(t *testing.T) {
|
||||
t.Setenv(testImageRequestsPerMinuteVar, "0")
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+"image_requests_per_minute: 30\n")
|
||||
if err != nil {
|
||||
t.Fatalf("limit from the environment should load: %v", err)
|
||||
}
|
||||
|
||||
if c.ImageRequestsPerMinute != 0 {
|
||||
t.Errorf("ImageRequestsPerMinute = %d, want 0 from the environment",
|
||||
c.ImageRequestsPerMinute)
|
||||
}
|
||||
|
||||
for _, value := range []string{"-5", "ten"} {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
t.Setenv(testImageRequestsPerMinuteVar, value)
|
||||
|
||||
_, err := configFromYAML(t, signingKeyLine)
|
||||
wantStartupError(t, err, testImageRequestsPerMinuteVar, value)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,166 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
|
||||
// pattern starting with "." and checks both are kept in order.
|
||||
func TestRefererBlocklistParsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- leech.example
|
||||
- .hotlinker.example
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("valid referer_blocklist should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"leech.example", ".hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
|
||||
// the IPv6 one written without brackets, are accepted as entries.
|
||||
func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- 192.0.2.7
|
||||
- "2001:db8::7"
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("IP address entries should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"192.0.2.7", "2001:db8::7"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistAcceptsUnderscore checks that a host name with an
|
||||
// underscore, which a page can be served from, is accepted as an entry.
|
||||
func TestRefererBlocklistAcceptsUnderscore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- my_site.leech.example
|
||||
- .my_site.hotlinker.example
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("host names with an underscore should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"my_site.leech.example", ".my_site.hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
|
||||
// referer.
|
||||
func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine)
|
||||
if err != nil {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
if len(c.RefererBlocklist) != 0 {
|
||||
t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
|
||||
// host, or a value that is not a list of them, aborts startup with an error
|
||||
// naming the key and the entry.
|
||||
func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runAbortCases(t, []abortCase{
|
||||
{
|
||||
name: "entry with a scheme",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - https://leech.example\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "https://leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "entry with a path",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example/page\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "leech.example/page",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wildcard entry",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "*.leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "entry with a port",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "leech.example:8080",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "two leading dots",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "..leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dot only",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
|
||||
},
|
||||
{
|
||||
name: "empty entry",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \"\"\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist},
|
||||
},
|
||||
{
|
||||
name: "entry not a string",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - 42\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, "42"},
|
||||
},
|
||||
{
|
||||
name: "null value",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
|
||||
// takes comma-separated entries, and that an entry in it that is not a host
|
||||
// aborts startup naming the variable and the entry.
|
||||
func TestRefererBlocklistFromEnvironment(t *testing.T) {
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
|
||||
|
||||
c, err := newFromSmartConfig(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"leech.example", ".hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
|
||||
|
||||
_, err = newFromSmartConfig(nil)
|
||||
wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
|
||||
}
|
||||
@@ -83,41 +83,3 @@ func TestTrustedProxiesInvalidAbortsStartup(t *testing.T) {
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestTrustedProxiesMappedEntryIsIPv4Range checks that an entry written in
|
||||
// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
|
||||
// since the client address is matched as the IPv4 address it holds. /96, the
|
||||
// shortest such entry, covers every IPv4 address.
|
||||
func TestTrustedProxiesMappedEntryIsIPv4Range(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+
|
||||
`trusted_proxies: ["::ffff:10.0.0.0/104", "::ffff:0:0/96"]`+"\n")
|
||||
if err != nil {
|
||||
t.Fatalf("IPv4-mapped trusted_proxies entries should load: %v", err)
|
||||
}
|
||||
|
||||
got := make([]string, len(c.TrustedProxies))
|
||||
for i, p := range c.TrustedProxies {
|
||||
got[i] = p.String()
|
||||
}
|
||||
|
||||
if joined := strings.Join(got, ","); joined != "10.0.0.0/8,0.0.0.0/0" {
|
||||
t.Errorf("TrustedProxies = %v, want 10.0.0.0/8 and 0.0.0.0/0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrustedProxiesMappedEntryShorterThan96AbortsStartup checks that an
|
||||
// entry in the IPv4-mapped form shorter than /96, which covers more than the
|
||||
// IPv4 addresses, aborts startup with an error naming the key and the value.
|
||||
func TestTrustedProxiesMappedEntryShorterThan96AbortsStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runAbortCases(t, []abortCase{
|
||||
{
|
||||
name: "mapped-95",
|
||||
yaml: signingKeyLine + `trusted_proxies: ["::ffff:0:0/95"]` + "\n",
|
||||
wantErrSubstrings: []string{keyTrustedProxies, "::ffff:0:0/95"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
)
|
||||
|
||||
// TestConcurrentWritesAllSucceed opens a database the way pixad does and
|
||||
// writes to it from several goroutines at once, as one request's writes and
|
||||
// the background eviction pass do. Every write must succeed, none failing
|
||||
// with "database is locked", whether or not db_url already has parameters,
|
||||
// and the parameters it has must still apply.
|
||||
func TestConcurrentWritesAllSucceed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantJournalMode string
|
||||
}{
|
||||
{
|
||||
name: "db_url without parameters",
|
||||
query: "",
|
||||
wantJournalMode: "delete",
|
||||
},
|
||||
{
|
||||
name: "db_url with the WAL parameter",
|
||||
query: "?_pragma=journal_mode(WAL)",
|
||||
wantJournalMode: "wal",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dbURL := "file:" + filepath.Join(t.TempDir(), "state.sqlite3") + tt.query
|
||||
|
||||
d := &Database{
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
config: &config.Config{DBURL: dbURL},
|
||||
}
|
||||
|
||||
err := d.connect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to connect to %q: %v", dbURL, err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = d.db.Close() })
|
||||
|
||||
writeConcurrently(t, d.db)
|
||||
|
||||
var journalMode string
|
||||
|
||||
err = d.db.QueryRowContext(t.Context(), "PRAGMA journal_mode").
|
||||
Scan(&journalMode)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the journal mode: %v", err)
|
||||
}
|
||||
|
||||
if journalMode != tt.wantJournalMode {
|
||||
t.Errorf("journal mode = %q, want %q", journalMode, tt.wantJournalMode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeConcurrently runs writeLikeOneRequest from several goroutines at once
|
||||
// and checks that every write was made.
|
||||
func writeConcurrently(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
const (
|
||||
writers = 4
|
||||
requestsEach = 20
|
||||
totalRequests = writers * requestsEach
|
||||
)
|
||||
|
||||
ctx := t.Context()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for writer := range writers {
|
||||
wg.Go(func() {
|
||||
for request := range requestsEach {
|
||||
key := fmt.Sprintf("%d-%d", writer, request)
|
||||
|
||||
err := writeLikeOneRequest(ctx, db, key)
|
||||
if err != nil {
|
||||
t.Errorf("writer %d: %v", writer, err)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
var hits, sources int
|
||||
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT hit_count, (SELECT COUNT(*) FROM source_content)
|
||||
FROM cache_stats WHERE id = 1
|
||||
`).Scan(&hits, &sources)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count the writes: %v", err)
|
||||
}
|
||||
|
||||
if hits != totalRequests || sources != totalRequests {
|
||||
t.Errorf("hit_count = %d and %d source_content rows, want %d of each",
|
||||
hits, sources, totalRequests)
|
||||
}
|
||||
}
|
||||
|
||||
// writeLikeOneRequest makes the writes one request and the eviction pass
|
||||
// make: it counts a cache hit, stores a source, records a transformed image
|
||||
// and deletes that record again.
|
||||
func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error {
|
||||
_, err := db.ExecContext(ctx,
|
||||
`UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("counting a cache hit: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO source_content
|
||||
(source_key, source_host, source_path, content_type, size_bytes)
|
||||
VALUES (?, 'example.com', '/image.png', 'image/png', 1)`, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storing a source: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO variant_content
|
||||
(cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recording a transformed image: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("evicting a transformed image: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -237,24 +237,13 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DB returns the underlying sql.DB. It has one connection, so close any
|
||||
// rows and end any transaction before running another query on it; a
|
||||
// query run while they are open waits forever.
|
||||
// DB returns the underlying sql.DB.
|
||||
func (s *Database) DB() *sql.DB {
|
||||
return s.db
|
||||
}
|
||||
|
||||
func (s *Database) connect(ctx context.Context) error {
|
||||
// With a busy timeout, a write that finds another program writing to
|
||||
// the same database file waits up to five seconds for it instead of
|
||||
// failing at once with "database is locked". The driver runs each
|
||||
// _pragma parameter on every connection it opens.
|
||||
separator := "?"
|
||||
if strings.Contains(s.config.DBURL, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
|
||||
dbURL := s.config.DBURL + separator + "_pragma=busy_timeout(5000)"
|
||||
dbURL := s.config.DBURL
|
||||
|
||||
s.log.Info("connecting to database", "url", dbURL)
|
||||
|
||||
@@ -265,11 +254,6 @@ func (s *Database) connect(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// One connection: pixa's own reads and writes run on it one at a
|
||||
// time instead of competing for SQLite's lock, where a write that
|
||||
// keeps losing can wait past the busy timeout and be lost.
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
err = db.PingContext(ctx)
|
||||
if err != nil {
|
||||
s.log.Error("failed to ping database", "error", err)
|
||||
|
||||
@@ -160,7 +160,7 @@ func TestApplyMigrations_CreatesSchemaAndTables(t *testing.T) {
|
||||
|
||||
// Verify that the application tables created by 001.sql exist.
|
||||
tables := []string{
|
||||
"source_content", "output_content",
|
||||
"source_content", "source_metadata", "output_content",
|
||||
"request_cache", "negative_cache", "cache_stats",
|
||||
}
|
||||
for _, table := range tables {
|
||||
|
||||
@@ -1,29 +1,50 @@
|
||||
-- Migration 001: Initial schema
|
||||
-- Creates all tables for the pixa caching image proxy
|
||||
|
||||
-- Source images, one row per source URL
|
||||
-- Files stored at: cache/sources/<ab>/<cd>/<source_key>, where source_key
|
||||
-- is the SHA-256 of the source's host, path and query
|
||||
-- Source content blobs
|
||||
-- Files stored at: cache/src-content/<ab>/<cd>/<sha256>
|
||||
-- last_accessed_at is NULL until the first LRU touch; eviction falls
|
||||
-- back to fetched_at for rows that have never been touched.
|
||||
CREATE TABLE IF NOT EXISTS source_content (
|
||||
source_key TEXT PRIMARY KEY,
|
||||
source_host TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_query TEXT NOT NULL DEFAULT '',
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
last_accessed_at DATETIME
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
|
||||
ON source_content(last_accessed_at);
|
||||
|
||||
-- Source URL metadata - maps URLs to content hashes
|
||||
-- JSON stored at: cache/src-metadata/<hostname>/<path_hash>.json
|
||||
CREATE TABLE IF NOT EXISTS source_metadata (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_host TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_query TEXT NOT NULL DEFAULT '',
|
||||
path_hash TEXT NOT NULL,
|
||||
content_hash TEXT,
|
||||
status_code INTEGER NOT NULL,
|
||||
content_type TEXT,
|
||||
response_headers TEXT,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at DATETIME,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
UNIQUE(source_host, source_path, source_query),
|
||||
FOREIGN KEY (content_hash) REFERENCES source_content(content_hash)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_host ON source_metadata(source_host);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_path_hash ON source_metadata(path_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_expires ON source_metadata(expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(content_hash);
|
||||
|
||||
-- Processed variant blobs
|
||||
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key>. Tracked here
|
||||
-- (like source content blobs above) so total cache usage is known
|
||||
-- without a directory scan, and so LRU eviction has a timestamp to order
|
||||
-- on. A cache hit does not read this table: it takes the content type
|
||||
-- from the file's first bytes, and its last_accessed_at is written
|
||||
-- later, in a batch.
|
||||
-- Files stored at: cache/variants/<ab>/<cd>/<cache_key> (plus a .meta
|
||||
-- sidecar with the content type). Tracked here (like source content
|
||||
-- blobs above) so total cache usage can be computed with a SUM query,
|
||||
-- never a directory scan, and so LRU eviction has a timestamp to order
|
||||
-- on.
|
||||
CREATE TABLE IF NOT EXISTS variant_content (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
@@ -34,77 +55,8 @@ CREATE TABLE IF NOT EXISTS variant_content (
|
||||
CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
|
||||
ON variant_content(last_accessed_at);
|
||||
|
||||
-- Total cache usage: the sum of size_bytes over source_content and
|
||||
-- variant_content, kept by the triggers below in the same statement
|
||||
-- that adds, removes or resizes a row, so eviction reads this one row
|
||||
-- instead of summing both tables. Each trigger also adds one to
|
||||
-- change_count; the reconciliation pass, which sums both tables a page
|
||||
-- at a time, corrects total_size_bytes only if change_count did not
|
||||
-- move while it summed.
|
||||
CREATE TABLE IF NOT EXISTS cache_usage (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
total_size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
change_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT OR IGNORE INTO cache_usage (id) VALUES (1);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_insert
|
||||
AFTER INSERT ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_delete
|
||||
AFTER DELETE ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS source_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON source_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_insert
|
||||
AFTER INSERT ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_delete
|
||||
AFTER DELETE ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS variant_content_usage_update
|
||||
AFTER UPDATE OF size_bytes ON variant_content
|
||||
BEGIN
|
||||
UPDATE cache_usage
|
||||
SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
|
||||
change_count = change_count + 1
|
||||
WHERE id = 1;
|
||||
END;
|
||||
|
||||
-- Output/transformed content blobs
|
||||
-- Not written: transformed images are stored in cache/variants and
|
||||
-- tracked in variant_content above.
|
||||
-- Files stored at: cache/dst-content/<ab>/<cd>/<sha256>
|
||||
CREATE TABLE IF NOT EXISTS output_content (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL,
|
||||
@@ -158,11 +110,3 @@ CREATE TABLE IF NOT EXISTS cache_stats (
|
||||
last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
INSERT OR IGNORE INTO cache_stats (id) VALUES (1);
|
||||
|
||||
-- A random value that a login session carries from its login. pixa makes it
|
||||
-- when the table has no row, and each logout replaces it; a login session
|
||||
-- carrying another value is refused, so logging out ends every login session.
|
||||
CREATE TABLE IF NOT EXISTS logout (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
logout_value TEXT NOT NULL
|
||||
);
|
||||
|
||||
+65
-52
@@ -7,15 +7,15 @@ import (
|
||||
|
||||
"github.com/fxamacker/cbor/v2"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/seal"
|
||||
)
|
||||
|
||||
// Default values for optional fields.
|
||||
const (
|
||||
DefaultQuality = 85
|
||||
DefaultFormat = imagerequest.FormatJXL
|
||||
DefaultFitMode = imagerequest.FitCover
|
||||
DefaultFormat = imgcache.FormatOriginal
|
||||
DefaultFitMode = imgcache.FitCover
|
||||
|
||||
// HKDF salt for URL encryption key derivation
|
||||
urlKeySalt = "pixa-urlenc-v1"
|
||||
@@ -28,19 +28,19 @@ var (
|
||||
ErrDecryptFailed = errors.New("failed to decrypt URL")
|
||||
)
|
||||
|
||||
// payload contains all image request parameters in a compact format.
|
||||
// Payload contains all image request parameters in a compact format.
|
||||
// Field names are short to minimize encoded size.
|
||||
// Fields with omitempty use sane defaults when absent.
|
||||
type payload struct {
|
||||
SourceHost string `cbor:"h"` // required
|
||||
SourcePath string `cbor:"p"` // required
|
||||
SourceQuery string `cbor:"q,omitempty"` // optional
|
||||
Width int `cbor:"w,omitempty"` // 0 = original
|
||||
Height int `cbor:"ht,omitempty"` // 0 = original
|
||||
Format imagerequest.Format `cbor:"f,omitempty"` // default: jxl
|
||||
Quality int `cbor:"ql,omitempty"` // default: 85
|
||||
FitMode imagerequest.Fit `cbor:"fm,omitempty"` // default: cover
|
||||
ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
|
||||
type Payload struct {
|
||||
SourceHost string `cbor:"h"` // required
|
||||
SourcePath string `cbor:"p"` // required
|
||||
SourceQuery string `cbor:"q,omitempty"` // optional
|
||||
Width int `cbor:"w,omitempty"` // 0 = original
|
||||
Height int `cbor:"ht,omitempty"` // 0 = original
|
||||
Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: orig
|
||||
Quality int `cbor:"ql,omitempty"` // default: 85
|
||||
FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover
|
||||
ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
|
||||
}
|
||||
|
||||
// Generator creates and parses encrypted URL tokens.
|
||||
@@ -59,27 +59,9 @@ func NewGenerator(signingKey string) (*Generator, error) {
|
||||
return &Generator{key: key}, nil
|
||||
}
|
||||
|
||||
// Generate creates an encrypted URL token from the request's source,
|
||||
// transform and expiry, each as the request holds it; an empty format, a
|
||||
// quality of 0 or an empty fit mode is left out of the token, and a zero
|
||||
// Expires makes a token that never expires. The token is CBOR-encoded,
|
||||
// encrypted with NaCl secretbox, and base64url-encoded.
|
||||
func (g *Generator) Generate(req *imagerequest.Request) (string, error) {
|
||||
p := &payload{
|
||||
SourceHost: req.Source.Host,
|
||||
SourcePath: req.Source.Path,
|
||||
SourceQuery: req.Source.Query,
|
||||
Width: req.Transform.Size.Width,
|
||||
Height: req.Transform.Size.Height,
|
||||
Format: req.Transform.Format,
|
||||
Quality: req.Transform.Quality,
|
||||
FitMode: req.Transform.FitMode,
|
||||
}
|
||||
|
||||
if !req.Expires.IsZero() {
|
||||
p.ExpiresAt = req.Expires.Unix()
|
||||
}
|
||||
|
||||
// Generate creates an encrypted URL token from the payload.
|
||||
// The token is CBOR-encoded, encrypted with NaCl secretbox, and base64url-encoded.
|
||||
func (g *Generator) Generate(p *Payload) (string, error) {
|
||||
// CBOR encode the payload
|
||||
data, err := cbor.Marshal(p)
|
||||
if err != nil {
|
||||
@@ -90,11 +72,9 @@ func (g *Generator) Generate(req *imagerequest.Request) (string, error) {
|
||||
return seal.Encrypt(g.key, data)
|
||||
}
|
||||
|
||||
// Parse decrypts and validates an encrypted URL token and returns the
|
||||
// request it holds, with default values for omitted optional fields. A token
|
||||
// that never expires gives the zero Expires.
|
||||
// Parse decrypts and validates an encrypted URL token.
|
||||
// Returns ErrExpired if the token has expired, or other errors for invalid tokens.
|
||||
func (g *Generator) Parse(token string) (*imagerequest.Request, error) {
|
||||
func (g *Generator) Parse(token string) (*Payload, error) {
|
||||
// Decrypt
|
||||
data, err := seal.Decrypt(g.key, token)
|
||||
if err != nil {
|
||||
@@ -107,7 +87,7 @@ func (g *Generator) Parse(token string) (*imagerequest.Request, error) {
|
||||
}
|
||||
|
||||
// CBOR decode
|
||||
var p payload
|
||||
var p Payload
|
||||
|
||||
err = cbor.Unmarshal(data, &p)
|
||||
if err != nil {
|
||||
@@ -119,6 +99,13 @@ func (g *Generator) Parse(token string) (*imagerequest.Request, error) {
|
||||
return nil, ErrExpired
|
||||
}
|
||||
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// ToImageRequest converts the payload to an ImageRequest.
|
||||
// Applies default values for omitted optional fields. An ExpiresAt of 0, a URL
|
||||
// that never expires, gives the zero Expires.
|
||||
func (p *Payload) ToImageRequest() *imgcache.ImageRequest {
|
||||
format := p.Format
|
||||
if format == "" {
|
||||
format = DefaultFormat
|
||||
@@ -139,18 +126,44 @@ func (g *Generator) Parse(token string) (*imagerequest.Request, error) {
|
||||
expires = time.Unix(p.ExpiresAt, 0)
|
||||
}
|
||||
|
||||
return &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: p.SourceHost,
|
||||
Path: p.SourcePath,
|
||||
Query: p.SourceQuery,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: p.Width, Height: p.Height},
|
||||
Format: format,
|
||||
Quality: quality,
|
||||
FitMode: fitMode,
|
||||
return &imgcache.ImageRequest{
|
||||
SourceHost: p.SourceHost,
|
||||
SourcePath: p.SourcePath,
|
||||
SourceQuery: p.SourceQuery,
|
||||
Size: imgcache.Size{
|
||||
Width: p.Width,
|
||||
Height: p.Height,
|
||||
},
|
||||
Format: format,
|
||||
Quality: quality,
|
||||
FitMode: fitMode,
|
||||
Expires: expires,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// FromImageRequest creates a Payload from an ImageRequest with the given expiration.
|
||||
func FromImageRequest(req *imgcache.ImageRequest, expiresAt time.Time) *Payload {
|
||||
p := &Payload{
|
||||
SourceHost: req.SourceHost,
|
||||
SourcePath: req.SourcePath,
|
||||
SourceQuery: req.SourceQuery,
|
||||
Width: req.Size.Width,
|
||||
Height: req.Size.Height,
|
||||
ExpiresAt: expiresAt.Unix(),
|
||||
}
|
||||
|
||||
// Only set non-default values to benefit from omitempty
|
||||
if req.Format != "" && req.Format != DefaultFormat {
|
||||
p.Format = req.Format
|
||||
}
|
||||
|
||||
if req.Quality != 0 && req.Quality != DefaultQuality {
|
||||
p.Quality = req.Quality
|
||||
}
|
||||
|
||||
if req.FitMode != "" && req.FitMode != DefaultFitMode {
|
||||
p.FitMode = req.FitMode
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
+174
-91
@@ -6,7 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// Shared test fixture strings.
|
||||
@@ -24,22 +24,19 @@ func TestGenerator_GenerateAndParse(t *testing.T) {
|
||||
t.Fatalf("NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: testSourceHost,
|
||||
Path: testSourcePath,
|
||||
Query: testSourceQuery,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 800, Height: 600},
|
||||
Format: imagerequest.FormatWebP,
|
||||
Quality: 90,
|
||||
FitMode: imagerequest.FitContain,
|
||||
},
|
||||
Expires: time.Now().Add(time.Hour),
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
SourceQuery: testSourceQuery,
|
||||
Width: 800,
|
||||
Height: 600,
|
||||
Format: imgcache.FormatWebP,
|
||||
Quality: 90,
|
||||
FitMode: imgcache.FitContain,
|
||||
ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
||||
}
|
||||
|
||||
token, err := gen.Generate(req)
|
||||
token, err := gen.Generate(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
@@ -54,45 +51,40 @@ func TestGenerator_GenerateAndParse(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify all fields
|
||||
if parsed.Source.Host != req.Source.Host {
|
||||
t.Errorf("SourceHost = %q, want %q", parsed.Source.Host, req.Source.Host)
|
||||
if parsed.SourceHost != payload.SourceHost {
|
||||
t.Errorf("SourceHost = %q, want %q", parsed.SourceHost, payload.SourceHost)
|
||||
}
|
||||
|
||||
if parsed.Source.Path != req.Source.Path {
|
||||
t.Errorf("SourcePath = %q, want %q", parsed.Source.Path, req.Source.Path)
|
||||
if parsed.SourcePath != payload.SourcePath {
|
||||
t.Errorf("SourcePath = %q, want %q", parsed.SourcePath, payload.SourcePath)
|
||||
}
|
||||
|
||||
if parsed.Source.Query != req.Source.Query {
|
||||
t.Errorf("SourceQuery = %q, want %q", parsed.Source.Query, req.Source.Query)
|
||||
if parsed.SourceQuery != payload.SourceQuery {
|
||||
t.Errorf("SourceQuery = %q, want %q", parsed.SourceQuery, payload.SourceQuery)
|
||||
}
|
||||
|
||||
if parsed.Transform.Size.Width != req.Transform.Size.Width {
|
||||
t.Errorf("Width = %d, want %d",
|
||||
parsed.Transform.Size.Width, req.Transform.Size.Width)
|
||||
if parsed.Width != payload.Width {
|
||||
t.Errorf("Width = %d, want %d", parsed.Width, payload.Width)
|
||||
}
|
||||
|
||||
if parsed.Transform.Size.Height != req.Transform.Size.Height {
|
||||
t.Errorf("Height = %d, want %d",
|
||||
parsed.Transform.Size.Height, req.Transform.Size.Height)
|
||||
if parsed.Height != payload.Height {
|
||||
t.Errorf("Height = %d, want %d", parsed.Height, payload.Height)
|
||||
}
|
||||
|
||||
if parsed.Transform.Format != req.Transform.Format {
|
||||
t.Errorf("Format = %q, want %q",
|
||||
parsed.Transform.Format, req.Transform.Format)
|
||||
if parsed.Format != payload.Format {
|
||||
t.Errorf("Format = %q, want %q", parsed.Format, payload.Format)
|
||||
}
|
||||
|
||||
if parsed.Transform.Quality != req.Transform.Quality {
|
||||
t.Errorf("Quality = %d, want %d",
|
||||
parsed.Transform.Quality, req.Transform.Quality)
|
||||
if parsed.Quality != payload.Quality {
|
||||
t.Errorf("Quality = %d, want %d", parsed.Quality, payload.Quality)
|
||||
}
|
||||
|
||||
if parsed.Transform.FitMode != req.Transform.FitMode {
|
||||
t.Errorf("FitMode = %q, want %q",
|
||||
parsed.Transform.FitMode, req.Transform.FitMode)
|
||||
if parsed.FitMode != payload.FitMode {
|
||||
t.Errorf("FitMode = %q, want %q", parsed.FitMode, payload.FitMode)
|
||||
}
|
||||
|
||||
if parsed.Expires.Unix() != req.Expires.Unix() {
|
||||
t.Errorf("ExpiresAt = %d, want %d", parsed.Expires.Unix(), req.Expires.Unix())
|
||||
if parsed.ExpiresAt != payload.ExpiresAt {
|
||||
t.Errorf("ExpiresAt = %d, want %d", parsed.ExpiresAt, payload.ExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,15 +93,13 @@ func TestGenerator_Parse_Expired(t *testing.T) {
|
||||
|
||||
gen, _ := encurl.NewGenerator("test-signing-key-12345")
|
||||
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: testSourceHost,
|
||||
Path: testSourcePath,
|
||||
},
|
||||
Expires: time.Now().Add(-time.Hour), // Already expired
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
ExpiresAt: time.Now().Add(-time.Hour).Unix(), // Already expired
|
||||
}
|
||||
|
||||
token, err := gen.Generate(req)
|
||||
token, err := gen.Generate(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
@@ -140,15 +130,13 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) {
|
||||
|
||||
gen, _ := encurl.NewGenerator("test-signing-key-12345")
|
||||
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: testSourceHost,
|
||||
Path: testSourcePath,
|
||||
},
|
||||
Expires: time.Now().Add(time.Hour),
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
||||
}
|
||||
|
||||
token, _ := gen.Generate(req)
|
||||
token, _ := gen.Generate(payload)
|
||||
|
||||
// Tamper with the token
|
||||
tampered := []byte(token)
|
||||
@@ -168,15 +156,13 @@ func TestGenerator_Parse_WrongKey(t *testing.T) {
|
||||
gen1, _ := encurl.NewGenerator("signing-key-1")
|
||||
gen2, _ := encurl.NewGenerator("signing-key-2")
|
||||
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: testSourceHost,
|
||||
Path: testSourcePath,
|
||||
},
|
||||
Expires: time.Now().Add(time.Hour),
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
||||
}
|
||||
|
||||
token, _ := gen1.Generate(req)
|
||||
token, _ := gen1.Generate(payload)
|
||||
|
||||
_, err := gen2.Parse(token)
|
||||
if err == nil {
|
||||
@@ -184,41 +170,140 @@ func TestGenerator_Parse_WrongKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerator_Parse_Defaults(t *testing.T) {
|
||||
func TestPayload_ToImageRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gen, _ := encurl.NewGenerator("test-signing-key-12345")
|
||||
|
||||
// Request with only required fields - should get defaults
|
||||
token, err := gen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: testSourceHost,
|
||||
Path: testSourcePath,
|
||||
},
|
||||
Expires: time.Now().Add(time.Hour),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
SourceQuery: testSourceQuery,
|
||||
Width: 800,
|
||||
Height: 600,
|
||||
Format: imgcache.FormatWebP,
|
||||
Quality: 90,
|
||||
FitMode: imgcache.FitContain,
|
||||
ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
||||
}
|
||||
|
||||
req, err := gen.Parse(token)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v", err)
|
||||
req := payload.ToImageRequest()
|
||||
|
||||
if req.SourceHost != payload.SourceHost {
|
||||
t.Errorf("SourceHost = %q, want %q", req.SourceHost, payload.SourceHost)
|
||||
}
|
||||
|
||||
if req.Transform.Format != encurl.DefaultFormat {
|
||||
t.Errorf("Format = %q, want default %q",
|
||||
req.Transform.Format, encurl.DefaultFormat)
|
||||
if req.SourcePath != payload.SourcePath {
|
||||
t.Errorf("SourcePath = %q, want %q", req.SourcePath, payload.SourcePath)
|
||||
}
|
||||
|
||||
if req.Transform.Quality != encurl.DefaultQuality {
|
||||
t.Errorf("Quality = %d, want default %d",
|
||||
req.Transform.Quality, encurl.DefaultQuality)
|
||||
if req.SourceQuery != payload.SourceQuery {
|
||||
t.Errorf("SourceQuery = %q, want %q", req.SourceQuery, payload.SourceQuery)
|
||||
}
|
||||
|
||||
if req.Transform.FitMode != encurl.DefaultFitMode {
|
||||
t.Errorf("FitMode = %q, want default %q",
|
||||
req.Transform.FitMode, encurl.DefaultFitMode)
|
||||
if req.Size.Width != payload.Width {
|
||||
t.Errorf("Width = %d, want %d", req.Size.Width, payload.Width)
|
||||
}
|
||||
|
||||
if req.Size.Height != payload.Height {
|
||||
t.Errorf("Height = %d, want %d", req.Size.Height, payload.Height)
|
||||
}
|
||||
|
||||
if req.Format != payload.Format {
|
||||
t.Errorf("Format = %q, want %q", req.Format, payload.Format)
|
||||
}
|
||||
|
||||
if req.Quality != payload.Quality {
|
||||
t.Errorf("Quality = %d, want %d", req.Quality, payload.Quality)
|
||||
}
|
||||
|
||||
if req.FitMode != payload.FitMode {
|
||||
t.Errorf("FitMode = %q, want %q", req.FitMode, payload.FitMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPayload_ToImageRequest_Defaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Payload with only required fields - should get defaults
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
||||
}
|
||||
|
||||
req := payload.ToImageRequest()
|
||||
|
||||
if req.Format != encurl.DefaultFormat {
|
||||
t.Errorf("Format = %q, want default %q", req.Format, encurl.DefaultFormat)
|
||||
}
|
||||
|
||||
if req.Quality != encurl.DefaultQuality {
|
||||
t.Errorf("Quality = %d, want default %d", req.Quality, encurl.DefaultQuality)
|
||||
}
|
||||
|
||||
if req.FitMode != encurl.DefaultFitMode {
|
||||
t.Errorf("FitMode = %q, want default %q", req.FitMode, encurl.DefaultFitMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromImageRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req := &imgcache.ImageRequest{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
SourceQuery: testSourceQuery,
|
||||
Size: imgcache.Size{Width: 800, Height: 600},
|
||||
Format: imgcache.FormatWebP,
|
||||
Quality: 90,
|
||||
FitMode: imgcache.FitContain,
|
||||
}
|
||||
|
||||
expiresAt := time.Now().Add(time.Hour)
|
||||
payload := encurl.FromImageRequest(req, expiresAt)
|
||||
|
||||
if payload.SourceHost != req.SourceHost {
|
||||
t.Errorf("SourceHost = %q, want %q", payload.SourceHost, req.SourceHost)
|
||||
}
|
||||
|
||||
if payload.SourcePath != req.SourcePath {
|
||||
t.Errorf("SourcePath = %q, want %q", payload.SourcePath, req.SourcePath)
|
||||
}
|
||||
|
||||
if payload.Width != req.Size.Width {
|
||||
t.Errorf("Width = %d, want %d", payload.Width, req.Size.Width)
|
||||
}
|
||||
|
||||
if payload.ExpiresAt != expiresAt.Unix() {
|
||||
t.Errorf("ExpiresAt = %d, want %d", payload.ExpiresAt, expiresAt.Unix())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromImageRequest_OmitsDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Request with default values - payload should omit them for
|
||||
// smaller encoding
|
||||
req := &imgcache.ImageRequest{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
Format: encurl.DefaultFormat,
|
||||
Quality: encurl.DefaultQuality,
|
||||
FitMode: encurl.DefaultFitMode,
|
||||
}
|
||||
|
||||
payload := encurl.FromImageRequest(req, time.Now().Add(time.Hour))
|
||||
|
||||
// These should be zero/empty because they match defaults
|
||||
if payload.Format != "" {
|
||||
t.Errorf("Format should be empty for default, got %q", payload.Format)
|
||||
}
|
||||
|
||||
if payload.Quality != 0 {
|
||||
t.Errorf("Quality should be 0 for default, got %d", payload.Quality)
|
||||
}
|
||||
|
||||
if payload.FitMode != "" {
|
||||
t.Errorf("FitMode should be empty for default, got %q", payload.FitMode)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,15 +312,13 @@ func TestGenerator_TokenIsURLSafe(t *testing.T) {
|
||||
|
||||
gen, _ := encurl.NewGenerator("test-signing-key-12345")
|
||||
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: testSourceHost,
|
||||
Path: testSourcePath,
|
||||
},
|
||||
Expires: time.Now().Add(time.Hour),
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: testSourceHost,
|
||||
SourcePath: testSourcePath,
|
||||
ExpiresAt: time.Now().Add(time.Hour).Unix(),
|
||||
}
|
||||
|
||||
token, _ := gen.Generate(req)
|
||||
token, _ := gen.Generate(payload)
|
||||
|
||||
// Check that token only contains URL-safe characters
|
||||
for _, c := range token {
|
||||
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
|
||||
const appname = "pixad"
|
||||
|
||||
// Version is set by app.Run to the version main was built with.
|
||||
var Version string //nolint:gochecknoglobals // set by app.Run
|
||||
// Version is populated from main() via ldflags.
|
||||
var Version string //nolint:gochecknoglobals // set from main
|
||||
|
||||
// Globals holds application-wide constants.
|
||||
type Globals struct {
|
||||
|
||||
+38
-116
@@ -9,14 +9,12 @@ import (
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/clientip"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/templates"
|
||||
)
|
||||
|
||||
@@ -91,18 +89,10 @@ func (s *Handlers) handleLoginPost(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// HandleLogout logs out, which ends every login session (see
|
||||
// session.Manager.Logout), and redirects to login.
|
||||
// HandleLogout clears the session and redirects to login.
|
||||
func (s *Handlers) HandleLogout() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
err := s.sessMgr.Logout(w, r)
|
||||
if err != nil {
|
||||
s.log.Error("failed to log out", "error", err)
|
||||
http.Error(w, "Failed to log out", http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.sessMgr.ClearSession(w)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
@@ -127,7 +117,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
||||
sourceURL := r.FormValue("url")
|
||||
|
||||
// Validate source URL
|
||||
parsed, err := parseSourceURL(sourceURL)
|
||||
parsed, err := url.Parse(sourceURL)
|
||||
if err != nil || parsed.Host == "" {
|
||||
s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form,
|
||||
http.StatusBadRequest)
|
||||
@@ -135,17 +125,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// /v1/image/ refuses a ".." segment in the source path, and so does
|
||||
// the generator. The path is decoded, so %2e%2e counts too
|
||||
if slices.Contains(strings.Split(parsed.Path, "/"), "..") {
|
||||
s.renderGeneratorWithForm(w, r,
|
||||
"Source URL path cannot hold a .. segment", r.Form,
|
||||
http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
req, expiresAt, ttl, err := buildGenerateRequest(parsed, r.Form)
|
||||
payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form)
|
||||
if err != nil {
|
||||
s.renderGeneratorWithForm(w, r, err.Error(), r.Form,
|
||||
http.StatusBadRequest)
|
||||
@@ -153,20 +133,8 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// The image URL must fetch the source URL as it was written
|
||||
req.AllowHTTP = s.config.AllowHTTP
|
||||
fetchURL := req.SourceURL()
|
||||
|
||||
if !fetchedAsWritten(parsed, fetchURL) {
|
||||
s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+
|
||||
"written: pixa would fetch "+fetchURL,
|
||||
r.Form, http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Generate encrypted token
|
||||
token, err := s.encGen.Generate(req)
|
||||
token, err := s.encGen.Generate(payload)
|
||||
if err != nil {
|
||||
s.log.Error("failed to generate encrypted URL", "error", err)
|
||||
s.renderGeneratorWithForm(w, r, "Failed to generate URL", r.Form,
|
||||
@@ -197,57 +165,14 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// parseSourceURL parses the source URL entered on the generator page with its
|
||||
// host lower-cased and an empty path written as "/", as /v1/image/ reads a
|
||||
// source URL, so that each spelling of one source URL makes one cache key and
|
||||
// one upstream request.
|
||||
func parseSourceURL(sourceURL string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(sourceURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parsed.Host = strings.ToLower(parsed.Host)
|
||||
|
||||
if parsed.Path == "" {
|
||||
parsed.Path = "/"
|
||||
}
|
||||
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// fetchedAsWritten reports whether fetchURL, the URL pixa fetches for an
|
||||
// image URL made from the source URL source, is source as it was written,
|
||||
// apart from the scheme, which the token does not keep. It is not when source
|
||||
// holds a user name and password, which the token does not keep either, or a
|
||||
// "?" with no query after it. The token keeps the source path decoded and
|
||||
// pixa fetches it percent-encoded again, so a path written another way, such
|
||||
// as with %2F, is fetched as another path. url.URL keeps the path as written
|
||||
// in RawPath when it is not the decoded path percent-encoded again.
|
||||
// EscapedPath ignores RawPath when it also holds a character net/url encodes,
|
||||
// such as a space, so RawPath is compared itself.
|
||||
func fetchedAsWritten(source *url.URL, fetchURL string) bool {
|
||||
writtenPath := source.RawPath
|
||||
if writtenPath == "" {
|
||||
writtenPath = source.EscapedPath()
|
||||
}
|
||||
|
||||
fetched, err := url.Parse(fetchURL)
|
||||
|
||||
return err == nil && source.User == nil && !source.ForceQuery &&
|
||||
fetched.EscapedPath() == writtenPath &&
|
||||
fetched.RawQuery == source.RawQuery
|
||||
}
|
||||
|
||||
// buildGenerateRequest parses the numeric form fields and assembles the
|
||||
// image request the encrypted URL's token is made from. ttl=0 means never
|
||||
// expires (Expires stays the zero time). A non-numeric or out-of-range
|
||||
// width, height, quality or ttl, or an unrecognized fit mode, is a client
|
||||
// error naming the offending field. The format field is passed through
|
||||
// unchecked.
|
||||
func buildGenerateRequest(
|
||||
// buildGeneratePayload parses the numeric form fields and assembles the
|
||||
// encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A
|
||||
// non-numeric or out-of-range width, height, quality or ttl, or an
|
||||
// unrecognized fit mode, is a client error naming the offending field. The
|
||||
// format field is passed through unchecked.
|
||||
func buildGeneratePayload(
|
||||
parsed *url.URL, form url.Values,
|
||||
) (*imagerequest.Request, time.Time, int, error) {
|
||||
) (*encurl.Payload, time.Time, int, error) {
|
||||
width, err := parseFormDimension(form, "width")
|
||||
if err != nil {
|
||||
return nil, time.Time{}, 0, err
|
||||
@@ -269,40 +194,42 @@ func buildGenerateRequest(
|
||||
return nil, time.Time{}, 0, err
|
||||
}
|
||||
|
||||
fitMode := imagerequest.Fit(form.Get("fit"))
|
||||
fitMode := imgcache.FitMode(form.Get("fit"))
|
||||
|
||||
err = imagerequest.ValidateFitMode(fitMode)
|
||||
err = imgcache.ValidateFitMode(fitMode)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, 0,
|
||||
fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, form.Get("fit"))
|
||||
fmt.Errorf("%w: %s", imgcache.ErrInvalidFitMode, form.Get("fit"))
|
||||
}
|
||||
|
||||
var expiresAt time.Time
|
||||
var (
|
||||
expiresAt time.Time
|
||||
expiresAtUnix int64
|
||||
)
|
||||
|
||||
if ttl > 0 {
|
||||
expiresAt = time.Now().Add(time.Duration(ttl) * time.Second)
|
||||
expiresAtUnix = expiresAt.Unix()
|
||||
}
|
||||
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: parsed.Host,
|
||||
Path: parsed.Path,
|
||||
Query: parsed.RawQuery,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: width, Height: height},
|
||||
Format: imagerequest.Format(form.Get("format")),
|
||||
Quality: quality,
|
||||
FitMode: fitMode,
|
||||
},
|
||||
Expires: expiresAt,
|
||||
payload := &encurl.Payload{
|
||||
SourceHost: parsed.Host,
|
||||
SourcePath: parsed.Path,
|
||||
SourceQuery: parsed.RawQuery,
|
||||
Width: width,
|
||||
Height: height,
|
||||
Format: imgcache.ImageFormat(form.Get("format")),
|
||||
Quality: quality,
|
||||
FitMode: fitMode,
|
||||
ExpiresAt: expiresAtUnix,
|
||||
}
|
||||
|
||||
return req, expiresAt, ttl, nil
|
||||
return payload, expiresAt, ttl, nil
|
||||
}
|
||||
|
||||
// parseFormDimension reads an optional width or height form field. An empty
|
||||
// value means "original size" (0). A non-numeric value, or one
|
||||
// imagerequest.ValidateDimension rejects, is an error naming the field.
|
||||
// imgcache.ValidateDimension rejects, is an error naming the field.
|
||||
func parseFormDimension(form url.Values, field string) (int, error) {
|
||||
raw := form.Get(field)
|
||||
if raw == "" {
|
||||
@@ -314,7 +241,7 @@ func parseFormDimension(form url.Values, field string) (int, error) {
|
||||
return 0, fmt.Errorf("%w %s: not a number", errInvalidFormField, field)
|
||||
}
|
||||
|
||||
err = imagerequest.ValidateDimension(field, value)
|
||||
err = imgcache.ValidateDimension(field, value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -442,15 +369,10 @@ func (s *Handlers) buildGeneratedURL(r *http.Request, token, format string) stri
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
// Determine file extension for the trailing filename. A form with no
|
||||
// format makes a token with none, which is served as encurl.DefaultFormat.
|
||||
// Determine file extension for the trailing filename
|
||||
ext := format
|
||||
|
||||
switch format {
|
||||
case "":
|
||||
ext = string(encurl.DefaultFormat)
|
||||
case "orig", "auto":
|
||||
ext = "jpg"
|
||||
if ext == "" || ext == "orig" {
|
||||
ext = "jpg" // Default extension
|
||||
}
|
||||
|
||||
return scheme + "://" + r.Host + "/v1/e/" + url.PathEscape(token) + "/img." + ext
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// Generator form field names, and a value that is not a number.
|
||||
@@ -125,10 +125,10 @@ func TestGeneratePostRejectsBadField(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildGenerateRequestDefaultAndLimits verifies that an empty quality
|
||||
// TestBuildGeneratePayloadDefaultAndLimits verifies that an empty quality
|
||||
// takes the default, and that the largest accepted width, height, quality and
|
||||
// ttl are accepted with an expiry still in the future.
|
||||
func TestBuildGenerateRequestDefaultAndLimits(t *testing.T) {
|
||||
func TestBuildGeneratePayloadDefaultAndLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
parsed, err := url.Parse(testSourceURL)
|
||||
@@ -136,19 +136,19 @@ func TestBuildGenerateRequestDefaultAndLimits(t *testing.T) {
|
||||
t.Fatalf("url.Parse() error = %v", err)
|
||||
}
|
||||
|
||||
req, _, _, err := buildGenerateRequest(parsed, url.Values{})
|
||||
payload, _, _, err := buildGeneratePayload(parsed, url.Values{})
|
||||
if err != nil {
|
||||
t.Fatalf("empty form: error = %v", err)
|
||||
}
|
||||
|
||||
if req.Transform.Quality != encurl.DefaultQuality {
|
||||
if payload.Quality != encurl.DefaultQuality {
|
||||
t.Errorf("empty quality gave %d, want %d",
|
||||
req.Transform.Quality, encurl.DefaultQuality)
|
||||
payload.Quality, encurl.DefaultQuality)
|
||||
}
|
||||
|
||||
_, expiresAt, _, err := buildGenerateRequest(parsed, url.Values{
|
||||
widthField: {strconv.Itoa(imagerequest.MaxDimension)},
|
||||
heightField: {strconv.Itoa(imagerequest.MaxDimension)},
|
||||
_, expiresAt, _, err := buildGeneratePayload(parsed, url.Values{
|
||||
widthField: {strconv.Itoa(imgcache.MaxDimension)},
|
||||
heightField: {strconv.Itoa(imgcache.MaxDimension)},
|
||||
qualityField: {strconv.Itoa(maxQuality)},
|
||||
ttlField: {strconv.Itoa(maxTTL)},
|
||||
})
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/session"
|
||||
)
|
||||
|
||||
// TestLoginLogLeavesOutSubmittedKey verifies that the log lines for a
|
||||
// failed and for a successful login do not contain the submitted key.
|
||||
func TestLoginLogLeavesOutSubmittedKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const wrongKey = "wrong-signing-key-fedcba9876543210"
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
sessMgr, err := session.NewManager(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("session.NewManager() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{
|
||||
log: slog.New(slog.NewJSONHandler(&buf, nil)),
|
||||
config: &config.Config{SigningKey: testSigningKey},
|
||||
sessMgr: sessMgr,
|
||||
}
|
||||
|
||||
submittedKeys := []string{wrongKey, testSigningKey}
|
||||
|
||||
for _, key := range submittedKeys {
|
||||
form := url.Values{loginKeyField: {key}}
|
||||
req := httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodPost, "/",
|
||||
strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
h.handleLoginPost(httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
for _, msg := range []string{"failed login attempt", "successful login"} {
|
||||
if !strings.Contains(buf.String(), msg) {
|
||||
t.Fatalf("log missing %q; got %q", msg, buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range submittedKeys {
|
||||
if strings.Contains(buf.String(), key) {
|
||||
t.Errorf("log contains submitted key %q; got %q", key, buf.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,384 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/session"
|
||||
)
|
||||
|
||||
// formatField is the generator form's format field name.
|
||||
const formatField = "format"
|
||||
|
||||
// Markers telling the login page from the generator page.
|
||||
const (
|
||||
loginForm = `action="/"`
|
||||
loginKeyInput = `name="key"`
|
||||
generatorForm = `action="/generate"`
|
||||
)
|
||||
|
||||
// generatedURLPattern extracts the path of the URL the generator page shows.
|
||||
// The test router runs with debug on, so the URL starts with http, and its
|
||||
// host is httptest's default request host.
|
||||
var generatedURLPattern = regexp.MustCompile(
|
||||
`value="http://example\.com(/v1/e/[^"]+)"`)
|
||||
|
||||
// findSessionCookie returns the session cookie rec sets, or nil if it sets
|
||||
// none.
|
||||
func findSessionCookie(rec *httptest.ResponseRecorder) *http.Cookie {
|
||||
for _, c := range rec.Result().Cookies() {
|
||||
if c.Name == session.CookieName {
|
||||
return c
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestHandleRoot_NoSession_ShowsLoginForm verifies that GET / without a
|
||||
// login session shows the login form.
|
||||
func TestHandleRoot_NoSession_ShowsLoginForm(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, "/", nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
|
||||
t.Errorf("page is not the login form: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginPost_WrongKey_ShowsErrorWithoutSession verifies that a wrong key
|
||||
// shows the login form again with an error, and sets no session cookie.
|
||||
func TestLoginPost_WrongKey_ShowsErrorWithoutSession(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
cookies, token := csrfCredentials(t, srv, nil)
|
||||
|
||||
rec := postForm(srv, "/", cookies, url.Values{
|
||||
loginKeyField: {"wrong-signing-key-fedcba9876543210"},
|
||||
csrfTokenField: {token},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
|
||||
t.Errorf("page is not the login form: %s", body)
|
||||
}
|
||||
|
||||
if !strings.Contains(body, "Invalid signing key") {
|
||||
t.Error("login form does not show the error")
|
||||
}
|
||||
|
||||
if c := findSessionCookie(rec); c != nil {
|
||||
t.Errorf("wrong key set a session cookie: %s", c)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginPost_RightKey_SetsSessionCookie verifies that the right key answers
|
||||
// 303 to / with a session cookie marked Secure, HttpOnly and SameSite=Strict,
|
||||
// and that GET / with that cookie shows the generator page.
|
||||
func TestLoginPost_RightKey_SetsSessionCookie(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
cookies, token := csrfCredentials(t, srv, nil)
|
||||
|
||||
rec := postForm(srv, "/", cookies, url.Values{
|
||||
loginKeyField: {testSigningKey},
|
||||
csrfTokenField: {token},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
sessionCookie := findSessionCookie(rec)
|
||||
if sessionCookie == nil {
|
||||
t.Fatal("right key set no session cookie")
|
||||
}
|
||||
|
||||
t.Logf("Set-Cookie: %s", sessionCookie)
|
||||
|
||||
if !sessionCookie.Secure {
|
||||
t.Error("session cookie is not Secure")
|
||||
}
|
||||
|
||||
if !sessionCookie.HttpOnly {
|
||||
t.Error("session cookie is not HttpOnly")
|
||||
}
|
||||
|
||||
if sessionCookie.SameSite != http.SameSiteStrictMode {
|
||||
t.Errorf("session cookie SameSite = %v, want Strict", sessionCookie.SameSite)
|
||||
}
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(sessionCookie)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK ||
|
||||
!strings.Contains(rec.Body.String(), generatorForm) {
|
||||
t.Errorf("GET / with the session cookie: status = %d, "+
|
||||
"want %d and the generator page", rec.Code, http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleLogout_ClearsSessionCookie verifies that GET /logout answers 303
|
||||
// to / and replaces the session cookie with an empty one sent with
|
||||
// Max-Age=0, which makes the browser delete it.
|
||||
func TestHandleLogout_ClearsSessionCookie(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, _ := newCSRFTestRouter(t)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, "/logout", nil)
|
||||
req.AddCookie(newSessionCookie(t, h))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleLogout().ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
t.Logf("Set-Cookie: %s", rec.Header().Get("Set-Cookie"))
|
||||
|
||||
sessionCookie := findSessionCookie(rec)
|
||||
if sessionCookie == nil {
|
||||
t.Fatal("logout did not set the session cookie")
|
||||
}
|
||||
|
||||
if sessionCookie.Value != "" {
|
||||
t.Errorf("session cookie value = %q, want empty", sessionCookie.Value)
|
||||
}
|
||||
|
||||
// net/http reads a Max-Age=0 attribute back as MaxAge -1.
|
||||
if sessionCookie.MaxAge != -1 {
|
||||
t.Errorf("session cookie MaxAge = %d, want -1 (Max-Age=0)",
|
||||
sessionCookie.MaxAge)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleLogout_EndsCopiedSession verifies that after POST /logout a copy
|
||||
// of the session cookie taken before it shows the login page, not the
|
||||
// generator page.
|
||||
func TestHandleLogout_EndsCopiedSession(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newCSRFTestRouter(t)
|
||||
|
||||
copied := newSessionCookie(t, h)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodPost, "/logout", nil)
|
||||
req.AddCookie(copied)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleLogout().ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("POST /logout status = %d, want %d",
|
||||
rec.Code, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(copied)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
body := rec.Body.String()
|
||||
t.Logf("GET / with the copied cookie: status %d, login page %v",
|
||||
rec.Code, strings.Contains(body, loginForm))
|
||||
|
||||
if !strings.Contains(body, loginForm) || strings.Contains(body, generatorForm) {
|
||||
t.Error("a cookie copied before /logout still opens the generator page")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleLogout_SaveFails_Answers500 verifies that when the logout cannot
|
||||
// be saved in the database, POST /logout answers 500, leaves the session
|
||||
// cookie in place and ends no session.
|
||||
func TestHandleLogout_SaveFails_Answers500(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newCSRFTestRouter(t)
|
||||
|
||||
db := setupTestDB(t)
|
||||
|
||||
sessMgr, err := session.NewManagerWithDatabase(
|
||||
t.Context(), testSigningKey, db)
|
||||
if err != nil {
|
||||
t.Fatalf("session.NewManagerWithDatabase() error = %v", err)
|
||||
}
|
||||
|
||||
h.sessMgr = sessMgr
|
||||
cookie := newSessionCookie(t, h)
|
||||
|
||||
// Saving the logout fails on a closed database.
|
||||
err = db.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("closing the database: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodPost, "/logout", nil)
|
||||
req.AddCookie(cookie)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleLogout().ServeHTTP(rec, req)
|
||||
|
||||
t.Logf("POST /logout: status %d, Set-Cookie %q",
|
||||
rec.Code, rec.Header().Get("Set-Cookie"))
|
||||
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Errorf("POST /logout status = %d, want %d",
|
||||
rec.Code, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if c := findSessionCookie(rec); c != nil {
|
||||
t.Errorf("a logout that failed set the session cookie: %s", c)
|
||||
}
|
||||
|
||||
req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(cookie)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if !strings.Contains(rec.Body.String(), generatorForm) {
|
||||
t.Error("a logout that failed ended the session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate
|
||||
// with a valid CSRF token but no login session answers 303 to / and makes no
|
||||
// URL.
|
||||
func TestGeneratePost_NoSession_RedirectsToLogin(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
cookies, token := csrfCredentials(t, srv, nil)
|
||||
|
||||
rec := postForm(srv, "/generate", cookies, url.Values{
|
||||
sourceURLField: {testSourceURL},
|
||||
csrfTokenField: {token},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
if strings.Contains(rec.Body.String(), "/v1/e/") {
|
||||
t.Error("a URL was made without a login session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_URLServesImage verifies that the URL the generator page
|
||||
// makes is served by /v1/e/. The image route runs on handlers of its own,
|
||||
// made with the same signing key.
|
||||
func TestGeneratePost_URLServesImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
formatField: {string(imagerequest.FormatJPEG)},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
t.Logf("generated URL path: %s", match[1])
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireServedPhoto(t, imageRec)
|
||||
}
|
||||
|
||||
// TestGeneratePost_URLWithTTLExpires verifies that a URL the generator page
|
||||
// makes with a ttl of one second is served by /v1/e/ at once and answers 410
|
||||
// once the ttl has passed.
|
||||
func TestGeneratePost_URLWithTTLExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
formatField: {string(imagerequest.FormatJPEG)},
|
||||
ttlField: {"1"},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireServedPhoto(t, imageRec)
|
||||
|
||||
// The URL keeps the time it expires in whole seconds and is served
|
||||
// through the whole of that second, so a ttl of one second has passed
|
||||
// for certain two seconds after the URL was made.
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
imageRec = httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
t.Logf("GET %s after the ttl: %d %q", match[1], imageRec.Code, imageRec.Body)
|
||||
|
||||
if imageRec.Code != http.StatusGone {
|
||||
t.Errorf("status after the ttl = %d, want %d",
|
||||
imageRec.Code, http.StatusGone)
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestAVIFOutputTooLargeAnswers400 checks that respondImageError and
|
||||
// handleImageError, which answer the errors of the two image routes, answer
|
||||
// 400 naming the AVIF pixel limit when the image processor refuses an AVIF
|
||||
// output above it, wrapped as the service wraps it. The request's source only
|
||||
// goes into a log line, so it is left empty.
|
||||
func TestAVIFOutputTooLargeAnswers400(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||
err := fmt.Errorf("image processing failed: %w: 4097x4096",
|
||||
imageprocessor.ErrAVIFOutputTooLarge)
|
||||
wantBody := fmt.Sprintf("above %d pixels", imageprocessor.MaxAVIFOutputPixels)
|
||||
|
||||
plain := httptest.NewRecorder()
|
||||
h.respondImageError(plain, &imagerequest.Request{}, err)
|
||||
|
||||
encrypted := httptest.NewRecorder()
|
||||
h.handleImageError(encrypted, err)
|
||||
|
||||
for name, rec := range map[string]*httptest.ResponseRecorder{
|
||||
"respondImageError": plain, "handleImageError": encrypted,
|
||||
} {
|
||||
t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String())
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("%s: status = %d, want %d",
|
||||
name, rec.Code, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
if !strings.Contains(rec.Body.String(), wantBody) {
|
||||
t.Errorf("%s: body = %q, want it to say %q",
|
||||
name, rec.Body.String(), wantBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,265 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// countingHost is the allowlisted host whose photoPath the server of
|
||||
// newCountingDatabaseServer serves.
|
||||
const countingHost = "counting.example.com"
|
||||
|
||||
// countingConnector opens connections to the SQLite database file name with
|
||||
// sqliteDriver, each one counting in statements the statements run on it.
|
||||
type countingConnector struct {
|
||||
sqliteDriver driver.Driver
|
||||
name string
|
||||
statements *atomic.Int64
|
||||
}
|
||||
|
||||
func (c countingConnector) Connect(context.Context) (driver.Conn, error) {
|
||||
conn, err := c.sqliteDriver.Open(c.name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return countingConn{Conn: conn, statements: c.statements}, nil
|
||||
}
|
||||
|
||||
func (c countingConnector) Driver() driver.Driver {
|
||||
return c.sqliteDriver
|
||||
}
|
||||
|
||||
// countingConn counts the statements run on its connection. It has only the
|
||||
// methods of driver.Conn, so database/sql prepares every query and every
|
||||
// other statement with Prepare, which counts it.
|
||||
type countingConn struct {
|
||||
driver.Conn
|
||||
|
||||
statements *atomic.Int64
|
||||
}
|
||||
|
||||
func (c countingConn) Prepare(query string) (driver.Stmt, error) {
|
||||
c.statements.Add(1)
|
||||
|
||||
return c.Conn.Prepare(query)
|
||||
}
|
||||
|
||||
// newCountingDatabase returns a database with pixa's schema, in a file of
|
||||
// its own and with one connection, and the count of the statements run on
|
||||
// it once the schema is in place.
|
||||
func newCountingDatabase(tb testing.TB) (*sql.DB, *atomic.Int64) {
|
||||
tb.Helper()
|
||||
|
||||
path := filepath.Join(tb.TempDir(), "state.sqlite3")
|
||||
|
||||
schemaDB, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
tb.Fatalf("opening the database: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = schemaDB.Close() }()
|
||||
|
||||
err = database.ApplyMigrations(tb.Context(), schemaDB, nil)
|
||||
if err != nil {
|
||||
tb.Fatalf("applying the migrations: %v", err)
|
||||
}
|
||||
|
||||
statements := new(atomic.Int64)
|
||||
|
||||
db := sql.OpenDB(countingConnector{
|
||||
sqliteDriver: schemaDB.Driver(),
|
||||
name: path,
|
||||
statements: statements,
|
||||
})
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
tb.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
return db, statements
|
||||
}
|
||||
|
||||
// newCountingDatabaseServer returns a router for both image routes whose
|
||||
// image cache uses newCountingDatabase, the count of the statements run on
|
||||
// that database, and a URL on each route for a different image of
|
||||
// photoPath on countingHost. Nothing writes the cache's pending counts and
|
||||
// access times in the background, so every statement counted comes from a
|
||||
// request.
|
||||
func newCountingDatabaseServer(
|
||||
tb testing.TB,
|
||||
) (http.Handler, *atomic.Int64, []string) {
|
||||
tb.Helper()
|
||||
|
||||
db, statements := newCountingDatabase(tb)
|
||||
|
||||
var photo bytes.Buffer
|
||||
|
||||
err := jpeg.Encode(&photo, image.NewRGBA(image.Rect(0, 0, 100, 100)), nil)
|
||||
if err != nil {
|
||||
tb.Fatalf("encoding the photo: %v", err)
|
||||
}
|
||||
|
||||
cache, err := imgcache.NewCache(db, imgcache.CacheConfig{
|
||||
StateDir: tb.TempDir(),
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
tb.Fatalf("imgcache.NewCache() error = %v", err)
|
||||
}
|
||||
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
Fetcher: newMockFetcher(fstest.MapFS{
|
||||
countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()},
|
||||
}),
|
||||
SigningKey: testSigningKey,
|
||||
Allowlist: []string{countingHost},
|
||||
Logger: slog.New(slog.DiscardHandler),
|
||||
})
|
||||
if err != nil {
|
||||
tb.Fatalf("imgcache.NewService() error = %v", err)
|
||||
}
|
||||
|
||||
encGen, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
tb.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
token, err := encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{Host: countingHost, Path: photoPath},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 40, Height: 40},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
tb.Fatalf("encGen.Generate() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
imgSvc: svc,
|
||||
encGen: encGen,
|
||||
}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
return r, statements, []string{
|
||||
"/v1/image/" + countingHost + photoPath + "/50x50.jpeg",
|
||||
"/v1/e/" + token + "/img.jpg",
|
||||
}
|
||||
}
|
||||
|
||||
// serveImage sends srv a GET for target and returns the response, failing
|
||||
// the test unless it is a 200 with an image.
|
||||
func serveImage(
|
||||
tb testing.TB, srv http.Handler, target string,
|
||||
) *httptest.ResponseRecorder {
|
||||
tb.Helper()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec,
|
||||
httptest.NewRequestWithContext(tb.Context(), http.MethodGet, target, nil))
|
||||
|
||||
if rec.Code != http.StatusOK || rec.Body.Len() == 0 {
|
||||
tb.Errorf("GET %s: %d with %d bytes, want 200 with an image",
|
||||
target, rec.Code, rec.Body.Len())
|
||||
}
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// TestImageRoutes_CacheHitsMakeNoDatabaseCall serves an image on each image
|
||||
// route once, which makes and caches it, then serves it again from the cache.
|
||||
// Serving those cache hits must run no statement on the database.
|
||||
func TestImageRoutes_CacheHitsMakeNoDatabaseCall(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv, statements, targets := newCountingDatabaseServer(t)
|
||||
|
||||
for _, target := range targets {
|
||||
before := statements.Load()
|
||||
|
||||
serveImage(t, srv, target)
|
||||
|
||||
afterMiss := statements.Load()
|
||||
t.Logf("GET %s, a miss: %d statements", target, afterMiss-before)
|
||||
|
||||
if afterMiss == before {
|
||||
t.Fatalf("GET %s, a miss: no statement counted, want some", target)
|
||||
}
|
||||
|
||||
for range 3 {
|
||||
rec := serveImage(t, srv, target)
|
||||
|
||||
cacheStatus := rec.Header().Get("X-Pixa-Cache")
|
||||
if cacheStatus != string(imgcache.CacheHit) {
|
||||
t.Errorf("GET %s: X-Pixa-Cache = %q, want %s",
|
||||
target, cacheStatus, imgcache.CacheHit)
|
||||
}
|
||||
}
|
||||
|
||||
if hitStatements := statements.Load() - afterMiss; hitStatements != 0 {
|
||||
t.Errorf("GET %s: 3 cache hits ran %d statements, want 0",
|
||||
target, hitStatements)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkHandleImage_CacheHits serves one cached image through the
|
||||
// /v1/image/ route from b.RunParallel's goroutines, and reports the cache
|
||||
// hits served per second. It fails if serving them ran any statement on the
|
||||
// database. script/bench runs it.
|
||||
func BenchmarkHandleImage_CacheHits(b *testing.B) {
|
||||
srv, statements, targets := newCountingDatabaseServer(b)
|
||||
target := targets[0]
|
||||
|
||||
serveImage(b, srv, target)
|
||||
|
||||
before := statements.Load()
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
rec := serveImage(b, srv, target)
|
||||
|
||||
cacheStatus := rec.Header().Get("X-Pixa-Cache")
|
||||
if cacheStatus != string(imgcache.CacheHit) {
|
||||
b.Errorf("GET %s: X-Pixa-Cache = %q, want %s",
|
||||
target, cacheStatus, imgcache.CacheHit)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "hits/s")
|
||||
|
||||
if hitStatements := statements.Load() - before; hitStatements != 0 {
|
||||
b.Fatalf("%d cache hits ran %d statements, want 0", b.N, hitStatements)
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/fx/fxtest"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
)
|
||||
|
||||
// TestNewCacheConfigFromCacheMaxBytes checks the cache configuration
|
||||
// built from cache_max_bytes: omitted, the cache works out the default
|
||||
// limit; 0 turns the disk cache off; a positive value is the limit,
|
||||
// unchanged.
|
||||
func TestNewCacheConfigFromCacheMaxBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const oneGiB = 1 << 30
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
cacheMaxBytes int64
|
||||
cacheMaxBytesExplicit bool
|
||||
wantMaxBytes int64
|
||||
wantUseDefaultMaxBytes bool
|
||||
wantDisableDiskCache bool
|
||||
}{
|
||||
{
|
||||
name: "cache_max_bytes omitted",
|
||||
cacheMaxBytes: 0,
|
||||
cacheMaxBytesExplicit: false,
|
||||
wantMaxBytes: 0,
|
||||
wantUseDefaultMaxBytes: true,
|
||||
wantDisableDiskCache: false,
|
||||
},
|
||||
{
|
||||
name: "cache_max_bytes: 0",
|
||||
cacheMaxBytes: 0,
|
||||
cacheMaxBytesExplicit: true,
|
||||
wantMaxBytes: 0,
|
||||
wantUseDefaultMaxBytes: false,
|
||||
wantDisableDiskCache: true,
|
||||
},
|
||||
{
|
||||
name: "cache_max_bytes: 1 GiB",
|
||||
cacheMaxBytes: oneGiB,
|
||||
cacheMaxBytesExplicit: true,
|
||||
wantMaxBytes: oneGiB,
|
||||
wantUseDefaultMaxBytes: false,
|
||||
wantDisableDiskCache: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := &config.Config{
|
||||
CacheMaxBytes: tc.cacheMaxBytes,
|
||||
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
|
||||
}
|
||||
|
||||
got := newCacheConfig(cfg, nil)
|
||||
t.Logf("MaxBytes = %d, UseDefaultMaxBytes = %v, DisableDiskCache = %v",
|
||||
got.MaxBytes, got.UseDefaultMaxBytes, got.DisableDiskCache)
|
||||
|
||||
if got.MaxBytes != tc.wantMaxBytes {
|
||||
t.Errorf("MaxBytes = %d, want %d", got.MaxBytes, tc.wantMaxBytes)
|
||||
}
|
||||
|
||||
if got.UseDefaultMaxBytes != tc.wantUseDefaultMaxBytes {
|
||||
t.Errorf("UseDefaultMaxBytes = %v, want %v",
|
||||
got.UseDefaultMaxBytes, tc.wantUseDefaultMaxBytes)
|
||||
}
|
||||
|
||||
if got.DisableDiskCache != tc.wantDisableDiskCache {
|
||||
t.Errorf("DisableDiskCache = %v, want %v",
|
||||
got.DisableDiskCache, tc.wantDisableDiskCache)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes starts the handlers
|
||||
// once with cache_max_bytes omitted and once with cache_max_bytes: 0,
|
||||
// and checks by whether the cache directories were created that the
|
||||
// disk cache is on in the first case and off in the second.
|
||||
func TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
cacheMaxBytesExplicit bool
|
||||
wantDiskCache bool
|
||||
}{
|
||||
{name: "cache_max_bytes omitted", cacheMaxBytesExplicit: false, wantDiskCache: true},
|
||||
{name: "cache_max_bytes: 0", cacheMaxBytesExplicit: true, wantDiskCache: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stateDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
SigningKey: testSigningKey,
|
||||
StateDir: stateDir,
|
||||
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||
CacheMaxBytes: 0,
|
||||
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
|
||||
}
|
||||
|
||||
lc := fxtest.NewLifecycle(t)
|
||||
|
||||
log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}})
|
||||
if err != nil {
|
||||
t.Fatalf("logger.New() error = %v", err)
|
||||
}
|
||||
|
||||
db, err := database.New(lc, database.Params{Logger: log, Config: cfg})
|
||||
if err != nil {
|
||||
t.Fatalf("database.New() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = New(lc, Params{Logger: log, Database: db, Config: cfg})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
lc.RequireStart()
|
||||
t.Cleanup(lc.RequireStop)
|
||||
|
||||
_, err = os.Stat(filepath.Join(stateDir, "cache", "variants"))
|
||||
gotDiskCache := err == nil
|
||||
|
||||
if gotDiskCache != tc.wantDiskCache {
|
||||
t.Errorf("cache directories created = %v, want %v",
|
||||
gotDiskCache, tc.wantDiskCache)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxtest"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
)
|
||||
|
||||
// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as
|
||||
// pixad does, in an fx app that provides no fetcher, and requests an image
|
||||
// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL
|
||||
// check accepts that address; only the dialer that refuses internal
|
||||
// addresses checks blocked_networks, so the answer is 403 only if the
|
||||
// fetcher the handlers build from the config connects with that dialer. Any
|
||||
// other dialer would try to connect until the upstream fetch timeout, which
|
||||
// is short so that the test then fails quickly.
|
||||
func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const host = "192.0.2.10"
|
||||
|
||||
stateDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
SigningKey: testSigningKey,
|
||||
StateDir: stateDir,
|
||||
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||
AllowlistHosts: []string{host},
|
||||
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
|
||||
UpstreamFetchTimeout: 2 * time.Second,
|
||||
// With no connection slots, the fetch would fail before dialing.
|
||||
UpstreamConnections: config.DefaultUpstreamConnections,
|
||||
}
|
||||
|
||||
var h *Handlers
|
||||
|
||||
app := fxtest.New(t,
|
||||
fx.Supply(cfg),
|
||||
fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New),
|
||||
fx.Populate(&h),
|
||||
)
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
|
||||
rec := sendGet(t, r, photoURL(host))
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "forbidden")
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"image/color"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// mismatchedSourcePath is the path of the source that
|
||||
// TestFirstBytesNotMatchingContentType serves on allowlistedHost. The mock
|
||||
// fetcher names a file's Content-Type from its extension, so the source is
|
||||
// sent as image/png.
|
||||
const mismatchedSourcePath = "/images/source.png"
|
||||
|
||||
// newRoutesFetchingThrough returns both image routes, and the Handlers behind
|
||||
// them, whose image service fetches through fetcher and has allowlistedHost on
|
||||
// its allowlist. The handlers and the image service log to log.
|
||||
func newRoutesFetchingThrough(
|
||||
t *testing.T, fetcher httpfetcher.Fetcher, log *slog.Logger,
|
||||
) (*Handlers, http.Handler) {
|
||||
t.Helper()
|
||||
|
||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||
StateDir: t.TempDir(),
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewCache() error = %v", err)
|
||||
}
|
||||
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
Fetcher: fetcher,
|
||||
SigningKey: testSigningKey,
|
||||
Allowlist: []string{allowlistedHost},
|
||||
Logger: log,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||
}
|
||||
|
||||
encGen, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{log: log, imgSvc: svc, encGen: encGen}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
return h, r
|
||||
}
|
||||
|
||||
// mismatchedSourcePlainURL returns the /v1/image/ URL of mismatchedSourcePath
|
||||
// on allowlistedHost at 50x50 as a JPEG.
|
||||
func mismatchedSourcePlainURL(*testing.T, *Handlers) string {
|
||||
return "/v1/image/" + allowlistedHost + mismatchedSourcePath + "/50x50.jpeg"
|
||||
}
|
||||
|
||||
// mismatchedSourceEncURL returns an encrypted /v1/e/ URL, made with h's
|
||||
// generator, of mismatchedSourcePath on allowlistedHost at 50x50 as a JPEG.
|
||||
func mismatchedSourceEncURL(t *testing.T, h *Handlers) string {
|
||||
t.Helper()
|
||||
|
||||
token, err := h.encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: allowlistedHost,
|
||||
Path: mismatchedSourcePath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
return "/v1/e/" + token + "/img.jpg"
|
||||
}
|
||||
|
||||
// loggedLevels reads the JSON log lines in logs and returns the messages
|
||||
// logged at each level.
|
||||
func loggedLevels(t *testing.T, logs *bytes.Buffer) map[string][]string {
|
||||
t.Helper()
|
||||
|
||||
t.Logf("logged:\n%s", logs.String())
|
||||
|
||||
messages := make(map[string][]string)
|
||||
|
||||
dec := json.NewDecoder(logs)
|
||||
for dec.More() {
|
||||
var line struct {
|
||||
Level string `json:"level"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
|
||||
err := dec.Decode(&line)
|
||||
if err != nil {
|
||||
t.Fatalf("decoding log line: %v", err)
|
||||
}
|
||||
|
||||
messages[line.Level] = append(messages[line.Level], line.Msg)
|
||||
}
|
||||
|
||||
return messages
|
||||
}
|
||||
|
||||
// TestFirstBytesNotMatchingContentType checks that both image routes answer
|
||||
// 415 for an upstream body whose first bytes are not those of the format its
|
||||
// Content-Type names, and log it as a refused source, not as an error of
|
||||
// pixa's. The next request for that source must answer 502 without fetching
|
||||
// it again, as after an upstream error status.
|
||||
func TestFirstBytesNotMatchingContentType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bodies := map[string][]byte{
|
||||
"JPEG sent as image/png": generateTestJPEG(t, 10, 10, color.Black),
|
||||
"text sent as image/png": []byte("This is just text, not a PNG"),
|
||||
"body too short to name a format": []byte("short"),
|
||||
}
|
||||
|
||||
// The source's URL on each image route.
|
||||
sourceURLs := map[string]func(*testing.T, *Handlers) string{
|
||||
"plain URL": mismatchedSourcePlainURL,
|
||||
"encrypted URL": mismatchedSourceEncURL,
|
||||
}
|
||||
|
||||
for bodyName, body := range bodies {
|
||||
for urlName, sourceURL := range sourceURLs {
|
||||
t.Run(urlName+", "+bodyName, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fetcher := &countingFetcher{
|
||||
Fetcher: httpfetcher.NewMock(fstest.MapFS{
|
||||
allowlistedHost + mismatchedSourcePath: &fstest.MapFile{
|
||||
Data: body,
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
var logs bytes.Buffer
|
||||
|
||||
h, routes := newRoutesFetchingThrough(t, fetcher,
|
||||
slog.New(slog.NewJSONHandler(&logs, nil)))
|
||||
target := sourceURL(t, h)
|
||||
|
||||
checkErrorBody(t, sendGet(t, routes, target),
|
||||
http.StatusUnsupportedMediaType, "unsupported source format")
|
||||
|
||||
levels := loggedLevels(t, &logs)
|
||||
if len(levels[slog.LevelError.String()]) != 0 {
|
||||
t.Errorf("logged at error level: %q",
|
||||
levels[slog.LevelError.String()])
|
||||
}
|
||||
|
||||
if !slices.Contains(levels[slog.LevelWarn.String()], "source refused") {
|
||||
t.Error(`no "source refused" line logged at warning level`)
|
||||
}
|
||||
|
||||
checkErrorBody(t, sendGet(t, routes, target),
|
||||
http.StatusBadGateway, "upstream error")
|
||||
|
||||
if n := fetcher.fetches.Load(); n != 1 {
|
||||
t.Errorf("upstream fetched %d times, want 1", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// Errors for an Accept header that an auto URL cannot be served for.
|
||||
var (
|
||||
errInvalidAccept = errors.New("invalid Accept header")
|
||||
errNotAcceptable = errors.New("not acceptable: auto serves " +
|
||||
"image/jxl, image/avif, image/webp or image/jpeg")
|
||||
)
|
||||
|
||||
// chooseAutoFormat replaces the format auto in req with the format
|
||||
// formatForAccept chooses from r's Accept header, and adds Vary: Accept to the
|
||||
// response, which then depends on that header. It answers 400 for an Accept
|
||||
// header that is not valid and 406 for one that allows none of the formats,
|
||||
// and reports whether req can be served. Any other format is left as it is.
|
||||
func (s *Handlers) chooseAutoFormat(
|
||||
w http.ResponseWriter, r *http.Request, req *imagerequest.Request,
|
||||
) bool {
|
||||
if req.Transform.Format != imagerequest.FormatAuto {
|
||||
return true
|
||||
}
|
||||
|
||||
w.Header().Add("Vary", "Accept")
|
||||
|
||||
format, err := formatForAccept(strings.Join(r.Header.Values("Accept"), ","))
|
||||
if errors.Is(err, errNotAcceptable) {
|
||||
s.respondError(w, err.Error(), http.StatusNotAcceptable)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
s.respondError(w, err.Error(), http.StatusBadRequest)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
req.Transform.Format = format
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// formatForAccept returns the format an auto URL is served in for the Accept
|
||||
// header accept: JPEG XL when it names image/jxl, else AVIF when it names
|
||||
// image/avif, else WebP when it names image/webp, else JPEG when its most
|
||||
// specific entry of image/jpeg, image/* and */* allows it, or when it names
|
||||
// nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as
|
||||
// clients that cannot show them send image/* and */* too.
|
||||
func formatForAccept(accept string) (imagerequest.Format, error) {
|
||||
qualities, err := parseAccept(accept)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(qualities) == 0 {
|
||||
return imagerequest.FormatJPEG, nil
|
||||
}
|
||||
|
||||
if qualities["image/jxl"] > 0 {
|
||||
return imagerequest.FormatJXL, nil
|
||||
}
|
||||
|
||||
if qualities["image/avif"] > 0 {
|
||||
return imagerequest.FormatAVIF, nil
|
||||
}
|
||||
|
||||
if qualities["image/webp"] > 0 {
|
||||
return imagerequest.FormatWebP, nil
|
||||
}
|
||||
|
||||
// For JPEG, the most specific entry the header has decides
|
||||
quality, named := qualities["image/jpeg"]
|
||||
if !named {
|
||||
quality, named = qualities["image/*"]
|
||||
}
|
||||
|
||||
if !named {
|
||||
quality = qualities["*/*"]
|
||||
}
|
||||
|
||||
if quality > 0 {
|
||||
return imagerequest.FormatJPEG, nil
|
||||
}
|
||||
|
||||
return "", errNotAcceptable
|
||||
}
|
||||
|
||||
// parseAccept returns the q of each media range the Accept header accept
|
||||
// names, 1 where it gives none. A media range named more than once keeps its
|
||||
// lowest q, so a refusal is never overridden. A media range that does not
|
||||
// parse, or a q that is not a number from 0 to 1, is an error.
|
||||
func parseAccept(accept string) (map[string]float64, error) {
|
||||
qualities := make(map[string]float64)
|
||||
|
||||
for entry := range strings.SplitSeq(accept, ",") {
|
||||
// A header field list may hold empty entries
|
||||
if strings.TrimSpace(entry) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
mediaRange, params, err := mime.ParseMediaType(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %q: %w", errInvalidAccept, entry, err)
|
||||
}
|
||||
|
||||
quality := 1.0
|
||||
|
||||
if qParam, given := params["q"]; given {
|
||||
quality, err = strconv.ParseFloat(qParam, 64)
|
||||
inRange := quality >= 0 && quality <= 1
|
||||
|
||||
if err != nil || !inRange {
|
||||
return nil, fmt.Errorf("%w: %q: q is not a number from 0 to 1",
|
||||
errInvalidAccept, entry)
|
||||
}
|
||||
}
|
||||
|
||||
previous, named := qualities[mediaRange]
|
||||
if !named || quality < previous {
|
||||
qualities[mediaRange] = quality
|
||||
}
|
||||
}
|
||||
|
||||
return qualities, nil
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// The content types the tests below expect.
|
||||
const (
|
||||
avifType = "image/avif"
|
||||
webpType = "image/webp"
|
||||
jpegType = "image/jpeg"
|
||||
jsonType = "application/json"
|
||||
)
|
||||
|
||||
// TestFormatForAccept verifies the format chosen for the format auto from each
|
||||
// Accept header below, and the error for one that allows none of AVIF, WebP
|
||||
// and JPEG or is not valid.
|
||||
func TestFormatForAccept(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept string
|
||||
want imagerequest.Format
|
||||
wantErr error
|
||||
}{
|
||||
{"AVIF-capable browser",
|
||||
"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
||||
imagerequest.FormatAVIF, nil},
|
||||
{"WebP-capable browser",
|
||||
"image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5",
|
||||
imagerequest.FormatWebP, nil},
|
||||
{"WebP only", webpType, imagerequest.FormatWebP, nil},
|
||||
{"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imagerequest.FormatJPEG, nil},
|
||||
{"wildcard only", "*/*", imagerequest.FormatJPEG, nil},
|
||||
{"image wildcard only", "image/*", imagerequest.FormatJPEG, nil},
|
||||
{"absent", "", imagerequest.FormatJPEG, nil},
|
||||
{"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imagerequest.FormatWebP, nil},
|
||||
{"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*",
|
||||
imagerequest.FormatJPEG, nil},
|
||||
{"upper case and spaces", " Image/AVIF ; Q=0.5 ", imagerequest.FormatAVIF, nil},
|
||||
{"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable},
|
||||
{"q=0 on everything", "*/*;q=0", "", errNotAcceptable},
|
||||
{"PNG only", "image/png", "", errNotAcceptable},
|
||||
{"malformed media range", "image/", "", errInvalidAccept},
|
||||
{"q not a number", "image/avif;q=high", "", errInvalidAccept},
|
||||
{"q above 1", "image/avif;q=2", "", errInvalidAccept},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := formatForAccept(tt.accept)
|
||||
t.Logf("Accept %q: %q, %v", tt.accept, got, err)
|
||||
|
||||
if got != tt.want || !errors.Is(err, tt.wantErr) {
|
||||
t.Errorf("formatForAccept(%q) = %q, %v, want %q, %v",
|
||||
tt.accept, got, err, tt.want, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL,
|
||||
// both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in
|
||||
// the format auto, made with h's image service and generator.
|
||||
func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatAuto,
|
||||
},
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
token, err := h.encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatAuto,
|
||||
},
|
||||
Expires: time.Now().Add(time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
return signedURL, "/v1/e/" + token + "/img.jpg"
|
||||
}
|
||||
|
||||
// requestImage sends method for target to srv with an Accept header line for
|
||||
// each of accept, and returns the response.
|
||||
func requestImage(
|
||||
t *testing.T, srv http.Handler, method, target string, accept ...string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), method, target, nil)
|
||||
|
||||
for _, value := range accept {
|
||||
req.Header.Add("Accept", value)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
t.Logf("%s %s with Accept %q: %d, Content-Type %s, Vary %v, X-Pixa-Cache %s",
|
||||
method, target, accept, rec.Code, rec.Header().Get("Content-Type"),
|
||||
rec.Header().Values("Vary"), rec.Header().Get("X-Pixa-Cache"))
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// TestFormatAuto_ChosenFromAccept requests an auto URL on each image route
|
||||
// with each Accept below, and checks the answer and that it carries
|
||||
// Vary: Accept. The signed URL is signed for auto, so it is valid whatever
|
||||
// Accept chooses.
|
||||
func TestFormatAuto_ChosenFromAccept(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept []string
|
||||
wantStatus int
|
||||
wantType string
|
||||
}{
|
||||
{"AVIF accepted", []string{"image/avif,image/webp,*/*;q=0.8"},
|
||||
http.StatusOK, avifType},
|
||||
{"WebP accepted", []string{"image/webp,*/*;q=0.8"}, http.StatusOK, webpType},
|
||||
{"no Accept", nil, http.StatusOK, jpegType},
|
||||
{"two Accept lines", []string{"image/png", webpType}, http.StatusOK, webpType},
|
||||
{"none of the three", []string{"image/gif"},
|
||||
http.StatusNotAcceptable, jsonType},
|
||||
{"not valid", []string{"image/avif;q=high"}, http.StatusBadRequest, jsonType},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, target := range []string{signedURL, encryptedURL} {
|
||||
rec := requestImage(t, srv, http.MethodGet, target, tt.accept...)
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
|
||||
if rec.Code != tt.wantStatus || gotType != tt.wantType {
|
||||
t.Errorf("%s: %d %s, want %d %s; body %s", target,
|
||||
rec.Code, gotType, tt.wantStatus, tt.wantType, rec.Body)
|
||||
}
|
||||
|
||||
if !slices.Contains(rec.Header().Values("Vary"), "Accept") {
|
||||
t.Errorf("%s: Vary = %v, want Accept in it",
|
||||
target, rec.Header().Values("Vary"))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_SignatureCoversAuto verifies that a /v1/image/ URL with the
|
||||
// format auto is checked against a signature for auto, not for the format
|
||||
// chosen: a URL signed for avif, with auto put in its path, is refused for a
|
||||
// client whose Accept chooses AVIF.
|
||||
func TestFormatAuto_SignatureCoversAuto(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
signedForAVIF, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatAVIF,
|
||||
},
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1)
|
||||
if target == signedForAVIF {
|
||||
t.Fatalf("no /50x50.avif? in %s", signedForAVIF)
|
||||
}
|
||||
|
||||
rec := requestImage(t, srv, http.MethodGet, target, avifType)
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("status = %d, want %d", rec.Code, http.StatusUnauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_CachesEachFormatApart requests an auto URL for AVIF, then
|
||||
// JPEG, then both again. Each format is processed once and then served from
|
||||
// the cache, with an ETag of its own, so a client never gets the other format
|
||||
// from the cache.
|
||||
func TestFormatAuto_CachesEachFormatApart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, _ := autoPhotoURLs(t, h)
|
||||
|
||||
steps := []struct {
|
||||
wantType string
|
||||
wantCache string
|
||||
}{
|
||||
{avifType, "MISS"},
|
||||
{jpegType, "MISS"},
|
||||
{avifType, "HIT"},
|
||||
{jpegType, "HIT"},
|
||||
}
|
||||
|
||||
etags := make(map[string]string)
|
||||
|
||||
for _, step := range steps {
|
||||
rec := requestImage(t, srv, http.MethodGet, signedURL, step.wantType)
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
gotCache := rec.Header().Get("X-Pixa-Cache")
|
||||
|
||||
if rec.Code != http.StatusOK || gotType != step.wantType ||
|
||||
gotCache != step.wantCache {
|
||||
t.Fatalf("Accept %s: %d %s %s, want 200 %s %s", step.wantType,
|
||||
rec.Code, gotType, gotCache, step.wantType, step.wantCache)
|
||||
}
|
||||
|
||||
etag := rec.Header().Get("ETag")
|
||||
if previous, seen := etags[gotType]; seen && previous != etag {
|
||||
t.Errorf("%s ETag changed from %s to %s", gotType, previous, etag)
|
||||
}
|
||||
|
||||
etags[gotType] = etag
|
||||
}
|
||||
|
||||
if etags[avifType] == etags[jpegType] {
|
||||
t.Errorf("AVIF and JPEG have the same ETag %s", etags[avifType])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_Vary verifies that on each image route a HEAD answer and a
|
||||
// 304 for an auto URL carry Vary: Accept, and that the answer for a URL with
|
||||
// a fixed format does not.
|
||||
func TestFormatAuto_Vary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
srv := chi.NewRouter()
|
||||
srv.Get("/v1/image/*", h.HandleImage())
|
||||
srv.Head("/v1/image/*", h.HandleImage())
|
||||
srv.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
srv.Head("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
for _, urls := range [][2]string{
|
||||
{signedURL, signedPhotoURL(t, h)},
|
||||
{encryptedURL, encPhotoURL(t, h)},
|
||||
} {
|
||||
autoURL, fixedURL := urls[0], urls[1]
|
||||
|
||||
head := requestImage(t, srv, http.MethodHead, autoURL, webpType)
|
||||
checkVaryAccept(t, head, http.StatusOK, true)
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
autoURL, nil)
|
||||
req.Header.Set("Accept", webpType)
|
||||
req.Header.Set("If-None-Match", head.Header().Get("ETag"))
|
||||
|
||||
notModified := httptest.NewRecorder()
|
||||
srv.ServeHTTP(notModified, req)
|
||||
checkVaryAccept(t, notModified, http.StatusNotModified, true)
|
||||
|
||||
fixed := requestImage(t, srv, http.MethodGet, fixedURL)
|
||||
checkVaryAccept(t, fixed, http.StatusOK, false)
|
||||
}
|
||||
}
|
||||
|
||||
// checkVaryAccept fails the test unless rec answered wantStatus and, as
|
||||
// wantVary says, has or has not Accept in its Vary header.
|
||||
func checkVaryAccept(
|
||||
t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantVary bool,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
vary := rec.Header().Values("Vary")
|
||||
t.Logf("status %d, Vary %v", rec.Code, vary)
|
||||
|
||||
if rec.Code != wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, wantStatus)
|
||||
}
|
||||
|
||||
if slices.Contains(vary, "Accept") != wantVary {
|
||||
t.Errorf("Vary = %v, want Accept in it: %v", vary, wantVary)
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,11 @@ package handlers
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/allowlist"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
@@ -29,11 +27,6 @@ type Params struct {
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Database *database.Database
|
||||
Config *config.Config
|
||||
|
||||
// Fetcher, when provided, fetches upstream images in place of the
|
||||
// fetcher the handlers build from the config. Only tests provide one;
|
||||
// pixad does not.
|
||||
Fetcher httpfetcher.Fetcher `optional:"true"`
|
||||
}
|
||||
|
||||
// Handlers provides HTTP request handlers.
|
||||
@@ -42,16 +35,11 @@ type Handlers struct {
|
||||
hc *healthcheck.Healthcheck
|
||||
db *database.Database
|
||||
config *config.Config
|
||||
fetcher httpfetcher.Fetcher
|
||||
imgSvc *imgcache.Service
|
||||
imgCache *imgcache.Cache
|
||||
sessMgr *session.Manager
|
||||
encGen *encurl.Generator
|
||||
csrfProtect func(http.Handler) http.Handler
|
||||
|
||||
// refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
|
||||
// reports whether a URL's host is on that list.
|
||||
refererBlocklist *allowlist.HostAllowList
|
||||
}
|
||||
|
||||
// New creates a new Handlers instance.
|
||||
@@ -62,61 +50,49 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
|
||||
}
|
||||
|
||||
s := &Handlers{
|
||||
log: params.Logger.Get(),
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
fetcher: params.Fetcher,
|
||||
csrfProtect: csrfProtect,
|
||||
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
|
||||
log: params.Logger.Get(),
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
csrfProtect: csrfProtect,
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
//nolint:contextcheck // the cache's goroutines outlive OnStart; OnStop stops them
|
||||
// The eviction goroutine must outlive OnStart, so it cannot
|
||||
// inherit this hook's context. It makes its own instead, which
|
||||
// leaves it uncancellable: an in-flight pass runs to completion
|
||||
// during OnStop regardless of the shutdown deadline. Making the
|
||||
// loop cancellable changes shutdown semantics and is tracked
|
||||
// separately in issue #102, rather than being folded into the
|
||||
// lint-conformance change that surfaced it.
|
||||
//nolint:contextcheck // see issue #102
|
||||
OnStart: func(_ context.Context) error {
|
||||
return s.initImageService()
|
||||
},
|
||||
// The pending counts and access times are written here, before the
|
||||
// database's stop hook closes it.
|
||||
OnStop: func(ctx context.Context) error {
|
||||
if s.imgCache == nil {
|
||||
return nil
|
||||
OnStop: func(_ context.Context) error {
|
||||
if s.imgCache != nil {
|
||||
s.imgCache.StopEviction()
|
||||
}
|
||||
|
||||
return errors.Join(
|
||||
s.imgCache.StopEviction(ctx),
|
||||
s.imgCache.StopPendingCountWrites(ctx),
|
||||
)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// WaitForProcessing waits until no image is being processed, or until ctx
|
||||
// ends, and returns how many images were still being processed then.
|
||||
func (s *Handlers) WaitForProcessing(ctx context.Context) int {
|
||||
return s.imgSvc.WaitForProcessing(ctx)
|
||||
}
|
||||
|
||||
// newCacheConfig builds the image cache's configuration from cfg.
|
||||
// cache_max_bytes: 0 disables the disk cache entirely; any other value
|
||||
// is the eviction limit in bytes; when it is omitted, the cache works
|
||||
// out the default limit itself.
|
||||
func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig {
|
||||
return imgcache.CacheConfig{
|
||||
StateDir: cfg.StateDir,
|
||||
NegativeTTL: imgcache.DefaultNegativeTTL,
|
||||
MaxBytes: cfg.CacheMaxBytes,
|
||||
UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit,
|
||||
DisableDiskCache: cfg.CacheMaxBytesExplicit && cfg.CacheMaxBytes == 0,
|
||||
Logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
// initImageService initializes the image cache and service.
|
||||
func (s *Handlers) initImageService() error {
|
||||
cache, err := imgcache.NewCache(s.db.DB(), newCacheConfig(s.config, s.log))
|
||||
// Create the cache. cache_max_bytes: 0 disables the disk cache
|
||||
// entirely; any other value is the eviction limit in bytes.
|
||||
cache, err := imgcache.NewCache(s.db.DB(), imgcache.CacheConfig{
|
||||
StateDir: s.config.StateDir,
|
||||
CacheTTL: imgcache.DefaultCacheTTL,
|
||||
NegativeTTL: imgcache.DefaultNegativeTTL,
|
||||
MaxBytes: s.config.CacheMaxBytes,
|
||||
DisableDiskCache: s.config.CacheMaxBytes == 0,
|
||||
Logger: s.log,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -124,14 +100,9 @@ func (s *Handlers) initImageService() error {
|
||||
s.imgCache = cache
|
||||
|
||||
// Background eviction: startup reconciliation, then periodic and
|
||||
// write-pressure passes, and the removal of expired negative cache
|
||||
// rows, which is all it does when the disk cache is disabled.
|
||||
// write-pressure passes. No-op when the disk cache is disabled.
|
||||
cache.StartEviction(imgcache.DefaultEvictionInterval)
|
||||
|
||||
// Writes the counts and the access times of cache hits that requests
|
||||
// hand over, so that no request waits for the database to record them
|
||||
cache.StartPendingCountWrites()
|
||||
|
||||
// Create the fetcher config
|
||||
fetcherCfg := httpfetcher.DefaultConfig()
|
||||
fetcherCfg.AllowHTTP = s.config.AllowHTTP
|
||||
@@ -145,12 +116,10 @@ func (s *Handlers) initImageService() error {
|
||||
fetcherCfg.MaxConnections = s.config.UpstreamConnections
|
||||
fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
|
||||
|
||||
// Create the service. With no fetcher provided, it builds its own from
|
||||
// fetcherCfg.
|
||||
// Create the service
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
FetcherConfig: fetcherCfg,
|
||||
Fetcher: s.fetcher,
|
||||
SigningKey: s.config.SigningKey,
|
||||
Allowlist: s.config.AllowlistHosts,
|
||||
MaxConcurrentProcessing: s.config.MaxConcurrentProcessing,
|
||||
@@ -165,10 +134,7 @@ func (s *Handlers) initImageService() error {
|
||||
|
||||
// Initialize session manager (signing key is validated at config load
|
||||
// time). Session cookies are always Secure/HttpOnly/SameSite=Strict.
|
||||
// The random value that each logout replaces is saved in the database,
|
||||
// so it outlasts a restart.
|
||||
sessMgr, err := session.NewManagerWithDatabase(
|
||||
context.Background(), s.config.SigningKey, s.db.DB())
|
||||
sessMgr, err := session.NewManager(s.config.SigningKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ func setupTestHandler(t *testing.T) *testFixtures {
|
||||
|
||||
cache, err := imgcache.NewCache(db, imgcache.CacheConfig{
|
||||
StateDir: tmpDir,
|
||||
CacheTTL: time.Hour,
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
+41
-126
@@ -9,24 +9,17 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/magic"
|
||||
)
|
||||
|
||||
// HandleImage handles the main image proxy route:
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>, or with no format
|
||||
// /v1/image/<host>/<path>/<width>x<height>
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
req, ok := s.parseImageRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
@@ -36,8 +29,8 @@ func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
err := s.imgSvc.ValidateRequest(req)
|
||||
if err != nil {
|
||||
s.log.Warn("signature validation failed",
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
"host", req.SourceHost,
|
||||
"path", req.SourcePath,
|
||||
"error", err,
|
||||
)
|
||||
s.respondError(w, "unauthorized", http.StatusUnauthorized)
|
||||
@@ -45,13 +38,8 @@ func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// The signature covers the format auto, not the format chosen
|
||||
if !s.chooseAutoFormat(w, r, req) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get cache key for logging
|
||||
cacheKey := imagerequest.CacheKey(req)
|
||||
cacheKey := imgcache.CacheKey(req)
|
||||
|
||||
// Get the image (from cache or fetch/process)
|
||||
startTime := time.Now()
|
||||
@@ -81,21 +69,20 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// parseImageRequest parses the image URL's path and query parameters into
|
||||
// an image request. On invalid input it writes an error response and
|
||||
// parseImageRequest parses the wildcard path and query parameters into
|
||||
// an ImageRequest. On invalid input it writes an error response and
|
||||
// returns false.
|
||||
func (s *Handlers) parseImageRequest(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) (*imagerequest.Request, bool) {
|
||||
// The path is parsed decoded once, as r.URL.Path holds it, and not from
|
||||
// chi's wildcard: chi routes on the path as the client wrote it whenever
|
||||
// that is not Go's own encoding of it, so its wildcard would still hold
|
||||
// the %3F before a source query when, say, a "/" in the query is written
|
||||
// as %2F
|
||||
req, err := imagerequest.ParseImageURL(r.URL.Path)
|
||||
) (*imgcache.ImageRequest, bool) {
|
||||
// Get the wildcard path from chi
|
||||
pathParam := chi.URLParam(r, "*")
|
||||
|
||||
// Parse the URL path
|
||||
parsed, err := imgcache.ParseImagePath(pathParam)
|
||||
if err != nil {
|
||||
s.log.Warn("failed to parse image URL",
|
||||
"path", r.URL.Path,
|
||||
"path", pathParam,
|
||||
"error", err,
|
||||
)
|
||||
s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest)
|
||||
@@ -103,6 +90,9 @@ func (s *Handlers) parseImageRequest(
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Convert to ImageRequest
|
||||
req := parsed.ToImageRequest()
|
||||
|
||||
// Parse signature params from query string. r.URL.Query() would silently
|
||||
// drop a pair it cannot decode, such as q=80%, so that q would be served
|
||||
// at 85; a query string that cannot be decoded is refused instead. A
|
||||
@@ -146,7 +136,7 @@ func (s *Handlers) parseImageRequest(
|
||||
return nil, false
|
||||
}
|
||||
|
||||
req.Transform.Quality, err = parseFormInt(query, "q",
|
||||
req.Quality, err = parseFormInt(query, "q",
|
||||
encurl.DefaultQuality, minQuality, maxQuality)
|
||||
if err != nil {
|
||||
s.respondError(w, fmt.Sprintf("%v, got %q", err, qStr),
|
||||
@@ -158,23 +148,23 @@ func (s *Handlers) parseImageRequest(
|
||||
// Only a fit missing from the URL is cover. A fit in the URL that is not a
|
||||
// fit mode is refused by the fit-mode check below; that check would take an
|
||||
// empty fit as missing, so an empty one is refused here.
|
||||
req.Transform.FitMode = imagerequest.Fit(query.Get("fit"))
|
||||
if query.Has("fit") && req.Transform.FitMode == "" {
|
||||
req.FitMode = imgcache.FitMode(query.Get("fit"))
|
||||
if query.Has("fit") && req.FitMode == "" {
|
||||
s.respondError(w, `invalid fit: not a fit mode, got ""`, http.StatusBadRequest)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Default fit mode if not set
|
||||
if req.Transform.FitMode == "" {
|
||||
req.Transform.FitMode = imagerequest.FitCover
|
||||
if req.FitMode == "" {
|
||||
req.FitMode = imgcache.FitCover
|
||||
}
|
||||
|
||||
// Enforce dimension and fit-mode bounds, shared with the encrypted-URL
|
||||
// route. Dimensions are already bounded by the path parser above; this
|
||||
// also rejects an unrecognized fit mode with 400 instead of letting it
|
||||
// reach the processor as a 500.
|
||||
err = imagerequest.ValidateImageRequest(req)
|
||||
err = imgcache.ValidateImageRequest(req)
|
||||
if err != nil {
|
||||
s.respondError(w, "invalid image request: "+err.Error(),
|
||||
http.StatusBadRequest)
|
||||
@@ -205,35 +195,13 @@ func parseExpires(query url.Values) (time.Time, error) {
|
||||
return time.Unix(exp, 0), nil
|
||||
}
|
||||
|
||||
// HandleTooManyRequests answers 429 Too Many Requests with the JSON error
|
||||
// body the image routes send for their other errors. The image routes' rate
|
||||
// limit calls it for a request over the limit.
|
||||
func (s *Handlers) HandleTooManyRequests() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, _ *http.Request) {
|
||||
s.respondError(w, "too many requests, try again later",
|
||||
http.StatusTooManyRequests)
|
||||
}
|
||||
}
|
||||
|
||||
// respondImageError maps image retrieval errors to HTTP responses.
|
||||
func (s *Handlers) respondImageError(
|
||||
w http.ResponseWriter, req *imagerequest.Request, err error,
|
||||
w http.ResponseWriter, req *imgcache.ImageRequest, err error,
|
||||
) {
|
||||
// An upstream body whose first bytes are not those of the format its
|
||||
// Content-Type names is the upstream's error, not pixa's: the image
|
||||
// service has logged it as a refused source.
|
||||
if errors.Is(err, magic.ErrMagicByteMismatch) ||
|
||||
errors.Is(err, magic.ErrUnknownFormat) ||
|
||||
errors.Is(err, magic.ErrNotEnoughData) {
|
||||
s.respondError(w, "unsupported source format",
|
||||
http.StatusUnsupportedMediaType)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Error("failed to get image",
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
"host", req.SourceHost,
|
||||
"path", req.SourcePath,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
@@ -251,38 +219,13 @@ func (s *Handlers) respondImageError(
|
||||
}
|
||||
|
||||
if errors.Is(err, httpfetcher.ErrTooManyConnections) ||
|
||||
errors.Is(err, imageprocessor.ErrTooManyImages) ||
|
||||
errors.Is(err, imgcache.ErrProcessingTimedOut) {
|
||||
errors.Is(err, imageprocessor.ErrTooManyImages) {
|
||||
s.respondError(w, "server busy, try again later",
|
||||
http.StatusServiceUnavailable)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, httpfetcher.ErrInvalidContentType) ||
|
||||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat) {
|
||||
s.respondError(w, "unsupported source format",
|
||||
http.StatusUnsupportedMediaType)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, imageprocessor.ErrOutputTooLarge) {
|
||||
s.respondError(w, fmt.Sprintf("size too large for the source: the side "+
|
||||
"computed from its proportions is above %d",
|
||||
imageprocessor.MaxOutputDimension), http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge) {
|
||||
s.respondError(w, fmt.Sprintf("size too large for AVIF: the output "+
|
||||
"would be above %d pixels", imageprocessor.MaxAVIFOutputPixels),
|
||||
http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.respondError(w, "internal error", http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
@@ -304,48 +247,12 @@ func cacheControl(expires time.Time) string {
|
||||
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
||||
}
|
||||
|
||||
// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
|
||||
// names a host on referer_blocklist, and reports whether it answered. A request
|
||||
// with no Referer, or one that does not parse as a URL with a host, is not
|
||||
// refused.
|
||||
func (s *Handlers) refuseBlockedReferer(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) bool {
|
||||
referer, err := url.Parse(r.Referer())
|
||||
if err != nil || !s.refererBlocklist.IsAllowed(referer) {
|
||||
return false
|
||||
}
|
||||
|
||||
s.respondError(w, "referer blocked", http.StatusForbidden)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// notModified sets the ETag header to etag and, when the request's
|
||||
// If-None-Match is that ETag, answers 304 Not Modified. It reports whether it
|
||||
// answered. An empty etag sets no header and never answers.
|
||||
func notModified(w http.ResponseWriter, r *http.Request, etag string) bool {
|
||||
if etag == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
w.Header().Set("ETag", etag)
|
||||
|
||||
if r.Header.Get("If-None-Match") != etag {
|
||||
return false
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// writeImageResponse writes headers and streams the image content,
|
||||
// handling conditional and HEAD requests.
|
||||
func (s *Handlers) writeImageResponse(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
req *imagerequest.Request, resp *imgcache.ImageResponse,
|
||||
cacheKey imagerequest.VariantKey, startTime time.Time,
|
||||
req *imgcache.ImageRequest, resp *imgcache.ImageResponse,
|
||||
cacheKey imgcache.VariantKey, startTime time.Time,
|
||||
) {
|
||||
// Set response headers
|
||||
w.Header().Set("Content-Type", resp.ContentType)
|
||||
@@ -358,8 +265,17 @@ func (s *Handlers) writeImageResponse(
|
||||
w.Header().Set("Cache-Control", cacheControl(req.Expires))
|
||||
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
|
||||
|
||||
if notModified(w, r, resp.ETag) {
|
||||
return
|
||||
if resp.ETag != "" {
|
||||
w.Header().Set("ETag", resp.ETag)
|
||||
|
||||
// Check for conditional request (If-None-Match)
|
||||
if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
|
||||
if ifNoneMatch == resp.ETag {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle HEAD request - return headers only
|
||||
@@ -382,11 +298,10 @@ func (s *Handlers) writeImageResponse(
|
||||
// Log cache status and timing after serving
|
||||
duration := time.Since(startTime)
|
||||
s.log.Info("image served",
|
||||
"request_id", middleware.GetReqID(r.Context()),
|
||||
"cache_key", cacheKey,
|
||||
"cache_status", resp.CacheStatus,
|
||||
"duration_ms", duration.Milliseconds(),
|
||||
"format", req.Transform.Format,
|
||||
"format", req.Format,
|
||||
"served_bytes", servedBytes,
|
||||
"fetched_bytes", resp.FetchedBytes,
|
||||
)
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
@@ -24,14 +23,13 @@ const photoPath = "/images/photo.jpg"
|
||||
// newSignedHostServer returns a router for both image routes, and the Handlers
|
||||
// behind it, whose fetcher serves a JPEG at photoPath on signedHost. signedHost
|
||||
// is not on the allowlist, so a /v1/image/ URL for it is served only with a
|
||||
// valid signature. The handlers and the image service log to log.
|
||||
func newSignedHostServer(
|
||||
t *testing.T, log *slog.Logger,
|
||||
) (*Handlers, http.Handler) {
|
||||
// valid signature.
|
||||
func newSignedHostServer(t *testing.T) (*Handlers, http.Handler) {
|
||||
t.Helper()
|
||||
|
||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||
StateDir: t.TempDir(),
|
||||
CacheTTL: time.Hour,
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -46,7 +44,6 @@ func newSignedHostServer(
|
||||
signedHost + photoPath: &fstest.MapFile{Data: jpegData},
|
||||
}),
|
||||
SigningKey: testSigningKey,
|
||||
Logger: log,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||
@@ -58,7 +55,7 @@ func newSignedHostServer(
|
||||
}
|
||||
|
||||
h := &Handlers{
|
||||
log: log,
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
imgSvc: svc,
|
||||
encGen: encGen,
|
||||
}
|
||||
@@ -106,17 +103,13 @@ func getMaxAge(t *testing.T, srv http.Handler, target string) int {
|
||||
func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
h, srv := newSignedHostServer(t)
|
||||
|
||||
signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
signedURL, err := h.imgSvc.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Size: imgcache.Size{Width: 50, Height: 50},
|
||||
Format: imgcache.FormatJPEG,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
@@ -173,31 +166,28 @@ func TestHandleImageEnc_MaxAge(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
expires time.Time
|
||||
expiresAt int64
|
||||
wantAtLeast int
|
||||
wantAtMost int
|
||||
}{
|
||||
{"60 second TTL", time.Now().Add(time.Minute), 50, 60},
|
||||
{"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour), 31536000, 31536000},
|
||||
{"no TTL", time.Time{}, 31536000, 31536000},
|
||||
{"60 second TTL", time.Now().Add(time.Minute).Unix(), 50, 60},
|
||||
{"two-year TTL", time.Now().Add(2 * 365 * 24 * time.Hour).Unix(), 31536000, 31536000},
|
||||
{"no TTL", 0, 31536000, 31536000},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
h, srv := newSignedHostServer(t)
|
||||
|
||||
token, err := h.encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
Expires: tt.expires,
|
||||
token, err := h.encGen.Generate(&encurl.Payload{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
Format: imgcache.FormatJPEG,
|
||||
ExpiresAt: tt.expiresAt,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image/color"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/signature"
|
||||
)
|
||||
|
||||
// allowlistedHost is the only host on the allowlist of the image route
|
||||
// newImageRoute builds.
|
||||
const allowlistedHost = "allowed.example.com"
|
||||
|
||||
// newImageRoute returns the image route of a Handlers whose service fetches
|
||||
// with fetcher and checks signatures with testSigningKey.
|
||||
func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler {
|
||||
t.Helper()
|
||||
|
||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||
StateDir: t.TempDir(),
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create cache: %v", err)
|
||||
}
|
||||
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
Fetcher: fetcher,
|
||||
SigningKey: testSigningKey,
|
||||
Allowlist: []string{allowlistedHost},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create service: %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{imgSvc: svc, log: slog.New(slog.DiscardHandler)}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// newPhotoFetcher returns a mock fetcher that serves a JPEG at photoPath on
|
||||
// each of hosts, and answers any other URL with an upstream error.
|
||||
func newPhotoFetcher(t *testing.T, hosts ...string) *httpfetcher.MockFetcher {
|
||||
t.Helper()
|
||||
|
||||
photo := &fstest.MapFile{
|
||||
Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
|
||||
}
|
||||
|
||||
files := fstest.MapFS{}
|
||||
for _, host := range hosts {
|
||||
files[host+photoPath] = photo
|
||||
}
|
||||
|
||||
return httpfetcher.NewMock(files)
|
||||
}
|
||||
|
||||
// photoURL returns the image route URL of photoPath on host, as a 50x50 JPEG.
|
||||
func photoURL(host string) string {
|
||||
return "/v1/image/" + host + photoPath + "/50x50.jpeg"
|
||||
}
|
||||
|
||||
// photoURLWithSig returns photoURL(host) with sig and expires as its sig and
|
||||
// exp.
|
||||
func photoURLWithSig(host, sig string, expires time.Time) string {
|
||||
return fmt.Sprintf("%s?sig=%s&exp=%d", photoURL(host), sig, expires.Unix())
|
||||
}
|
||||
|
||||
// photoSignature returns the signature of photoURL(host) at the default
|
||||
// quality and fit, made with key and expiring at expires.
|
||||
func photoSignature(key, host string, expires time.Time) string {
|
||||
return signature.New(key).Sign(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: host,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
Expires: expires,
|
||||
})
|
||||
}
|
||||
|
||||
// sendGet sends a GET for target to route and returns the response.
|
||||
func sendGet(
|
||||
t *testing.T, route http.Handler, target string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
route.ServeHTTP(rec, req)
|
||||
t.Logf("GET %s: %d", target, rec.Code)
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// checkErrorBody checks that rec has status wantStatus and the JSON error body
|
||||
// the image route sends: wantError, wantStatus and the time in RFC 3339.
|
||||
func checkErrorBody(
|
||||
t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantError string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, wantStatus)
|
||||
}
|
||||
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
|
||||
t.Errorf("Content-Type = %q, want application/json", ct)
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Error string `json:"error"`
|
||||
Status int `json:"status"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
err := json.NewDecoder(rec.Body).Decode(&body)
|
||||
if err != nil {
|
||||
t.Fatalf("decoding response body: %v", err)
|
||||
}
|
||||
|
||||
if body.Error != wantError || body.Status != wantStatus {
|
||||
t.Errorf("body error and status = %q %d, want %q %d",
|
||||
body.Error, body.Status, wantError, wantStatus)
|
||||
}
|
||||
|
||||
_, err = time.Parse(time.RFC3339, body.Timestamp)
|
||||
if err != nil {
|
||||
t.Errorf("body timestamp: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_ErrorAnswers checks the status and the JSON error body the
|
||||
// image route answers each request below with. The JPEG at photoPath exists on
|
||||
// signedHost and on each host below that differs from it, so a request refused
|
||||
// with 401 would otherwise be served.
|
||||
func TestHandleImage_ErrorAnswers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A signature for signedHost must not verify for any of these.
|
||||
parentHost := "example.com"
|
||||
siblingHost := "other.example.com"
|
||||
subdomainHost := "img." + signedHost
|
||||
appendedHost := signedHost + ".example.net"
|
||||
|
||||
photos := newPhotoFetcher(t,
|
||||
signedHost, parentHost, siblingHost, subdomainHost, appendedHost)
|
||||
// The real fetcher refuses localhost before any lookup or connection.
|
||||
realFetcher := httpfetcher.New(httpfetcher.DefaultConfig())
|
||||
|
||||
exp := time.Now().Add(time.Hour)
|
||||
expired := time.Now().Add(-time.Hour)
|
||||
sig := photoSignature(testSigningKey, signedHost, exp)
|
||||
otherKeySig := photoSignature("another-signing-key", signedHost, exp)
|
||||
expiredSig := photoSignature(testSigningKey, signedHost, expired)
|
||||
localhostSig := photoSignature(testSigningKey, "localhost", exp)
|
||||
|
||||
// The error every request refused for its signature gets.
|
||||
const unauthorized = "unauthorized"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
fetcher httpfetcher.Fetcher
|
||||
target string
|
||||
wantStatus int
|
||||
wantError string
|
||||
}{
|
||||
{"no sig or exp", photos, photoURL(signedHost),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"exp but no sig", photos,
|
||||
fmt.Sprintf("%s?exp=%d", photoURL(signedHost), exp.Unix()),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig made with another key", photos,
|
||||
photoURLWithSig(signedHost, otherKeySig, exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig without its = padding", photos,
|
||||
photoURLWithSig(signedHost, strings.TrimRight(sig, "="), exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig in upper case", photos,
|
||||
photoURLWithSig(signedHost, strings.ToUpper(sig), exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"expired sig", photos, photoURLWithSig(signedHost, expiredSig, expired),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig sent for the parent domain", photos,
|
||||
photoURLWithSig(parentHost, sig, exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig sent for a sibling host", photos,
|
||||
photoURLWithSig(siblingHost, sig, exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig sent for a subdomain", photos,
|
||||
photoURLWithSig(subdomainHost, sig, exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"sig sent with another domain appended", photos,
|
||||
photoURLWithSig(appendedHost, sig, exp),
|
||||
http.StatusUnauthorized, unauthorized},
|
||||
{"unparseable path", photos,
|
||||
"/v1/image/" + allowlistedHost + photoPath + "/big.jpeg",
|
||||
http.StatusBadRequest, "invalid image URL: invalid size format"},
|
||||
{"blocked upstream address", realFetcher,
|
||||
photoURLWithSig("localhost", localhostSig, exp),
|
||||
http.StatusForbidden, "forbidden"},
|
||||
{"upstream error", photos,
|
||||
"/v1/image/" + allowlistedHost + "/images/missing.jpg/50x50.jpeg",
|
||||
http.StatusBadGateway, "upstream error"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rec := sendGet(t, newImageRoute(t, tt.fetcher), tt.target)
|
||||
checkErrorBody(t, rec, tt.wantStatus, tt.wantError)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_AllowlistOrSignature checks that the image route serves an
|
||||
// image without a signature for a host on the allowlist only, and for another
|
||||
// host only with a valid signature.
|
||||
func TestHandleImage_AllowlistOrSignature(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photos := newPhotoFetcher(t, allowlistedHost, signedHost)
|
||||
exp := time.Now().Add(time.Hour)
|
||||
sig := photoSignature(testSigningKey, signedHost, exp)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
target string
|
||||
wantStatus int
|
||||
}{
|
||||
{"allowlisted host, no sig", photoURL(allowlistedHost), http.StatusOK},
|
||||
{"other host, no sig", photoURL(signedHost), http.StatusUnauthorized},
|
||||
{"other host, valid sig", photoURLWithSig(signedHost, sig, exp),
|
||||
http.StatusOK},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rec := sendGet(t, newImageRoute(t, photos), tt.target)
|
||||
if rec.Code != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_InvalidHost_Returns400 sends image URLs whose host no URL
|
||||
// host can be, and checks that each is refused with 400 naming the host as
|
||||
// decoded: allowlistedHost with a character after it that a URL host cannot
|
||||
// hold, percent-encoded, an IPv6 address without brackets, and brackets
|
||||
// around something that is not an IPv6 address. A "?" or "#" after
|
||||
// allowlistedHost would otherwise end the host when the source URL is read,
|
||||
// so the request would pass the allowlist and be fetched and cached under a
|
||||
// host that is not the one fetched.
|
||||
func TestHandleImage_InvalidHost_Returns400(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photos := newPhotoFetcher(t, allowlistedHost)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
writtenHost string
|
||||
decodedHost string
|
||||
}{
|
||||
{"? written as %3F", allowlistedHost + "%3Fx=1", allowlistedHost + "?x=1"},
|
||||
{"# written as %23", allowlistedHost + "%23x", allowlistedHost + "#x"},
|
||||
{"% written as %25", allowlistedHost + "%253F", allowlistedHost + "%3F"},
|
||||
{`\ written as %5C`, allowlistedHost + "%5Cx", allowlistedHost + `\x`},
|
||||
{"space written as %20", allowlistedHost + "%20", allowlistedHost + " "},
|
||||
{"IPv6 address without brackets", "2001:db8::1", "2001:db8::1"},
|
||||
{"IPv4 address in brackets", "[203.0.113.5]", "[203.0.113.5]"},
|
||||
{"no address in brackets", "[abc]", "[abc]"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rec := sendGet(t, newImageRoute(t, photos), photoURL(tt.writtenHost))
|
||||
checkErrorBody(t, rec, http.StatusBadRequest,
|
||||
fmt.Sprintf("invalid image URL: invalid source host: %q",
|
||||
tt.decodedHost))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_HostForms_Served checks that the image route serves, with
|
||||
// a valid signature, the image at photoPath on a host name, an IPv4 address,
|
||||
// an IPv6 address in brackets, and a host followed by ":" with or without a
|
||||
// port.
|
||||
func TestHandleImage_HostForms_Served(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hosts := []string{
|
||||
allowlistedHost,
|
||||
"img_1.example.com",
|
||||
"203.0.113.5",
|
||||
"[2001:db8::1]",
|
||||
allowlistedHost + ":8443",
|
||||
"[2001:db8::1]:8443",
|
||||
allowlistedHost + ":",
|
||||
}
|
||||
|
||||
photos := newPhotoFetcher(t, hosts...)
|
||||
exp := time.Now().Add(time.Hour)
|
||||
|
||||
for _, host := range hosts {
|
||||
t.Run(host, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sig := photoSignature(testSigningKey, host, exp)
|
||||
|
||||
rec := sendGet(t, newImageRoute(t, photos),
|
||||
photoURLWithSig(host, sig, exp))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("status = %d, want %d: %s",
|
||||
rec.Code, http.StatusOK, rec.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_HashInSourcePathOrQuery_Returns400 sends image URLs whose
|
||||
// source path or source query holds a "#", written as %23, and checks that
|
||||
// each is refused with 400, the one signed for that source path included. The
|
||||
// source URL would otherwise end at the "#" when it is fetched, so each text
|
||||
// after it would fetch the image at photoPath and cache it under keys of its
|
||||
// own.
|
||||
func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photos := newPhotoFetcher(t, allowlistedHost, signedHost)
|
||||
exp := time.Now().Add(time.Hour)
|
||||
|
||||
sig := signature.New(testSigningKey).Sign(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath + "#x",
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
Expires: exp,
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
target string
|
||||
}{
|
||||
{"# in the source path",
|
||||
"/v1/image/" + allowlistedHost + photoPath + "%23x/50x50.jpeg"},
|
||||
{"# in the source query",
|
||||
"/v1/image/" + allowlistedHost + photoPath + "%3Fv=1%23x/50x50.jpeg"},
|
||||
{"# in the source path, signed",
|
||||
fmt.Sprintf("/v1/image/%s%s%%23x/50x50.jpeg?sig=%s&exp=%d",
|
||||
signedHost, photoPath, sig, exp.Unix())},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rec := sendGet(t, newImageRoute(t, photos), tt.target)
|
||||
checkErrorBody(t, rec, http.StatusBadRequest,
|
||||
`invalid image URL: invalid image path: `+
|
||||
`the source path or query holds a "#"`)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/signature"
|
||||
)
|
||||
|
||||
@@ -44,18 +44,15 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) {
|
||||
|
||||
expires := time.Now().Add(time.Hour)
|
||||
signer := signature.New("test-signing-key-must-be-32-chars")
|
||||
sig := signer.Sign(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: "/images/photo.jpg",
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
Expires: expires,
|
||||
sig := signer.Sign(&signature.Request{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: "/images/photo.jpg",
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
Format: string(imgcache.FormatJPEG),
|
||||
Quality: 85,
|
||||
FitMode: string(imgcache.FitCover),
|
||||
Expires: expires,
|
||||
})
|
||||
signedURL := fmt.Sprintf("/v1/image/%s/images/photo.jpg/50x50.jpeg?sig=%s&exp=%d",
|
||||
signedHost, sig, expires.Unix())
|
||||
@@ -91,9 +88,9 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
quality int
|
||||
fitMode imagerequest.Fit
|
||||
fitMode imgcache.FitMode
|
||||
}{
|
||||
{"quality 40 and fit contain", 40, imagerequest.FitContain},
|
||||
{"quality 40 and fit contain", 40, imgcache.FitContain},
|
||||
{"quality and fit unset", 0, ""},
|
||||
}
|
||||
|
||||
@@ -103,17 +100,13 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) {
|
||||
|
||||
fix := setupTestHandler(t)
|
||||
|
||||
signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: "/images/photo.jpg",
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: tt.quality,
|
||||
FitMode: tt.fitMode,
|
||||
},
|
||||
signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: "/images/photo.jpg",
|
||||
Size: imgcache.Size{Width: 50, Height: 50},
|
||||
Format: imgcache.FormatJPEG,
|
||||
Quality: tt.quality,
|
||||
FitMode: tt.fitMode,
|
||||
}, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
@@ -176,180 +169,3 @@ func TestHandleImage_InvalidExp_Returns400(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies sends URLs built
|
||||
// by the service's signed-URL generator for source URLs with a query to the
|
||||
// route, through a chi router: chi routes on the path as the client wrote it
|
||||
// whenever that is not Go's own encoding of it. A 502 means the signature
|
||||
// passed, so the route read the source path and query that were signed.
|
||||
func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
}{
|
||||
{"slash and colon", "url=https://origin.example.com/cat.jpg"},
|
||||
{"semicolon", "a=1;b=2"},
|
||||
{"comma", "crop=0,0,50,50"},
|
||||
{"equals and ampersand", "a=1&b=2"},
|
||||
{"percent-escape", "name=my%20cat"},
|
||||
{"question mark", "next=/a?b=1"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fix := setupTestHandler(t)
|
||||
|
||||
signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
Query: tt.query,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
}, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
status := getImage(t, fix, signedURL)
|
||||
if status != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_SourceQueryEncodedAnyWay signs one source URL with a query
|
||||
// and writes it into the image URL in ways the README allows: the "?" before
|
||||
// the query and each "%" in it percent-encoded, and every other character as
|
||||
// itself or percent-encoded. A 502 means the signature passed.
|
||||
func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expires := time.Now().Add(time.Hour)
|
||||
signer := signature.New("test-signing-key-must-be-32-chars")
|
||||
sig := signer.Sign(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
Query: "url=https://origin.example.com/my%20cat.jpg",
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
Expires: expires,
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
writtenQuery string
|
||||
}{
|
||||
{"only ? and % encoded",
|
||||
"%3Furl=https://origin.example.com/my%2520cat.jpg"},
|
||||
{"every reserved character encoded",
|
||||
"%3Furl%3Dhttps%3A%2F%2Forigin.example.com%2Fmy%2520cat.jpg"},
|
||||
{"lower-case hex digits",
|
||||
"%3furl=https://origin.example.com/my%2520cat.jpg"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
target := fmt.Sprintf("/v1/image/%s%s%s/50x50.jpeg?sig=%s&exp=%d",
|
||||
signedHost, photoPath, tt.writtenQuery, sig, expires.Unix())
|
||||
|
||||
status := getImage(t, setupTestHandler(t), target)
|
||||
if status != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImage_SourceURLHoldingPercent25Verifies signs a source path and a
|
||||
// source query that each hold %25, the percent-escape of a literal "%", and
|
||||
// sends each to the route written as the README says, with that "%" written as
|
||||
// %25, and as GenerateSignedURL writes it. A 502 means the signature passed.
|
||||
func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sourcePath string
|
||||
sourceQuery string
|
||||
// writtenPathAndQuery is the source path and query as the README says
|
||||
// to write them into the image URL
|
||||
writtenPathAndQuery string
|
||||
}{
|
||||
{"path", "/sale/50%25off.jpg", "", "/sale/50%2525off.jpg"},
|
||||
{"query", photoPath, "discount=50%25", photoPath + "%3Fdiscount=50%2525"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name+" written by the README rule", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expires := time.Now().Add(time.Hour)
|
||||
signer := signature.New("test-signing-key-must-be-32-chars")
|
||||
sig := signer.Sign(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: tt.sourcePath,
|
||||
Query: tt.sourceQuery,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
Expires: expires,
|
||||
})
|
||||
|
||||
target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d",
|
||||
signedHost, tt.writtenPathAndQuery, sig, expires.Unix())
|
||||
|
||||
status := getImage(t, setupTestHandler(t), target)
|
||||
if status != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(tt.name+" written by GenerateSignedURL", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fix := setupTestHandler(t)
|
||||
|
||||
signedURL, err := fix.service.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: tt.sourcePath,
|
||||
Query: tt.sourceQuery,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
}, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
status := getImage(t, fix, signedURL)
|
||||
if status != http.StatusBadGateway {
|
||||
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,14 +9,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/magic"
|
||||
)
|
||||
|
||||
// HandleImageEnc handles requests to /v1/e/{token}/* for encrypted
|
||||
@@ -24,25 +21,55 @@ import (
|
||||
// browsers identify the content type.
|
||||
func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
start := time.Now()
|
||||
|
||||
req, ok := s.parseImageEncRequest(w, r)
|
||||
if !ok || !s.chooseAutoFormat(w, r, req) {
|
||||
// Extract token from URL
|
||||
token := chi.URLParam(r, "token")
|
||||
if token == "" {
|
||||
s.respondError(w, "missing token", http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Decrypt and validate the payload
|
||||
payload, err := s.encGen.Parse(token)
|
||||
if err != nil {
|
||||
if errors.Is(err, encurl.ErrExpired) {
|
||||
s.log.Debug("encrypted URL expired", "error", err)
|
||||
s.respondError(w, "URL has expired", http.StatusGone)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Debug("failed to decrypt URL", "error", err)
|
||||
s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Convert payload to ImageRequest
|
||||
req := payload.ToImageRequest()
|
||||
|
||||
// Apply the same dimension and fit-mode bounds as the plain image
|
||||
// route: a sealed payload is trusted for its origin, not for staying
|
||||
// within limits, so an over-limit size or unknown fit mode is a 400
|
||||
// here rather than an out-of-memory or a 500 from the processor.
|
||||
err = imgcache.ValidateImageRequest(req)
|
||||
if err != nil {
|
||||
s.log.Debug("encrypted URL failed validation", "error", err)
|
||||
s.respondError(w, "invalid encrypted URL: "+err.Error(),
|
||||
http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Log the request
|
||||
s.log.Debug("encrypted image request",
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
"dimensions", fmt.Sprintf("%dx%d",
|
||||
req.Transform.Size.Width, req.Transform.Size.Height),
|
||||
"format", req.Transform.Format,
|
||||
"host", req.SourceHost,
|
||||
"path", req.SourcePath,
|
||||
"dimensions", fmt.Sprintf("%dx%d", req.Size.Width, req.Size.Height),
|
||||
"format", req.Format,
|
||||
)
|
||||
|
||||
// Fetch and process the image (no signature validation
|
||||
@@ -67,17 +94,6 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
w.Header().Set("Cache-Control", cacheControl(req.Expires))
|
||||
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
|
||||
|
||||
if notModified(w, r, resp.ETag) {
|
||||
return
|
||||
}
|
||||
|
||||
// A HEAD request gets the headers only
|
||||
if r.Method == http.MethodHead {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Stream the response
|
||||
written, err := io.Copy(w, resp.Content)
|
||||
if err != nil {
|
||||
@@ -89,11 +105,10 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
// Log completion
|
||||
duration := time.Since(start)
|
||||
s.log.Info("image served",
|
||||
"request_id", middleware.GetReqID(ctx),
|
||||
"cache_key", imagerequest.CacheKey(req),
|
||||
"host", req.Source.Host,
|
||||
"path", req.Source.Path,
|
||||
"format", req.Transform.Format,
|
||||
"cache_key", imgcache.CacheKey(req),
|
||||
"host", req.SourceHost,
|
||||
"path", req.SourcePath,
|
||||
"format", req.Format,
|
||||
"cache_status", resp.CacheStatus,
|
||||
"served_bytes", written,
|
||||
"duration_ms", duration.Milliseconds(),
|
||||
@@ -101,53 +116,6 @@ func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// parseImageEncRequest decrypts the token of an encrypted image URL into an
|
||||
// image request and checks it. On a token that is missing, does not decrypt,
|
||||
// has expired or asks for something not valid, it writes an error response
|
||||
// and returns false.
|
||||
func (s *Handlers) parseImageEncRequest(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) (*imagerequest.Request, bool) {
|
||||
// Extract token from URL
|
||||
token := chi.URLParam(r, "token")
|
||||
if token == "" {
|
||||
s.respondError(w, "missing token", http.StatusBadRequest)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Decrypt and validate the payload
|
||||
req, err := s.encGen.Parse(token)
|
||||
if err != nil {
|
||||
if errors.Is(err, encurl.ErrExpired) {
|
||||
s.log.Debug("encrypted URL expired", "error", err)
|
||||
s.respondError(w, "URL has expired", http.StatusGone)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
s.log.Debug("failed to decrypt URL", "error", err)
|
||||
s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Apply the same dimension and fit-mode bounds as the plain image
|
||||
// route: a sealed payload is trusted for its origin, not for staying
|
||||
// within limits, so an over-limit size or unknown fit mode is a 400
|
||||
// here rather than an out-of-memory or a 500 from the processor.
|
||||
err = imagerequest.ValidateImageRequest(req)
|
||||
if err != nil {
|
||||
s.log.Debug("encrypted URL failed validation", "error", err)
|
||||
s.respondError(w, "invalid encrypted URL: "+err.Error(),
|
||||
http.StatusBadRequest)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return req, true
|
||||
}
|
||||
|
||||
// handleImageError converts image service errors to HTTP responses.
|
||||
func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
@@ -158,25 +126,9 @@ func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
|
||||
case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
|
||||
s.respondError(w, "upstream timeout", http.StatusGatewayTimeout)
|
||||
case errors.Is(err, httpfetcher.ErrTooManyConnections),
|
||||
errors.Is(err, imageprocessor.ErrTooManyImages),
|
||||
errors.Is(err, imgcache.ErrProcessingTimedOut):
|
||||
errors.Is(err, imageprocessor.ErrTooManyImages):
|
||||
s.respondError(w, "server busy, try again later",
|
||||
http.StatusServiceUnavailable)
|
||||
case errors.Is(err, httpfetcher.ErrInvalidContentType),
|
||||
errors.Is(err, imageprocessor.ErrUnsupportedInputFormat),
|
||||
errors.Is(err, magic.ErrMagicByteMismatch),
|
||||
errors.Is(err, magic.ErrUnknownFormat),
|
||||
errors.Is(err, magic.ErrNotEnoughData):
|
||||
s.respondError(w, "unsupported source format",
|
||||
http.StatusUnsupportedMediaType)
|
||||
case errors.Is(err, imageprocessor.ErrOutputTooLarge):
|
||||
s.respondError(w, fmt.Sprintf("size too large for the source: the side "+
|
||||
"computed from its proportions is above %d",
|
||||
imageprocessor.MaxOutputDimension), http.StatusBadRequest)
|
||||
case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge):
|
||||
s.respondError(w, fmt.Sprintf("size too large for AVIF: the output "+
|
||||
"would be above %d pixels", imageprocessor.MaxAVIFOutputPixels),
|
||||
http.StatusBadRequest)
|
||||
default:
|
||||
s.log.Error("image request failed", "error", err)
|
||||
s.respondError(w, "internal error", http.StatusInternalServerError)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// newEncTestServer builds a router serving the encrypted-URL route with a
|
||||
@@ -54,14 +54,11 @@ func TestHandleImageEnc_OverLimitDimension_Returns400(t *testing.T) {
|
||||
|
||||
encGen, srv := newEncTestServer(t)
|
||||
|
||||
token, err := encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: "cdn.example.com",
|
||||
Path: "/photo.jpg",
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 100000, Height: 100000},
|
||||
},
|
||||
token, err := encGen.Generate(&encurl.Payload{
|
||||
SourceHost: "cdn.example.com",
|
||||
SourcePath: "/photo.jpg",
|
||||
Width: 100000,
|
||||
Height: 100000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
@@ -82,15 +79,12 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) {
|
||||
|
||||
encGen, srv := newEncTestServer(t)
|
||||
|
||||
token, err := encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: "cdn.example.com",
|
||||
Path: "/photo.jpg",
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 800, Height: 600},
|
||||
FitMode: imagerequest.Fit("bogus"),
|
||||
},
|
||||
token, err := encGen.Generate(&encurl.Payload{
|
||||
SourceHost: "cdn.example.com",
|
||||
SourcePath: "/photo.jpg",
|
||||
Width: 800,
|
||||
Height: 600,
|
||||
FitMode: imgcache.FitMode("bogus"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
@@ -102,70 +96,3 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImageEnc_IfNoneMatch_Returns304 verifies that an image served
|
||||
// through an encrypted URL carries an ETag, and that a request whose
|
||||
// If-None-Match is that ETag is answered 304 Not Modified with no body.
|
||||
func TestHandleImageEnc_IfNoneMatch_Returns304(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
target := encPhotoURL(t, h)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
|
||||
etag := rec.Header().Get("ETag")
|
||||
t.Logf("GET: %d, ETag %q", rec.Code, etag)
|
||||
|
||||
if rec.Code != http.StatusOK || etag == "" {
|
||||
t.Fatalf("GET: status = %d, ETag = %q, want %d and an ETag",
|
||||
rec.Code, etag, http.StatusOK)
|
||||
}
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
t.Logf("GET with If-None-Match: %d, %d body bytes", rec.Code, rec.Body.Len())
|
||||
|
||||
if rec.Code != http.StatusNotModified || rec.Body.Len() != 0 {
|
||||
t.Errorf("status = %d with %d body bytes, want %d with none",
|
||||
rec.Code, rec.Body.Len(), http.StatusNotModified)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImageEnc_HEAD_ReturnsHeadersOnly verifies that HEAD on an
|
||||
// encrypted URL is answered 200 with the headers GET sends and no body.
|
||||
func TestHandleImageEnc_HEAD_ReturnsHeadersOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Head("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodHead, encPhotoURL(t, h), nil))
|
||||
t.Logf("HEAD: %d, headers %v, %d body bytes",
|
||||
rec.Code, rec.Header(), rec.Body.Len())
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"Content-Type", "Content-Length", "Cache-Control", "ETag",
|
||||
} {
|
||||
if rec.Header().Get(name) == "" {
|
||||
t.Errorf("HEAD response has no %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
if rec.Body.Len() != 0 {
|
||||
t.Errorf("HEAD response body has %d bytes, want none", rec.Body.Len())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// requireServedPhoto requires that rec answers 200 with the JPEG at photoPath
|
||||
// on signedHost at the 50x50 that encPhotoURL and the generator tests ask for.
|
||||
func requireServedPhoto(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body %q",
|
||||
rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
contentType := rec.Header().Get("Content-Type")
|
||||
if contentType != "image/jpeg" {
|
||||
t.Errorf("Content-Type = %q, want image/jpeg", contentType)
|
||||
}
|
||||
|
||||
img, err := jpeg.DecodeConfig(rec.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("body is not a JPEG: %v", err)
|
||||
}
|
||||
|
||||
if img.Width != 50 || img.Height != 50 {
|
||||
t.Errorf("image is %dx%d, want 50x50", img.Width, img.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImageEnc_ValidToken_ServesImage verifies that a token made with
|
||||
// the signing key serves the image it asks for.
|
||||
func TestHandleImageEnc_ValidToken_ServesImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, encPhotoURL(t, h), nil))
|
||||
|
||||
requireServedPhoto(t, rec)
|
||||
}
|
||||
|
||||
// TestHandleImageEnc_RejectedToken verifies that a token that has expired
|
||||
// answers 410, and that a token with one character changed, a token cut
|
||||
// short, and a token made with another signing key answer 400. The server
|
||||
// would serve the photo for a token it accepted.
|
||||
func TestHandleImageEnc_RejectedToken(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
photo := imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
},
|
||||
}
|
||||
|
||||
valid, err := h.encGen.Generate(&photo)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
expiredPhoto := photo
|
||||
expiredPhoto.Expires = time.Now().Add(-time.Minute)
|
||||
|
||||
expired, err := h.encGen.Generate(&expiredPhoto)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
otherGen, err := encurl.NewGenerator("another-signing-key-fedcba9876543210")
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
otherKey, err := otherGen.Generate(&photo)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
// Changing a character in the middle always changes the decoded bytes;
|
||||
// the last character of unpadded base64 can carry unused bits.
|
||||
middle := len(valid) / 2
|
||||
|
||||
replacement := "A"
|
||||
if valid[middle] == 'A' {
|
||||
replacement = "B"
|
||||
}
|
||||
|
||||
changed := valid[:middle] + replacement + valid[middle+1:]
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
token string
|
||||
wantStatus int
|
||||
}{
|
||||
{"expired", expired, http.StatusGone},
|
||||
{"one character changed", changed, http.StatusBadRequest},
|
||||
{"cut short", valid[:middle], http.StatusBadRequest},
|
||||
{"another signing key", otherKey, http.StatusBadRequest},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rec := getEncToken(srv, tt.token)
|
||||
t.Logf("GET /v1/e/%s/img.jpg: %d %s", tt.token, rec.Code, rec.Body)
|
||||
|
||||
if rec.Code != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/signature"
|
||||
)
|
||||
|
||||
// requireJPEGXL requires that rec answers 200 with a JPEG XL image.
|
||||
func requireJPEGXL(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body %q",
|
||||
rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
if got := rec.Header().Get("Content-Type"); got != jxlType {
|
||||
t.Errorf("Content-Type = %q, want %s", got, jxlType)
|
||||
}
|
||||
|
||||
if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
|
||||
t.Errorf("body is %s, want jxl", vips.ImageTypes[got])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageWithoutFormat_ServesJPEGXL verifies that a /v1/image/ URL whose
|
||||
// last segment is a size with no format, 50x50 or orig, answers JPEG XL.
|
||||
func TestImageWithoutFormat_ServesJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := newImageRoute(t, newPhotoFetcher(t, allowlistedHost))
|
||||
|
||||
for _, size := range []string{"50x50", "orig"} {
|
||||
target := "/v1/image/" + allowlistedHost + photoPath + "/" + size
|
||||
requireJPEGXL(t, sendGet(t, route, target))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageWithoutFormat_SignedAsJXL verifies that a /v1/image/ URL with no
|
||||
// format is signed as jxl: the signature made for the URL ending in .jxl is
|
||||
// accepted for the same URL without .jxl.
|
||||
func TestImageWithoutFormat_SignedAsJXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := newImageRoute(t, newPhotoFetcher(t, signedHost))
|
||||
expires := time.Now().Add(time.Hour)
|
||||
|
||||
sig := signature.New(testSigningKey).Sign(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJXL,
|
||||
Quality: encurl.DefaultQuality,
|
||||
FitMode: imagerequest.FitCover,
|
||||
},
|
||||
Expires: expires,
|
||||
})
|
||||
query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix())
|
||||
|
||||
for _, size := range []string{"50x50.jxl", "50x50"} {
|
||||
target := "/v1/image/" + signedHost + photoPath + "/" + size + query
|
||||
requireJPEGXL(t, sendGet(t, route, target))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageEncWithoutFormat_ServesJPEGXL verifies that an encrypted URL whose
|
||||
// token holds no format answers JPEG XL.
|
||||
func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
token, err := h.encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
requireJPEGXL(t, getEncToken(srv, token))
|
||||
}
|
||||
|
||||
// TestGeneratorPage_SelectsJPEGXL verifies that the generator page's format
|
||||
// choice is JPEG XL until another is chosen.
|
||||
func TestGeneratorPage_SelectsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newCSRFTestRouter(t)
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(newSessionCookie(t, h))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if !strings.Contains(rec.Body.String(), `<option value="jxl" selected>`) {
|
||||
t.Errorf("generator page does not select JPEG XL: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_NoFormat_MakesJPEGXLURL verifies that the generator form
|
||||
// sent with an empty format field, or with none, makes a URL whose name ends
|
||||
// in .jxl and which answers JPEG XL.
|
||||
func TestGeneratePost_NoFormat_MakesJPEGXLURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photo := url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
}
|
||||
|
||||
emptyFormat := maps.Clone(photo)
|
||||
emptyFormat.Set(formatField, "")
|
||||
|
||||
for name, form := range map[string]url.Values{
|
||||
"empty format field": emptyFormat,
|
||||
"no format field": photo,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, form)
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %d %s",
|
||||
rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
t.Logf("generated URL path: %s", match[1])
|
||||
|
||||
if !strings.HasSuffix(match[1], "/img.jxl") {
|
||||
t.Errorf("generated URL %s does not end in /img.jxl", match[1])
|
||||
}
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireJPEGXL(t, imageRec)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// jxlType is the content type of JPEG XL.
|
||||
const jxlType = "image/jxl"
|
||||
|
||||
// TestFormatForAccept_JPEGXL verifies that the format auto chooses JPEG XL
|
||||
// when Accept names image/jxl, ahead of AVIF whatever their q, and AVIF when
|
||||
// Accept refuses JPEG XL with q=0.
|
||||
func TestFormatForAccept_JPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
accept string
|
||||
want imagerequest.Format
|
||||
}{
|
||||
{"JPEG XL-capable browser",
|
||||
"image/jxl,image/avif,image/webp,image/*,*/*;q=0.8", imagerequest.FormatJXL},
|
||||
{"JPEG XL only", jxlType, imagerequest.FormatJXL},
|
||||
{"JPEG XL with a lower q than AVIF", "image/avif,image/jxl;q=0.5",
|
||||
imagerequest.FormatJXL},
|
||||
{"q=0 on JPEG XL", "image/jxl;q=0,image/avif,*/*", imagerequest.FormatAVIF},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := formatForAccept(tt.accept)
|
||||
if got != tt.want || err != nil {
|
||||
t.Errorf("formatForAccept(%q) = %q, %v, want %q",
|
||||
tt.accept, got, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatAuto_JPEGXL requests an auto URL on each image route with an
|
||||
// Accept header that names image/jxl, and checks that the answer is a JPEG XL
|
||||
// image.
|
||||
func TestFormatAuto_JPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
signedURL, encryptedURL := autoPhotoURLs(t, h)
|
||||
|
||||
for _, target := range []string{signedURL, encryptedURL} {
|
||||
rec := requestImage(t, srv, http.MethodGet, target,
|
||||
"image/jxl,image/avif,image/webp,*/*;q=0.8")
|
||||
gotType := rec.Header().Get("Content-Type")
|
||||
|
||||
if rec.Code != http.StatusOK || gotType != jxlType {
|
||||
t.Errorf("%s: %d %s, want 200 %s; body %s",
|
||||
target, rec.Code, gotType, jxlType, rec.Body)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
|
||||
t.Errorf("%s: body is %s, want jxl", target, vips.ImageTypes[got])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/allowlist"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// blockedReferer is a page on leech.example, which newRefererRoutes puts on
|
||||
// referer_blocklist.
|
||||
const blockedReferer = "https://leech.example/page.html"
|
||||
|
||||
// countingFetcher passes each fetch on to the fetcher it holds and counts it.
|
||||
type countingFetcher struct {
|
||||
httpfetcher.Fetcher
|
||||
|
||||
fetches atomic.Int32
|
||||
}
|
||||
|
||||
// Fetch counts the fetch and passes it on.
|
||||
func (f *countingFetcher) Fetch(
|
||||
ctx context.Context, url string,
|
||||
) (*httpfetcher.FetchResult, error) {
|
||||
f.fetches.Add(1)
|
||||
|
||||
return f.Fetcher.Fetch(ctx, url)
|
||||
}
|
||||
|
||||
// newRefererRoutes returns both image routes of a Handlers whose
|
||||
// referer_blocklist is "leech.example" and ".hotlinker.example", the
|
||||
// Handlers, and the fetcher the routes fetch through. The JPEG at photoPath
|
||||
// exists on allowlistedHost and on signedHost.
|
||||
func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) {
|
||||
t.Helper()
|
||||
|
||||
fetcher := &countingFetcher{
|
||||
Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost),
|
||||
}
|
||||
|
||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||
StateDir: t.TempDir(),
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewCache() error = %v", err)
|
||||
}
|
||||
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
Fetcher: fetcher,
|
||||
SigningKey: testSigningKey,
|
||||
Allowlist: []string{allowlistedHost},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||
}
|
||||
|
||||
encGen, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
imgSvc: svc,
|
||||
encGen: encGen,
|
||||
refererBlocklist: allowlist.New(
|
||||
[]string{"leech.example", ".hotlinker.example"}),
|
||||
}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
return r, h, fetcher
|
||||
}
|
||||
|
||||
// getWithReferer sends a GET for target to routes with referer as its
|
||||
// Referer header, or with none when referer is empty, and returns the
|
||||
// response.
|
||||
func getWithReferer(
|
||||
t *testing.T, routes http.Handler, target, referer string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||
if referer != "" {
|
||||
req.Header.Set("Referer", referer)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
routes.ServeHTTP(rec, req)
|
||||
t.Logf("GET %s with Referer %q: %d", target, referer, rec.Code)
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// TestRefererBlocklist verifies that both image routes refuse a request whose
|
||||
// Referer names a host on referer_blocklist with 403 and the JSON error,
|
||||
// without fetching from the upstream host, and serve a request with no
|
||||
// Referer, one that does not parse, or one naming any other host. Hosts are
|
||||
// matched as allowlist_hosts matches them.
|
||||
func TestRefererBlocklist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
referer string
|
||||
want int
|
||||
}{
|
||||
{"no referer", "", http.StatusOK},
|
||||
{"unlisted host", "https://unlisted.example/page.html", http.StatusOK},
|
||||
{"unparseable", "%zz", http.StatusOK},
|
||||
{"listed host", blockedReferer, http.StatusForbidden},
|
||||
{"subdomain of listed host", "https://www.leech.example/", http.StatusOK},
|
||||
{"subdomain of dot pattern", "https://www.hotlinker.example/a.html",
|
||||
http.StatusForbidden},
|
||||
{"dot pattern without its dot", "https://hotlinker.example/",
|
||||
http.StatusForbidden},
|
||||
{"host continuing past dot pattern",
|
||||
"https://hotlinker.example.evil.example/", http.StatusOK},
|
||||
}
|
||||
|
||||
// The photo's URL on each image route.
|
||||
photoURLs := map[string]func(t *testing.T, h *Handlers) string{
|
||||
"plain URL": func(t *testing.T, _ *Handlers) string {
|
||||
t.Helper()
|
||||
|
||||
return photoURL(allowlistedHost)
|
||||
},
|
||||
"encrypted URL": encPhotoURL,
|
||||
}
|
||||
|
||||
for urlName, photoURLFor := range photoURLs {
|
||||
for _, tc := range cases {
|
||||
t.Run(urlName+", "+tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
routes, h, fetcher := newRefererRoutes(t)
|
||||
|
||||
rec := getWithReferer(t, routes, photoURLFor(t, h), tc.referer)
|
||||
|
||||
if tc.want == http.StatusOK {
|
||||
requireServedPhoto(t, rec)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
|
||||
|
||||
if n := fetcher.fetches.Load(); n != 0 {
|
||||
t.Errorf("upstream fetched %d times, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockedRefererRefusedWhenImageIsCached verifies that a request whose
|
||||
// Referer is on referer_blocklist is refused even when the image it asks for
|
||||
// is already cached, so the answer does not depend on the cache.
|
||||
func TestBlockedRefererRefusedWhenImageIsCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
routes, h, _ := newRefererRoutes(t)
|
||||
|
||||
for _, target := range []string{photoURL(allowlistedHost), encPhotoURL(t, h)} {
|
||||
requireServedPhoto(t, getWithReferer(t, routes, target, ""))
|
||||
|
||||
rec := getWithReferer(t, routes, target, blockedReferer)
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
|
||||
}
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// signedPhotoURL returns a signed /v1/image/ URL, valid for a minute, for the
|
||||
// JPEG at photoPath on signedHost at 50x50, made with h's image service.
|
||||
func signedPhotoURL(t *testing.T, h *Handlers) string {
|
||||
t.Helper()
|
||||
|
||||
signedURL, err := h.imgSvc.GenerateSignedURL("", &imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateSignedURL() error = %v", err)
|
||||
}
|
||||
|
||||
return signedURL
|
||||
}
|
||||
|
||||
// encPhotoURL returns an encrypted /v1/e/ URL, which never expires, for the
|
||||
// JPEG at photoPath on signedHost at 50x50, made with h's generator.
|
||||
func encPhotoURL(t *testing.T, h *Handlers) string {
|
||||
t.Helper()
|
||||
|
||||
token, err := h.encGen.Generate(&imagerequest.Request{
|
||||
Source: imagerequest.Source{
|
||||
Host: signedHost,
|
||||
Path: photoPath,
|
||||
},
|
||||
Transform: imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
return "/v1/e/" + token + "/img.jpg"
|
||||
}
|
||||
|
||||
// requestIDByMessage reads the JSON log lines in logs and returns the
|
||||
// request_id of each line, by its message.
|
||||
func requestIDByMessage(t *testing.T, logs io.Reader) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
logged := make(map[string]string)
|
||||
|
||||
dec := json.NewDecoder(logs)
|
||||
for dec.More() {
|
||||
var line map[string]any
|
||||
|
||||
err := dec.Decode(&line)
|
||||
if err != nil {
|
||||
t.Fatalf("decoding log line: %v", err)
|
||||
}
|
||||
|
||||
msg, _ := line["msg"].(string)
|
||||
requestID, _ := line["request_id"].(string)
|
||||
logged[msg] = requestID
|
||||
}
|
||||
|
||||
return logged
|
||||
}
|
||||
|
||||
// TestImageLogLinesCarryRequestID verifies that the lines logged when an image
|
||||
// is fetched, converted and served through either image route carry the
|
||||
// request's ID as request_id, as the request log line does, so they can be
|
||||
// found from it.
|
||||
func TestImageLogLinesCarryRequestID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const requestID = "test-request-id"
|
||||
|
||||
imageURLs := map[string]func(*testing.T, *Handlers) string{
|
||||
"/v1/image/": signedPhotoURL,
|
||||
"/v1/e/": encPhotoURL,
|
||||
}
|
||||
|
||||
for route, imageURL := range imageURLs {
|
||||
t.Run(route, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var logs bytes.Buffer
|
||||
|
||||
h, srv := newSignedHostServer(t,
|
||||
slog.New(slog.NewJSONHandler(&logs, nil)))
|
||||
|
||||
ctx := context.WithValue(t.Context(),
|
||||
middleware.RequestIDKey, requestID)
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
ctx, http.MethodGet, imageURL(t, h), nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
t.Logf("logged:\n%s", logs.String())
|
||||
|
||||
logged := requestIDByMessage(t, &logs)
|
||||
|
||||
for _, msg := range []string{
|
||||
"upstream fetched", "image converted", "image served",
|
||||
} {
|
||||
got, ok := logged[msg]
|
||||
if !ok {
|
||||
t.Errorf("no %q line logged", msg)
|
||||
} else if got != requestID {
|
||||
t.Errorf("%q line has request_id %q, want %q",
|
||||
msg, got, requestID)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/fx/fxtest"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
)
|
||||
|
||||
// TestHandleRobotsTxt checks that /robots.txt asks every crawler to stay off
|
||||
// the whole site.
|
||||
func TestHandleRobotsTxt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||
rec := sendGet(t, h.HandleRobotsTxt(), "/robots.txt")
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "text/plain" {
|
||||
t.Errorf("Content-Type = %q, want text/plain", ct)
|
||||
}
|
||||
|
||||
want := "User-agent: *\nDisallow: /\n"
|
||||
if rec.Body.String() != want {
|
||||
t.Errorf("body = %q, want %q", rec.Body.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleHealthCheck checks that the health check answers 200 with status
|
||||
// ok, the app's name and version, now, uptime_seconds, uptime_human and
|
||||
// maintenance_mode, which is true here: the health check stays 200 while
|
||||
// maintenance mode is on.
|
||||
func TestHandleHealthCheck(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lc := fxtest.NewLifecycle(t)
|
||||
|
||||
log, err := logger.New(lc, logger.Params{Globals: &globals.Globals{}})
|
||||
if err != nil {
|
||||
t.Fatalf("logger.New() error = %v", err)
|
||||
}
|
||||
|
||||
hc, err := healthcheck.New(lc, healthcheck.Params{
|
||||
Globals: &globals.Globals{Appname: "pixad", Version: "v1.2.3"},
|
||||
Config: &config.Config{MaintenanceMode: true},
|
||||
Logger: log,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("healthcheck.New() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{hc: hc, log: slog.New(slog.DiscardHandler)}
|
||||
rec := sendGet(t, h.HandleHealthCheck(), "/.well-known/healthcheck.json")
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
|
||||
t.Errorf("Content-Type = %q, want application/json", ct)
|
||||
}
|
||||
|
||||
var body map[string]any
|
||||
|
||||
err = json.NewDecoder(rec.Body).Decode(&body)
|
||||
if err != nil {
|
||||
t.Fatalf("decoding response body: %v", err)
|
||||
}
|
||||
|
||||
if body["status"] != "ok" || body["appname"] != "pixad" ||
|
||||
body["version"] != "v1.2.3" || body["maintenance_mode"] != true {
|
||||
t.Errorf("body = %v, want status ok, appname pixad, version v1.2.3 "+
|
||||
"and maintenance_mode true", body)
|
||||
}
|
||||
|
||||
for _, key := range []string{"now", "uptime_seconds", "uptime_human"} {
|
||||
if _, ok := body[key]; !ok {
|
||||
t.Errorf("body = %v, has no %s", body, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -11,7 +10,6 @@ import (
|
||||
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
@@ -22,9 +20,7 @@ func TestServerBusyAnswers503(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{Host: "img.example.com", Path: "/a.jpg"},
|
||||
}
|
||||
req := &imgcache.ImageRequest{SourceHost: "img.example.com", SourcePath: "/a.jpg"}
|
||||
|
||||
for _, err := range []error{
|
||||
fmt.Errorf("upstream fetch failed: %w", httpfetcher.ErrTooManyConnections),
|
||||
@@ -51,40 +47,3 @@ func TestServerBusyAnswers503(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessingTimedOutAnswers503 checks that respondImageError and
|
||||
// handleImageError, which answer the errors of the two image routes, answer
|
||||
// 503 with the server busy error when the image service reports that
|
||||
// processing stopped at the deadline of the request doing it, wrapped as the
|
||||
// service wraps it. A request that waited for that processing gets this error
|
||||
// while its own deadline may have time left. The request's source only goes
|
||||
// into a log line, so it is left empty.
|
||||
func TestProcessingTimedOutAnswers503(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||
err := fmt.Errorf("%w: failed to encode: %w",
|
||||
imgcache.ErrProcessingTimedOut, context.DeadlineExceeded)
|
||||
|
||||
plain := httptest.NewRecorder()
|
||||
h.respondImageError(plain, &imagerequest.Request{}, err)
|
||||
|
||||
encrypted := httptest.NewRecorder()
|
||||
h.handleImageError(encrypted, err)
|
||||
|
||||
for name, rec := range map[string]*httptest.ResponseRecorder{
|
||||
"respondImageError": plain, "handleImageError": encrypted,
|
||||
} {
|
||||
t.Logf("%s: status = %d, body = %q", name, rec.Code, rec.Body.String())
|
||||
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("%s: status = %d, want %d",
|
||||
name, rec.Code, http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
if !strings.Contains(rec.Body.String(), "server busy, try again later") {
|
||||
t.Errorf("%s: body = %q, want the server busy error",
|
||||
name, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imageprocessor"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestUnsupportedSourceFormatAnswers415 checks that both image routes answer
|
||||
// 415 with a clear error when the upstream answers with a content type pixa
|
||||
// does not accept, such as SVG, and when orig is asked of a source in a
|
||||
// format the processor cannot name, wrapped as the service wraps them.
|
||||
func TestUnsupportedSourceFormatAnswers415(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h := &Handlers{log: slog.New(slog.DiscardHandler)}
|
||||
req := &imagerequest.Request{
|
||||
Source: imagerequest.Source{Host: "img.example.com", Path: "/a.svg"},
|
||||
}
|
||||
|
||||
for _, err := range []error{
|
||||
fmt.Errorf("upstream fetch failed: %w: image/svg+xml",
|
||||
httpfetcher.ErrInvalidContentType),
|
||||
fmt.Errorf("image processing failed: %w: unknown",
|
||||
imageprocessor.ErrUnsupportedInputFormat),
|
||||
} {
|
||||
plain := httptest.NewRecorder()
|
||||
h.respondImageError(plain, req, err)
|
||||
|
||||
encrypted := httptest.NewRecorder()
|
||||
h.handleImageError(encrypted, err)
|
||||
|
||||
for route, rec := range map[string]*httptest.ResponseRecorder{
|
||||
"HandleImage": plain, "HandleImageEnc": encrypted,
|
||||
} {
|
||||
t.Logf("%s for %v: %d %s", route, err, rec.Code, rec.Body.String())
|
||||
|
||||
if rec.Code != http.StatusUnsupportedMediaType {
|
||||
t.Errorf("%s for %v: status = %d, want %d",
|
||||
route, err, rec.Code, http.StatusUnsupportedMediaType)
|
||||
}
|
||||
|
||||
if !strings.Contains(rec.Body.String(), "unsupported source format") {
|
||||
t.Errorf("%s for %v: body = %q, want the unsupported source format error",
|
||||
route, err, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package httpfetcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built
|
||||
// without DialContext, as pixa builds it, refuses to connect to a local
|
||||
// server.
|
||||
func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
transport := transportOf(t, New(DefaultConfig()))
|
||||
|
||||
addr := srv.Listener.Addr().String()
|
||||
|
||||
_, err := transport.DialContext(testContext(t), "tcp", addr)
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Fatalf("DialContext(%s) error = %v, want ErrSSRFBlocked", addr, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDialContextReplacesOnlyTheDialer checks that a fetcher built with
|
||||
// DialContext connects through it, while the URL check still refuses a
|
||||
// loopback URL and the redirect check a redirect to a link-local address.
|
||||
func TestDialContextReplacesOnlyTheDialer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
dialer := &recordingDialer{target: srv.Listener.Addr().String()}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.AllowHTTP = true
|
||||
cfg.DialContext = dialer.dialContext
|
||||
f := New(cfg)
|
||||
|
||||
if body := fetchBody(t, f, "/image"); body != imagePayload {
|
||||
t.Errorf("body = %q, want %q", body, imagePayload)
|
||||
}
|
||||
|
||||
_, err := f.Fetch(testContext(t), "http://127.0.0.1/image")
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err)
|
||||
}
|
||||
|
||||
_, err = f.Fetch(testContext(t), upstreamURL("/redirect/private"))
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err)
|
||||
}
|
||||
|
||||
// The upstream server is reached through DialContext, and nothing else
|
||||
// is asked of it.
|
||||
dialed := dialer.dialedAddrs()
|
||||
if len(dialed) == 0 {
|
||||
t.Error("DialContext was never called")
|
||||
}
|
||||
|
||||
for _, addr := range dialed {
|
||||
if addr != net.JoinHostPort(testPublicHost, "80") {
|
||||
t.Errorf("DialContext was asked to connect to %s", addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -192,23 +191,7 @@ func fetchBody(t *testing.T, f *HTTPFetcher, path string) string {
|
||||
|
||||
// semLen reports how many per-host semaphore slots are currently held.
|
||||
func semLen(f *HTTPFetcher, host string) int {
|
||||
f.hostSemMu.Lock()
|
||||
defer f.hostSemMu.Unlock()
|
||||
|
||||
sem, ok := f.hostSems[host]
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
|
||||
return len(sem.slots)
|
||||
}
|
||||
|
||||
// hostSemCount reports how many hosts have a semaphore in hostSems.
|
||||
func hostSemCount(f *HTTPFetcher) int {
|
||||
f.hostSemMu.Lock()
|
||||
defer f.hostSemMu.Unlock()
|
||||
|
||||
return len(f.hostSems)
|
||||
return len(f.getHostSemaphore(host))
|
||||
}
|
||||
|
||||
func TestFetchRedirectToPrivateIPBlocked(t *testing.T) {
|
||||
@@ -298,139 +281,6 @@ func TestFetchMaxResponseSizeEnforced(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fetchWithMaxResponseSize serves handler as the fake upstream and fetches
|
||||
// its /image route with MaxResponseSize set to maxResponseSize.
|
||||
func fetchWithMaxResponseSize(
|
||||
t *testing.T,
|
||||
maxResponseSize int64,
|
||||
handler http.HandlerFunc,
|
||||
) *FetchResult {
|
||||
t.Helper()
|
||||
|
||||
srv := httptest.NewServer(handler)
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxResponseSize = maxResponseSize
|
||||
|
||||
f, _ := newServerFetcher(t, srv, cfg)
|
||||
|
||||
res := fetchImage(t, f, "/image")
|
||||
t.Cleanup(func() { _ = res.Content.Close() })
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// TestFetchContentLengthBodyAtMaxResponseSize checks that a body sent with a
|
||||
// Content-Length is read whole at exactly MaxResponseSize bytes and refused
|
||||
// at one byte more.
|
||||
func TestFetchContentLengthBodyAtMaxResponseSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const maxResponseSize = 8
|
||||
|
||||
for _, tc := range []struct {
|
||||
size int
|
||||
wantErr error
|
||||
}{
|
||||
{maxResponseSize, nil},
|
||||
{maxResponseSize + 1, ErrResponseTooLarge},
|
||||
} {
|
||||
t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Repeat("x", tc.size)
|
||||
|
||||
res := fetchWithMaxResponseSize(t, maxResponseSize,
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", contentTypeJPEG)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(tc.size))
|
||||
_, _ = io.WriteString(w, body)
|
||||
})
|
||||
|
||||
if res.ContentLength != int64(tc.size) {
|
||||
t.Fatalf("ContentLength = %d, want %d", res.ContentLength, tc.size)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(res.Content)
|
||||
t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err)
|
||||
|
||||
if !errors.Is(err, tc.wantErr) {
|
||||
t.Fatalf("read error = %v, want %v", err, tc.wantErr)
|
||||
}
|
||||
|
||||
if err == nil && string(data) != body {
|
||||
t.Errorf("read %q, want %q", data, body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchChunkedBodyAtMaxResponseSize checks that a chunked body is read
|
||||
// whole at exactly MaxResponseSize bytes and refused at one byte more. The
|
||||
// upstream sends the end of the body only once the test has read
|
||||
// MaxResponseSize bytes, so the read after them comes before the end of the
|
||||
// body has arrived, as it can from a real upstream.
|
||||
func TestFetchChunkedBodyAtMaxResponseSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const maxResponseSize = 8
|
||||
|
||||
for _, tc := range []struct {
|
||||
size int
|
||||
wantErr error
|
||||
}{
|
||||
{maxResponseSize, nil},
|
||||
{maxResponseSize + 1, ErrResponseTooLarge},
|
||||
} {
|
||||
t.Run(strconv.Itoa(tc.size)+" bytes", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Repeat("x", tc.size)
|
||||
endBody := make(chan struct{})
|
||||
|
||||
res := fetchWithMaxResponseSize(t, maxResponseSize,
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", contentTypeJPEG)
|
||||
_, _ = io.WriteString(w, body)
|
||||
// With no Content-Length, flushing sends the body
|
||||
// chunked; its end is sent when the handler returns.
|
||||
_ = http.NewResponseController(w).Flush()
|
||||
|
||||
select {
|
||||
case <-endBody:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
})
|
||||
|
||||
if res.ContentLength != -1 {
|
||||
t.Fatalf("ContentLength = %d, want -1 (chunked)", res.ContentLength)
|
||||
}
|
||||
|
||||
head := make([]byte, maxResponseSize)
|
||||
|
||||
_, err := io.ReadFull(res.Content, head)
|
||||
if err != nil {
|
||||
t.Fatalf("read the first %d bytes: %v", maxResponseSize, err)
|
||||
}
|
||||
|
||||
close(endBody)
|
||||
|
||||
rest, err := io.ReadAll(res.Content)
|
||||
data := string(head) + string(rest)
|
||||
t.Logf("body of %d bytes: read %d, error %v", tc.size, len(data), err)
|
||||
|
||||
if !errors.Is(err, tc.wantErr) {
|
||||
t.Fatalf("read error = %v, want %v", err, tc.wantErr)
|
||||
}
|
||||
|
||||
if err == nil && data != body {
|
||||
t.Errorf("read %q, want %q", data, body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchSemaphoreReleasedOnError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -18,8 +18,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
// Fetcher configuration constants.
|
||||
@@ -45,7 +43,6 @@ const (
|
||||
contentTypeGIF = "image/gif"
|
||||
contentTypeWebP = "image/webp"
|
||||
contentTypeAVIF = "image/avif"
|
||||
contentTypeJXL = "image/jxl"
|
||||
contentTypeSVG = "image/svg+xml"
|
||||
contentTypeOctetStream = "application/octet-stream"
|
||||
)
|
||||
@@ -59,17 +56,15 @@ const (
|
||||
// builtinBlockedPrefixes are internal or special-use ranges that Go's
|
||||
// net.IP predicates (IsPrivate, IsLinkLocalUnicast, and the like) do not
|
||||
// already cover. They are always blocked, in addition to any
|
||||
// operator-supplied networks. IPv4-mapped and IPv4-compatible IPv6
|
||||
// addresses are read as the IPv4 address they hold before matching, so
|
||||
// these IPv4 ranges are caught in all three forms.
|
||||
// operator-supplied networks. IPv4-mapped IPv6 addresses are unmapped
|
||||
// before matching, so these IPv4 ranges are caught in both forms.
|
||||
//
|
||||
//nolint:gochecknoglobals // immutable built-in blocklist
|
||||
var builtinBlockedPrefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT / carrier-grade NAT
|
||||
netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments
|
||||
netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range
|
||||
netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4)
|
||||
netip.MustParsePrefix("64:ff9b:1::/48"), // RFC 8215 local-use NAT64 (maps onto IPv4)
|
||||
netip.MustParsePrefix("100.64.0.0/10"), // RFC 6598 CGNAT / carrier-grade NAT
|
||||
netip.MustParsePrefix("192.0.0.0/24"), // RFC 6890 IETF protocol assignments
|
||||
netip.MustParsePrefix("198.18.0.0/15"), // RFC 2544 benchmarking range
|
||||
netip.MustParsePrefix("64:ff9b::/96"), // RFC 6052 NAT64 (maps onto IPv4)
|
||||
}
|
||||
|
||||
// Fetcher errors.
|
||||
@@ -140,11 +135,6 @@ type Config struct {
|
||||
// BlockedNetworks are operator-supplied CIDR ranges refused by the
|
||||
// dialer, in addition to the always-enforced built-in ranges.
|
||||
BlockedNetworks []netip.Prefix
|
||||
// DialContext, when set, makes the fetcher's connections in place of
|
||||
// the dialer that refuses internal addresses; the URL and redirect
|
||||
// checks still run. Only tests set it, to reach a local server; the
|
||||
// config file and the environment cannot.
|
||||
DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// DefaultConfig returns a Config with sensible defaults.
|
||||
@@ -159,7 +149,7 @@ func DefaultConfig() *Config {
|
||||
contentTypeGIF,
|
||||
contentTypeWebP,
|
||||
contentTypeAVIF,
|
||||
contentTypeJXL,
|
||||
contentTypeSVG,
|
||||
},
|
||||
AllowHTTP: false,
|
||||
MaxConnectionsPerHost: DefaultMaxConnectionsPerHost,
|
||||
@@ -170,13 +160,10 @@ func DefaultConfig() *Config {
|
||||
// HTTPFetcher implements Fetcher with SSRF protection and connection limits
|
||||
// per host and for all hosts together.
|
||||
type HTTPFetcher struct {
|
||||
client *http.Client
|
||||
config *Config
|
||||
// hostSems holds the semaphore of each host with a fetch holding or
|
||||
// waiting for one of its slots; the entry is removed when the host's
|
||||
// last such fetch gives its slot back or stops waiting.
|
||||
hostSems map[string]*hostSemaphore
|
||||
hostSemMu sync.Mutex // protects hostSems and each entry's count
|
||||
client *http.Client
|
||||
config *Config
|
||||
hostSems map[string]chan struct{} // per-host semaphores
|
||||
hostSemMu sync.Mutex // protects hostSems map
|
||||
// allHostsSemaphore has one slot per connection allowed to all hosts
|
||||
// together (config.MaxConnections).
|
||||
allHostsSemaphore chan struct{}
|
||||
@@ -184,33 +171,19 @@ type HTTPFetcher struct {
|
||||
connectionWaitTimeout time.Duration
|
||||
}
|
||||
|
||||
// hostSemaphore is one host's connection slots
|
||||
// (config.MaxConnectionsPerHost) and the number of fetches holding or
|
||||
// waiting for one of them.
|
||||
type hostSemaphore struct {
|
||||
slots chan struct{}
|
||||
count int
|
||||
}
|
||||
|
||||
// New creates a new HTTPFetcher with SSRF protection.
|
||||
func New(config *Config) *HTTPFetcher {
|
||||
if config == nil {
|
||||
config = DefaultConfig()
|
||||
}
|
||||
|
||||
// Unless config.DialContext replaces it, the transport connects with
|
||||
// the SSRF-safe dialer, which re-resolves and re-checks at connect time
|
||||
// (closing the DNS-rebinding window) against both the built-in ranges
|
||||
// and the operator-supplied blocklist.
|
||||
dialContext := config.DialContext
|
||||
if dialContext == nil {
|
||||
dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialSSRFSafe(ctx, network, addr, config.BlockedNetworks)
|
||||
}
|
||||
}
|
||||
|
||||
// Create transport with SSRF-safe dialer. The dialer re-resolves and
|
||||
// re-checks at connect time (closing the DNS-rebinding window) against
|
||||
// both the built-in ranges and the operator-supplied blocklist.
|
||||
transport := &http.Transport{
|
||||
DialContext: dialContext,
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialSSRFSafe(ctx, network, addr, config.BlockedNetworks)
|
||||
},
|
||||
TLSHandshakeTimeout: DefaultTLSTimeout,
|
||||
MaxIdleConns: DefaultMaxIdleConns,
|
||||
IdleConnTimeout: DefaultIdleConnTimeout,
|
||||
@@ -238,7 +211,7 @@ func New(config *Config) *HTTPFetcher {
|
||||
return &HTTPFetcher{
|
||||
client: client,
|
||||
config: config,
|
||||
hostSems: make(map[string]*hostSemaphore),
|
||||
hostSems: make(map[string]chan struct{}),
|
||||
allHostsSemaphore: make(chan struct{}, config.MaxConnections),
|
||||
connectionWaitTimeout: ConnectionWaitTimeout,
|
||||
}
|
||||
@@ -283,13 +256,6 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*FetchResult, erro
|
||||
req.Header.Set("User-Agent", f.config.UserAgent)
|
||||
req.Header.Set("Accept", strings.Join(f.config.AllowedContentTypes, ", "))
|
||||
|
||||
// The ID of the request this fetch serves, so the fetch can be found in
|
||||
// the upstream host's logs
|
||||
requestID := middleware.GetReqID(ctx)
|
||||
if requestID != "" {
|
||||
req.Header.Set(middleware.RequestIDHeader, requestID)
|
||||
}
|
||||
|
||||
// Use httptrace to capture connection details
|
||||
var remoteAddr string
|
||||
|
||||
@@ -341,8 +307,6 @@ func (f *HTTPFetcher) acquireConnection(
|
||||
select {
|
||||
case hostSem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
f.putHostSemaphore(host)
|
||||
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
@@ -350,55 +314,32 @@ func (f *HTTPFetcher) acquireConnection(
|
||||
case f.allHostsSemaphore <- struct{}{}:
|
||||
case <-time.After(f.connectionWaitTimeout):
|
||||
<-hostSem
|
||||
f.putHostSemaphore(host)
|
||||
|
||||
return nil, ErrTooManyConnections
|
||||
case <-ctx.Done():
|
||||
<-hostSem
|
||||
f.putHostSemaphore(host)
|
||||
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
return func() {
|
||||
<-hostSem
|
||||
f.putHostSemaphore(host)
|
||||
<-f.allHostsSemaphore
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getHostSemaphore returns the semaphore for a host, creating it if
|
||||
// necessary, and counts the caller among the fetches using it. The caller
|
||||
// calls putHostSemaphore once it holds no slot and waits for none.
|
||||
// getHostSemaphore returns the semaphore for a host, creating it if necessary.
|
||||
func (f *HTTPFetcher) getHostSemaphore(host string) chan struct{} {
|
||||
f.hostSemMu.Lock()
|
||||
defer f.hostSemMu.Unlock()
|
||||
|
||||
sem, ok := f.hostSems[host]
|
||||
if !ok {
|
||||
sem = &hostSemaphore{
|
||||
slots: make(chan struct{}, f.config.MaxConnectionsPerHost),
|
||||
}
|
||||
sem = make(chan struct{}, f.config.MaxConnectionsPerHost)
|
||||
f.hostSems[host] = sem
|
||||
}
|
||||
|
||||
sem.count++
|
||||
|
||||
return sem.slots
|
||||
}
|
||||
|
||||
// putHostSemaphore stops counting the caller among the fetches using the
|
||||
// host's semaphore, and removes the semaphore when no fetch uses it.
|
||||
func (f *HTTPFetcher) putHostSemaphore(host string) {
|
||||
f.hostSemMu.Lock()
|
||||
defer f.hostSemMu.Unlock()
|
||||
|
||||
sem := f.hostSems[host]
|
||||
|
||||
sem.count--
|
||||
if sem.count == 0 {
|
||||
delete(f.hostSems, host)
|
||||
}
|
||||
return sem
|
||||
}
|
||||
|
||||
// buildResult validates the upstream response and assembles a FetchResult
|
||||
@@ -475,18 +416,18 @@ func validateURL(ctx context.Context, rawURL string, allowHTTP bool) error {
|
||||
return ErrUnsupportedScheme
|
||||
}
|
||||
|
||||
parsedURL, err := neturl.Parse(rawURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrInvalidHost, err)
|
||||
}
|
||||
|
||||
// Hostname drops any port, and the brackets of an IPv6 address whether
|
||||
// or not a port follows them
|
||||
host := parsedURL.Hostname()
|
||||
// Parse to extract host
|
||||
host := extractHost(rawURL)
|
||||
if host == "" {
|
||||
return ErrInvalidHost
|
||||
}
|
||||
|
||||
// Remove port if present
|
||||
h, _, err := net.SplitHostPort(host)
|
||||
if err == nil {
|
||||
host = h
|
||||
}
|
||||
|
||||
// Block obvious localhost patterns
|
||||
if isLocalhost(host) {
|
||||
return ErrSSRFBlocked
|
||||
@@ -545,9 +486,6 @@ func isPrivateIP(ip net.IP) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// An IPv4-compatible IPv6 address is judged as the IPv4 address it holds
|
||||
ip = ipv4CompatibleToIPv4(ip)
|
||||
|
||||
// Check for loopback
|
||||
if ip.IsLoopback() {
|
||||
return true
|
||||
@@ -599,12 +537,7 @@ func isPrivateIP(ip net.IP) bool {
|
||||
}
|
||||
|
||||
// isBlockedIP reports whether ip is refused, either by the built-in
|
||||
// internal-range check or by one of the operator-supplied prefixes. An
|
||||
// IPv4-compatible address (in ::/96) is refused when a prefix contains the
|
||||
// address as written or the IPv4 address it holds. An IPv4-mapped address (in
|
||||
// ::ffff:0:0/96) is matched only as the IPv4 address it holds: Go keeps a
|
||||
// plain IPv4 address in that form too, and an IPv6 prefix such as ::/0 must
|
||||
// not refuse every IPv4 address.
|
||||
// internal-range check or by one of the operator-supplied prefixes.
|
||||
func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
|
||||
if isPrivateIP(ip) {
|
||||
return true
|
||||
@@ -617,28 +550,11 @@ func isBlockedIP(ip net.IP, blocked []netip.Prefix) bool {
|
||||
|
||||
addr = addr.Unmap()
|
||||
|
||||
// ip read as IPv4 where it is or holds an IPv4 address, otherwise the
|
||||
// zero netip.Addr, which no prefix contains. Matching asIPv4 as well as
|
||||
// addr refuses more only for an IPv4-compatible address.
|
||||
asIPv4, _ := netip.AddrFromSlice(ipv4CompatibleToIPv4(ip).To4())
|
||||
|
||||
return slices.ContainsFunc(blocked, func(prefix netip.Prefix) bool {
|
||||
return prefix.Contains(addr) || prefix.Contains(asIPv4)
|
||||
return prefix.Contains(addr)
|
||||
})
|
||||
}
|
||||
|
||||
// ipv4CompatibleToIPv4 returns the IPv4 address that an IPv4-compatible IPv6
|
||||
// address (in ::/96, such as ::127.0.0.1) holds in its last four bytes, and
|
||||
// any other address unchanged.
|
||||
func ipv4CompatibleToIPv4(ip net.IP) net.IP {
|
||||
addr, ok := netip.AddrFromSlice(ip)
|
||||
if ok && netip.MustParsePrefix("::/96").Contains(addr) {
|
||||
return ip[12:]
|
||||
}
|
||||
|
||||
return ip
|
||||
}
|
||||
|
||||
// ssrfSafeDialer validates IP addresses against the built-in blocked ranges
|
||||
// before connecting. New wraps dialSSRFSafe with the operator-supplied
|
||||
// blocklist; this entry point enforces the built-in ranges alone.
|
||||
@@ -687,33 +603,14 @@ func dialSSRFSafe(
|
||||
return nil, fmt.Errorf("%w to %s", errConnectFailed, host)
|
||||
}
|
||||
|
||||
// limitedReader wraps a reader and refuses a body longer than remaining
|
||||
// bytes with ErrResponseTooLarge. remaining goes below 0 once the body is
|
||||
// found to be longer.
|
||||
// limitedReader wraps a reader and limits the number of bytes read.
|
||||
type limitedReader struct {
|
||||
reader io.Reader
|
||||
remaining int64
|
||||
}
|
||||
|
||||
// Read reads at most the remaining bytes. Once they are read, it reads one
|
||||
// more byte to tell the end of the body from more data: at the end it
|
||||
// returns the wrapped reader's error, io.EOF; with more data it returns
|
||||
// ErrResponseTooLarge, as does every later Read.
|
||||
func (r *limitedReader) Read(p []byte) (int, error) {
|
||||
if r.remaining < 0 {
|
||||
return 0, ErrResponseTooLarge
|
||||
}
|
||||
|
||||
if r.remaining == 0 {
|
||||
var extra [1]byte
|
||||
|
||||
n, err := r.reader.Read(extra[:])
|
||||
r.remaining -= int64(n)
|
||||
|
||||
if n == 0 {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if r.remaining <= 0 {
|
||||
return 0, ErrResponseTooLarge
|
||||
}
|
||||
|
||||
|
||||
@@ -186,44 +186,6 @@ func TestIsPrivateIP(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsPrivateIPJudgesEmbeddedIPv4Address checks IPv6 addresses that hold an
|
||||
// IPv4 address in their last four bytes: the IPv4-compatible form (::/96) and
|
||||
// the NAT64 form (64:ff9b::/96). One holding a loopback, private or
|
||||
// link-local IPv4 address is refused as that address is, and one holding a
|
||||
// public IPv4 address in the IPv4-compatible form passes.
|
||||
func TestIsPrivateIPJudgesEmbeddedIPv4Address(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
ip string
|
||||
want bool
|
||||
}{
|
||||
{"::127.0.0.1", true}, // loopback
|
||||
{"::10.0.0.1", true}, // private
|
||||
{"::169.254.169.254", true}, // link-local
|
||||
{"64:ff9b::7f00:1", true}, // NAT64 form of 127.0.0.1
|
||||
{"::8.8.8.8", false}, // public
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.ip, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ip := net.ParseIP(tc.ip)
|
||||
if ip == nil {
|
||||
t.Fatalf("failed to parse IP %q", tc.ip)
|
||||
}
|
||||
|
||||
got := isPrivateIP(ip)
|
||||
t.Logf("isPrivateIP(%s) = %v", ip, got)
|
||||
|
||||
if got != tc.want {
|
||||
t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateURL_RejectsNonHTTPS(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package httpfetcher
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestJPEGXLContentType verifies that the fetcher accepts an upstream answer
|
||||
// of type image/jxl by default, and that the mock fetcher serves a .jxl file
|
||||
// as image/jxl.
|
||||
func TestJPEGXLContentType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const jxlType = "image/jxl"
|
||||
|
||||
if !New(DefaultConfig()).isAllowedContentType(jxlType) {
|
||||
t.Errorf("isAllowedContentType(%q) = false, want true", jxlType)
|
||||
}
|
||||
|
||||
if got := detectContentTypeFromPath("images/photo.jxl"); got != jxlType {
|
||||
t.Errorf("detectContentTypeFromPath(photo.jxl) = %q, want %q", got, jxlType)
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -120,98 +119,6 @@ func TestFetchFreesHostSlotWhenContextEndsWaitingForConnection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchRemovesIdleHostSemaphores checks that a host's semaphore is
|
||||
// removed once no fetch holds or waits for one of its slots: after 100
|
||||
// concurrent fetches from 50 hosts have all finished, no semaphore is left.
|
||||
func TestFetchRemovesIdleHostSemaphores(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
f, _ := newServerFetcher(t, srv, nil)
|
||||
ctx := testContext(t)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := range 100 {
|
||||
wg.Go(func() {
|
||||
res, err := f.Fetch(ctx, imageURLOnPort(1+i%50))
|
||||
if err != nil {
|
||||
t.Errorf("Fetch() error = %v", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
_ = res.Content.Close()
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if n := hostSemCount(f); n != 0 {
|
||||
t.Errorf("%d host semaphores left after every fetch finished, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchRemovesHostSemaphoreWhenNoConnection checks that a fetch that
|
||||
// ends without a connection leaves no semaphore behind: when it is refused
|
||||
// after waiting for a connection shared by all hosts, when its context ends
|
||||
// while it waits for its host's slot, and when its context ends while it
|
||||
// waits for a connection shared by all hosts, long before the 10 second
|
||||
// wait timeout.
|
||||
func TestFetchRemovesHostSemaphoreWhenNoConnection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxConnections = 1
|
||||
cfg.MaxConnectionsPerHost = 1
|
||||
|
||||
f, _ := newServerFetcher(t, srv, cfg)
|
||||
f.connectionWaitTimeout = 100 * time.Millisecond
|
||||
|
||||
open, err := f.Fetch(testContext(t), imageURLOnPort(81))
|
||||
if err != nil {
|
||||
t.Fatalf("first Fetch() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = f.Fetch(testContext(t), imageURLOnPort(82))
|
||||
if !errors.Is(err, ErrTooManyConnections) {
|
||||
t.Fatalf("Fetch() from another host: error = %v, "+
|
||||
"want ErrTooManyConnections", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
_, err = f.Fetch(ctx, imageURLOnPort(81))
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Fetch() from the busy host: error = %v, "+
|
||||
"want context.DeadlineExceeded", err)
|
||||
}
|
||||
|
||||
// Back to the 10 second wait, so the next fetch's context ends first.
|
||||
f.connectionWaitTimeout = ConnectionWaitTimeout
|
||||
|
||||
ctx, cancel = context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
_, err = f.Fetch(ctx, imageURLOnPort(83))
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Fetch() from a host with nothing open: error = %v, "+
|
||||
"want context.DeadlineExceeded", err)
|
||||
}
|
||||
|
||||
err = open.Content.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("close first body: %v", err)
|
||||
}
|
||||
|
||||
if n := hostSemCount(f); n != 0 {
|
||||
t.Errorf("%d host semaphores left after every fetch finished, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchReleasesConnectionOnError checks that a fetch that fails after
|
||||
// taking its connection gives it back: with MaxConnections at 1, the slot
|
||||
// must be free after the failure and the next fetch must succeed.
|
||||
|
||||
@@ -109,8 +109,6 @@ func detectContentTypeFromPath(path string) string {
|
||||
return contentTypeWebP
|
||||
case strings.HasSuffix(path, ".avif"):
|
||||
return contentTypeAVIF
|
||||
case strings.HasSuffix(path, ".jxl"):
|
||||
return contentTypeJXL
|
||||
case strings.HasSuffix(path, ".svg"):
|
||||
return contentTypeSVG
|
||||
default:
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package httpfetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
// TestFetchSendsRequestID verifies that a fetch sends the ID of the request
|
||||
// it serves, which the RequestID middleware stores in the request context,
|
||||
// to the upstream host as X-Request-Id, so the fetch can be found in that
|
||||
// host's logs.
|
||||
func TestFetchSendsRequestID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const requestID = "test-request-id"
|
||||
|
||||
received := make(chan string, 1)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
received <- r.Header.Get("X-Request-Id")
|
||||
|
||||
w.Header().Set("Content-Type", contentTypeJPEG)
|
||||
_, _ = io.WriteString(w, imagePayload)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
f, _ := newServerFetcher(t, srv, nil)
|
||||
|
||||
ctx := context.WithValue(testContext(t), middleware.RequestIDKey, requestID)
|
||||
|
||||
res, err := f.Fetch(ctx, upstreamURL("/image"))
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch() error = %v", err)
|
||||
}
|
||||
|
||||
_ = res.Content.Close()
|
||||
|
||||
got := <-received
|
||||
t.Logf("upstream received X-Request-Id %q", got)
|
||||
|
||||
if got != requestID {
|
||||
t.Errorf("upstream X-Request-Id = %q, want %q", got, requestID)
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_RefusesAVIFAboveMaxPixels verifies that an AVIF output
|
||||
// of more than MaxAVIFOutputPixels is refused with ErrAVIFOutputTooLarge
|
||||
// before anything is decoded, whatever size and fit give it, the source's own
|
||||
// size included, and that an AVIF output of at most that many pixels, or one
|
||||
// in another format, is not. One input is a 64x32 PNG cut off in its pixel
|
||||
// data. The other is testdata/red.avif, a 4x4 AVIF, with the size in its
|
||||
// header changed to 4097x4096. libvips reads each input's size from its
|
||||
// header but fails when it decodes its pixel data, so a request that gets
|
||||
// that failure was not refused, and a refusal came before any decoding.
|
||||
func TestImageProcessor_RefusesAVIFAboveMaxPixels(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
whole := createTestPNG(t, 64, 32)
|
||||
cutOff := whole[:len(whole)/2]
|
||||
|
||||
largeAVIF, err := os.ReadFile("testdata/red.avif")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test AVIF: %v", err)
|
||||
}
|
||||
|
||||
// The ispe box gives the width and then the height, 4 bytes each, after
|
||||
// the box's type and its 4 bytes of version and flags
|
||||
ispe := bytes.Index(largeAVIF, []byte("ispe"))
|
||||
binary.BigEndian.PutUint32(largeAVIF[ispe+8:], 4097)
|
||||
binary.BigEndian.PutUint32(largeAVIF[ispe+12:], 4096)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
format imagerequest.Format
|
||||
width, height int
|
||||
fit imagerequest.Fit
|
||||
refused bool
|
||||
}{
|
||||
{"AVIF at the limit", cutOff, imagerequest.FormatAVIF,
|
||||
4096, 4096, imagerequest.FitCover, false},
|
||||
{"AVIF above the limit", cutOff, imagerequest.FormatAVIF,
|
||||
4096, 4097, imagerequest.FitCover, true},
|
||||
{"WebP above the AVIF limit", cutOff, imagerequest.FormatWebP,
|
||||
4096, 4097, imagerequest.FitCover, false},
|
||||
// FitInside does not enlarge, so the output stays 64x32
|
||||
{"AVIF inside a larger size", cutOff, imagerequest.FormatAVIF,
|
||||
8192, 8192, imagerequest.FitInside, false},
|
||||
// FitOutside scales 64x32 by 128 to cover 8192x1: 8192x4096
|
||||
{"AVIF outside a smaller size", cutOff, imagerequest.FormatAVIF,
|
||||
8192, 1, imagerequest.FitOutside, true},
|
||||
// With no size, the output keeps the source's 4097x4096
|
||||
{"AVIF of the source's size", largeAVIF, imagerequest.FormatAVIF,
|
||||
0, 0, imagerequest.FitCover, true},
|
||||
{"orig of an AVIF source", largeAVIF, imagerequest.FormatOriginal,
|
||||
0, 0, imagerequest.FitCover, true},
|
||||
{"JPEG of the source's size", largeAVIF, imagerequest.FormatJPEG,
|
||||
0, 0, imagerequest.FitCover, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New(Params{}).Process(t.Context(), bytes.NewReader(tt.input),
|
||||
&imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: tt.width, Height: tt.height},
|
||||
Format: tt.format,
|
||||
FitMode: tt.fit,
|
||||
})
|
||||
t.Logf("Process() error = %v", err)
|
||||
|
||||
if tt.refused {
|
||||
if !errors.Is(err, ErrAVIFOutputTooLarge) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrAVIFOutputTooLarge)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil || errors.Is(err, ErrAVIFOutputTooLarge) {
|
||||
t.Errorf("Process() error = %v, want the failure to decode "+
|
||||
"the input", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_RefusesComputedSideAboveMaximum verifies that a size
|
||||
// with one side 0, which Process computes from the input's proportions, is
|
||||
// served when that side comes out at MaxOutputDimension and refused with
|
||||
// ErrOutputTooLarge when it would be above. A 4096x1 or 1x4096 input given
|
||||
// 2 for its short side has 8192 for its long side.
|
||||
func TestImageProcessor_RefusesComputedSideAboveMaximum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size imagerequest.Size
|
||||
wantErr error
|
||||
}{
|
||||
{"width at the maximum", 4096, 1,
|
||||
imagerequest.Size{Height: 2}, nil},
|
||||
{"width above the maximum", 4096, 1,
|
||||
imagerequest.Size{Height: 3}, ErrOutputTooLarge},
|
||||
{"height at the maximum", 1, 4096,
|
||||
imagerequest.Size{Width: 2}, nil},
|
||||
{"height above the maximum", 1, 4096,
|
||||
imagerequest.Size{Width: 3}, ErrOutputTooLarge},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatJPEG,
|
||||
FitMode: imagerequest.FitCover,
|
||||
})
|
||||
if result != nil {
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
|
||||
if max(result.Width, result.Height) != MaxOutputDimension {
|
||||
t.Errorf("Process() made %dx%d, want a side of %d",
|
||||
result.Width, result.Height, MaxOutputDimension)
|
||||
}
|
||||
}
|
||||
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Errorf("Process() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_ComputedSideIsAtLeastOne verifies that, for a size with
|
||||
// one side 0, the side computed from the input's proportions is 1 where it
|
||||
// would round down to 0, and that Process makes that size in every fit mode
|
||||
// but FitOutside, which scales the input to cover the size and so can make
|
||||
// it longer. Each case checks the size targetDimensions gives before it
|
||||
// calls Process: with that side at 0, an 8192x1 input at 1300x0 makes
|
||||
// libvips ask for about 39 GB with FitCover.
|
||||
func TestImageProcessor_ComputedSideIsAtLeastOne(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size imagerequest.Size
|
||||
wantWidth, wantHeight int
|
||||
}{
|
||||
{"8192x1 at 100x0", 8192, 1, imagerequest.Size{Width: 100}, 100, 1},
|
||||
{"8192x1 at 1300x0", 8192, 1, imagerequest.Size{Width: 1300}, 1300, 1},
|
||||
{"8192x2 at 100x0", 8192, 2, imagerequest.Size{Width: 100}, 100, 1},
|
||||
{"1x8192 at 0x100", 1, 8192, imagerequest.Size{Height: 100}, 1, 100},
|
||||
{"1x8192 at 0x1300", 1, 8192, imagerequest.Size{Height: 1300}, 1, 1300},
|
||||
{"2x8192 at 0x100", 2, 8192, imagerequest.Size{Height: 100}, 1, 100},
|
||||
}
|
||||
|
||||
fits := []imagerequest.Fit{
|
||||
imagerequest.FitCover, imagerequest.FitContain, imagerequest.FitFill,
|
||||
imagerequest.FitInside, imagerequest.FitOutside,
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
for _, fit := range fits {
|
||||
t.Run(tt.name+" "+string(fit), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
width, height, err := targetDimensions(tt.size,
|
||||
tt.inputWidth, tt.inputHeight)
|
||||
if err != nil || width != tt.wantWidth || height != tt.wantHeight {
|
||||
t.Fatalf("targetDimensions() = %dx%d, %v, want %dx%d",
|
||||
width, height, err, tt.wantWidth, tt.wantHeight)
|
||||
}
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatJPEG,
|
||||
FitMode: fit,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
|
||||
ok := result.Width == tt.wantWidth && result.Height == tt.wantHeight
|
||||
if fit == imagerequest.FitOutside {
|
||||
ok = result.Width >= tt.wantWidth && result.Height >= tt.wantHeight
|
||||
}
|
||||
|
||||
if !ok {
|
||||
t.Errorf("Process() made %dx%d, want %dx%d (at least, for %s)",
|
||||
result.Width, result.Height, tt.wantWidth, tt.wantHeight,
|
||||
imagerequest.FitOutside)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// readerAfterDeadline is a Process input whose first Read waits until done
|
||||
// is closed, so the deadline passes after Process has taken its slot and
|
||||
// before it encodes.
|
||||
type readerAfterDeadline struct {
|
||||
data *bytes.Reader
|
||||
done <-chan struct{}
|
||||
waited bool
|
||||
}
|
||||
|
||||
func (r *readerAfterDeadline) Read(p []byte) (int, error) {
|
||||
if !r.waited {
|
||||
r.waited = true
|
||||
|
||||
<-r.done
|
||||
}
|
||||
|
||||
return r.data.Read(p)
|
||||
}
|
||||
|
||||
// inputNotToBeRead is a Process input that fails the test when it is read.
|
||||
type inputNotToBeRead struct {
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (r inputNotToBeRead) Read([]byte) (int, error) {
|
||||
r.t.Error("Process read its input although its deadline had passed")
|
||||
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// TestProcessStopsWhenDeadlinePassed checks that Process returns
|
||||
// context.DeadlineExceeded, and holds no slot afterwards, when its context's
|
||||
// deadline has passed before it starts, then without reading its input, or
|
||||
// while it reads its input. govips gives pixa no way to stop libvips once an
|
||||
// encode has begun, so Process must not begin one for a request that has
|
||||
// ended.
|
||||
func TestProcessStopsWhenDeadlinePassed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 50, 50)
|
||||
|
||||
t.Run("before Process starts", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithDeadline(t.Context(),
|
||||
time.Now().Add(-time.Second))
|
||||
defer cancel()
|
||||
|
||||
// With a slot free, the select in acquireSlot takes the slot or sees
|
||||
// the ended context at random, so Process is called many times
|
||||
for range 100 {
|
||||
checkProcessStopsAtDeadline(ctx, t, inputNotToBeRead{t: t})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("while Process reads its input", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
checkProcessStopsAtDeadline(ctx, t, &readerAfterDeadline{
|
||||
data: bytes.NewReader(input), done: ctx.Done(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// checkProcessStopsAtDeadline runs Process on input with ctx, whose deadline
|
||||
// has passed by the time Process could encode, and fails the test unless
|
||||
// Process returns context.DeadlineExceeded and gives its slot back.
|
||||
func checkProcessStopsAtDeadline(
|
||||
ctx context.Context, t *testing.T, input io.Reader,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
proc := New(Params{MaxConcurrentProcessing: 1})
|
||||
|
||||
result, err := proc.Process(ctx, input, smallJPEGTransform())
|
||||
if result != nil {
|
||||
t.Logf("Process() made a %dx%d image after its deadline passed",
|
||||
result.Width, result.Height)
|
||||
|
||||
_ = result.Content.Close()
|
||||
}
|
||||
|
||||
t.Logf("Process() error = %v", err)
|
||||
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("Process() error = %v, want context.DeadlineExceeded", err)
|
||||
}
|
||||
|
||||
if held := len(proc.processingSemaphore); held != 0 {
|
||||
t.Errorf("%d slots still held after Process() returned", held)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_EmptyFormatRefused verifies that a request with no format
|
||||
// is refused, as both image routes give every request a format before it is
|
||||
// processed.
|
||||
func TestImageProcessor_EmptyFormatRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(createTestJPEG(t, 20, 20)),
|
||||
&imagerequest.Transform{},
|
||||
)
|
||||
if !errors.Is(err, ErrUnsupportedOutputFormat) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedOutputFormat)
|
||||
}
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// processedSize runs input through Process and returns the output's size in
|
||||
// bytes.
|
||||
func processedSize(
|
||||
t *testing.T, input []byte, transform *imagerequest.Transform,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(), bytes.NewReader(input), transform)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
_ = result.Content.Close()
|
||||
|
||||
return result.ContentLength
|
||||
}
|
||||
|
||||
// encodePNG encodes img as PNG with Go's encoder.
|
||||
func encodePNG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := png.Encode(&buf, img)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encode test PNG: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// decode decodes input with vips, as Process does.
|
||||
func decode(t *testing.T, input []byte) *vips.ImageRef {
|
||||
t.Helper()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(input)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode input: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(img.Close)
|
||||
|
||||
return img
|
||||
}
|
||||
|
||||
// TestImageProcessor_PNGIsCompressed verifies that a PNG output is
|
||||
// compressed: a flat 200x150 image, 90,000 bytes of raw pixels, comes out at
|
||||
// a small fraction of that.
|
||||
func TestImageProcessor_PNGIsCompressed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const width, height = 200, 150
|
||||
|
||||
flat := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
flat.Set(x, y, color.RGBA{R: 40, G: 120, B: 200, A: 255})
|
||||
}
|
||||
}
|
||||
|
||||
size := processedSize(t, encodePNG(t, flat),
|
||||
&imagerequest.Transform{Format: imagerequest.FormatPNG})
|
||||
|
||||
const rawBytes = width * height * 3
|
||||
if size > rawBytes/10 {
|
||||
t.Errorf("PNG output is %d bytes, want under a tenth of its %d raw",
|
||||
size, rawBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_WebPAtEffort2 verifies that WebP is saved at effort 2,
|
||||
// not libvips' default of 4: the output is the size of the same image saved
|
||||
// at effort 2.
|
||||
func TestImageProcessor_WebPAtEffort2(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
|
||||
size := processedSize(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatWebP, Quality: 85})
|
||||
|
||||
want, _, err := decode(t, input).ExportWebp(&vips.WebpExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: 85,
|
||||
ReductionEffort: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportWebp() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("WebP output is %d bytes, want %d, the size at effort 2",
|
||||
size, len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_AVIFAtEffort1 verifies that AVIF is saved at effort 1
|
||||
// with 8 bits per sample: the output is the size of the same image saved
|
||||
// with those settings. The source is 16-bit, which libvips saves with 12
|
||||
// bits when it is not given a bit depth, and 640x480: on a small image,
|
||||
// such as 64x48, efforts 1 and 2 give the same output.
|
||||
func TestImageProcessor_AVIFAtEffort1(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const width, height = 640, 480
|
||||
|
||||
source := image.NewRGBA64(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
source.Set(x, y, color.RGBA64{
|
||||
R: uint16((x * 65535 / width) & 0xffff),
|
||||
G: uint16((y * 65535 / height) & 0xffff),
|
||||
B: 32768,
|
||||
A: 65535,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
input := encodePNG(t, source)
|
||||
|
||||
size := processedSize(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatAVIF, Quality: 85})
|
||||
|
||||
want, _, err := decode(t, input).ExportAvif(&vips.AvifExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: 85,
|
||||
Effort: 1,
|
||||
Bitdepth: 8,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAvif() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("AVIF output is %d bytes, want %d, the size at effort 1 "+
|
||||
"and 8 bits", size, len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLAtEffort4 verifies that JPEG XL is saved at
|
||||
// effort 4, not libvips' default of 7: the output is the size of the same
|
||||
// image saved at effort 4, with the distance exportJXL gives quality 85.
|
||||
func TestImageProcessor_JPEGXLAtEffort4(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
|
||||
size := processedSize(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: 85})
|
||||
|
||||
params := vips.NewJxlExportParams()
|
||||
params.Quality = 85
|
||||
params.Distance = jxlDistance(85)
|
||||
params.Effort = 4
|
||||
|
||||
want, _, err := decode(t, input).ExportJxl(params)
|
||||
if err != nil {
|
||||
t.Fatalf("ExportJxl() error = %v", err)
|
||||
}
|
||||
|
||||
if size != int64(len(want)) {
|
||||
t.Errorf("JPEG XL output is %d bytes, want %d, the size at effort 4",
|
||||
size, len(want))
|
||||
}
|
||||
}
|
||||
@@ -1,278 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_FitCoverServesThinSource verifies that FitCover serves
|
||||
// a thin source at the requested size. An 8192x1 source scaled to cover
|
||||
// 8192x8192 is 8192 times as long as the output on its long side. The centre
|
||||
// of an 8192x1 source with the proportions of 1x8192 is 1/8192 of a pixel
|
||||
// wide. Each case is also turned on its side.
|
||||
func TestImageProcessor_FitCoverServesThinSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size imagerequest.Size
|
||||
}{
|
||||
{"8192x1 at 8192x8192", 8192, 1, imagerequest.Size{Width: 8192, Height: 8192}},
|
||||
{"8192x1 at 1x8192", 8192, 1, imagerequest.Size{Width: 1, Height: 8192}},
|
||||
{"1x8192 at 8192x8192", 1, 8192, imagerequest.Size{Width: 8192, Height: 8192}},
|
||||
{"1x8192 at 8192x1", 1, 8192, imagerequest.Size{Width: 8192, Height: 1}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatJPEG,
|
||||
FitMode: imagerequest.FitCover,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
|
||||
if result.Width != tt.size.Width || result.Height != tt.size.Height {
|
||||
t.Errorf("Process() made %dx%d, want %dx%d",
|
||||
result.Width, result.Height, tt.size.Width, tt.size.Height)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// createBandsPNG creates a PNG of three equal bands, red, green and blue,
|
||||
// side by side when it is wider than tall and top to bottom otherwise.
|
||||
func createBandsPNG(t *testing.T, width, height int) []byte {
|
||||
t.Helper()
|
||||
|
||||
bands := []color.RGBA{
|
||||
{R: 255, A: 255},
|
||||
{G: 255, A: 255},
|
||||
{B: 255, A: 255},
|
||||
}
|
||||
|
||||
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
band := 3 * y / height
|
||||
if width > height {
|
||||
band = 3 * x / width
|
||||
}
|
||||
|
||||
img.Set(x, y, bands[band])
|
||||
}
|
||||
}
|
||||
|
||||
return encodePNG(t, img)
|
||||
}
|
||||
|
||||
// TestImageProcessor_FitCoverCropsTheCentre verifies that FitCover keeps the
|
||||
// centre of the source: a 300x100 source of red, green and blue bands side
|
||||
// by side, or a 100x300 one with them top to bottom, comes out at 50x50 as
|
||||
// its green band.
|
||||
func TestImageProcessor_FitCoverCropsTheCentre(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
}{
|
||||
{"bands side by side", 300, 100},
|
||||
{"bands top to bottom", 100, 300},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createBandsPNG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 50, Height: 50},
|
||||
Format: imagerequest.FormatPNG,
|
||||
FitMode: imagerequest.FitCover,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
output, err := png.Decode(result.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode output: %v", err)
|
||||
}
|
||||
|
||||
bounds := output.Bounds()
|
||||
t.Logf("Process() made %dx%d", bounds.Dx(), bounds.Dy())
|
||||
|
||||
if bounds.Dx() != 50 || bounds.Dy() != 50 {
|
||||
t.Fatalf("Process() made %dx%d, want 50x50",
|
||||
bounds.Dx(), bounds.Dy())
|
||||
}
|
||||
|
||||
// Resampling may blend a little of the bands beside the
|
||||
// green one into the output's edges.
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x++ {
|
||||
r, g, b, _ := output.At(x, y).RGBA()
|
||||
if r>>8 > 64 || g>>8 < 192 || b>>8 > 64 {
|
||||
t.Fatalf("output pixel %d,%d is %d,%d,%d, want green",
|
||||
x, y, r>>8, g>>8, b>>8)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The greys of createSquarePNG, and the grey halfway between them.
|
||||
const (
|
||||
backgroundGrey = 192
|
||||
squareGrey = 64
|
||||
halfwayGrey = (backgroundGrey + squareGrey) / 2
|
||||
)
|
||||
|
||||
// createSquarePNG creates a grey PNG side pixels square with a darker grey
|
||||
// square squareSide pixels wide in its centre.
|
||||
func createSquarePNG(t *testing.T, side, squareSide int) []byte {
|
||||
t.Helper()
|
||||
|
||||
first := (side - squareSide) / 2
|
||||
last := first + squareSide - 1
|
||||
|
||||
img := image.NewGray(image.Rect(0, 0, side, side))
|
||||
for y := range side {
|
||||
for x := range side {
|
||||
grey := uint8(backgroundGrey)
|
||||
if x >= first && x <= last && y >= first && y <= last {
|
||||
grey = squareGrey
|
||||
}
|
||||
|
||||
img.SetGray(x, y, color.Gray{Y: grey})
|
||||
}
|
||||
}
|
||||
|
||||
return encodePNG(t, img)
|
||||
}
|
||||
|
||||
// squareLength returns how long the square is along greys, the greys of a
|
||||
// row or column of the output that crosses it: the distance between the
|
||||
// first and the last point where greys passes halfwayGrey, or 0 when there
|
||||
// is no such point. Resampling blurs the square's edges over several
|
||||
// pixels, and each point is placed between the two pixels on either side
|
||||
// of it in proportion to their greys, so the length is measured to a
|
||||
// fraction of a pixel.
|
||||
func squareLength(greys []float64) float64 {
|
||||
first, last := -1.0, -1.0
|
||||
|
||||
for i := 1; i < len(greys); i++ {
|
||||
before, after := greys[i-1], greys[i]
|
||||
if (before > halfwayGrey) == (after > halfwayGrey) {
|
||||
continue
|
||||
}
|
||||
|
||||
point := float64(i-1) + (before-halfwayGrey)/(before-after)
|
||||
if first < 0 {
|
||||
first = point
|
||||
}
|
||||
|
||||
last = point
|
||||
}
|
||||
|
||||
return last - first
|
||||
}
|
||||
|
||||
// TestImageProcessor_FitCoverKeepsProportions verifies that FitCover scales
|
||||
// both sides of the source by the same amount: a square in the centre of
|
||||
// the source comes out square. Its width and height are measured along the
|
||||
// output's middle row and column.
|
||||
func TestImageProcessor_FitCoverKeepsProportions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
side, squareSide int
|
||||
size imagerequest.Size
|
||||
}{
|
||||
{"16x16 at 100x60", 16, 8, imagerequest.Size{Width: 100, Height: 60}},
|
||||
{"16x16 at 60x100", 16, 8, imagerequest.Size{Width: 60, Height: 100}},
|
||||
{"10x10 at 100x35", 10, 2, imagerequest.Size{Width: 100, Height: 35}},
|
||||
{"10x10 at 35x100", 10, 2, imagerequest.Size{Width: 35, Height: 100}},
|
||||
{"64x64 at 300x100", 64, 16, imagerequest.Size{Width: 300, Height: 100}},
|
||||
{"64x64 at 100x300", 64, 16, imagerequest.Size{Width: 100, Height: 300}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createSquarePNG(t, tt.side, tt.squareSide)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatPNG,
|
||||
FitMode: imagerequest.FitCover,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
output, err := png.Decode(result.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode output: %v", err)
|
||||
}
|
||||
|
||||
bounds := output.Bounds()
|
||||
if bounds.Dx() != tt.size.Width || bounds.Dy() != tt.size.Height {
|
||||
t.Fatalf("Process() made %dx%d, want %dx%d",
|
||||
bounds.Dx(), bounds.Dy(), tt.size.Width, tt.size.Height)
|
||||
}
|
||||
|
||||
row := make([]float64, bounds.Dx())
|
||||
for x := range row {
|
||||
r, _, _, _ := output.At(x, bounds.Dy()/2).RGBA()
|
||||
row[x] = float64(r >> 8)
|
||||
}
|
||||
|
||||
column := make([]float64, bounds.Dy())
|
||||
for y := range column {
|
||||
r, _, _, _ := output.At(bounds.Dx()/2, y).RGBA()
|
||||
column[y] = float64(r >> 8)
|
||||
}
|
||||
|
||||
width, height := squareLength(row), squareLength(column)
|
||||
t.Logf("square is %.2f wide and %.2f high", width, height)
|
||||
|
||||
if width == 0 || math.Abs(width-height) > 0.5 {
|
||||
t.Errorf("square is %.2f wide and %.2f high, want it square",
|
||||
width, height)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_RefusesFitOutsideAboveMaximum verifies that FitOutside,
|
||||
// which scales the input to cover the requested size and so past it on one
|
||||
// side, is refused with ErrOutputTooLarge when that side would come out above
|
||||
// MaxOutputDimension, and otherwise makes the scaled size. A 1024x256 or
|
||||
// 256x1024 input scaled by 8 has 8192 for its long side.
|
||||
func TestImageProcessor_RefusesFitOutsideAboveMaximum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size imagerequest.Size
|
||||
// wantWidth and wantHeight are the output's, 0 for a refusal
|
||||
wantWidth, wantHeight int
|
||||
}{
|
||||
{"width above the maximum", 3000, 600,
|
||||
imagerequest.Size{Width: 1, Height: 8192}, 0, 0},
|
||||
{"height above the maximum", 600, 3000,
|
||||
imagerequest.Size{Width: 8192, Height: 1}, 0, 0},
|
||||
{"width at the maximum", 1024, 256,
|
||||
imagerequest.Size{Width: 1, Height: 2048}, 8192, 2048},
|
||||
{"height at the maximum", 256, 1024,
|
||||
imagerequest.Size{Width: 2048, Height: 1}, 2048, 8192},
|
||||
{"within the maximum", 3000, 600,
|
||||
imagerequest.Size{Width: 1, Height: 1200}, 6000, 1200},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatJPEG,
|
||||
FitMode: imagerequest.FitOutside,
|
||||
})
|
||||
if result != nil {
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
}
|
||||
|
||||
if tt.wantWidth == 0 {
|
||||
if !errors.Is(err, ErrOutputTooLarge) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrOutputTooLarge)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
if result.Width != tt.wantWidth || result.Height != tt.wantHeight {
|
||||
t.Errorf("Process() made %dx%d, want %dx%d",
|
||||
result.Width, result.Height, tt.wantWidth, tt.wantHeight)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_FitOutsideSideIsAtLeastOne verifies that FitOutside
|
||||
// keeps each side at 1 or more when it works both sides out again from the
|
||||
// larger scale. A side of 49 scaled to 1 is 49 times 1/49, which in floating
|
||||
// point is just under 1 and so rounded down to 0, a width libvips refuses.
|
||||
func TestImageProcessor_FitOutsideSideIsAtLeastOne(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
inputWidth, inputHeight int
|
||||
size imagerequest.Size
|
||||
wantWidth, wantHeight int
|
||||
}{
|
||||
{"49x8192 at 0x100", 49, 8192, imagerequest.Size{Height: 100}, 1, 167},
|
||||
{"49x49 at 1x1", 49, 49, imagerequest.Size{Width: 1, Height: 1}, 1, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, tt.inputWidth, tt.inputHeight)
|
||||
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatJPEG,
|
||||
FitMode: imagerequest.FitOutside,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
t.Logf("Process() made %dx%d", result.Width, result.Height)
|
||||
|
||||
if result.Width != tt.wantWidth || result.Height != tt.wantHeight {
|
||||
t.Errorf("Process() made %dx%d, want %dx%d",
|
||||
result.Width, result.Height, tt.wantWidth, tt.wantHeight)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// vipsOnce ensures vips is initialized exactly once.
|
||||
@@ -21,10 +20,11 @@ import (
|
||||
var vipsOnce sync.Once
|
||||
|
||||
// initVips initializes libvips with quiet logging, one worker thread per
|
||||
// image and no operation cache. Each request decodes different source
|
||||
// bytes, so the operation cache would rarely be hit and would hold memory
|
||||
// outside MaxConcurrentProcessing; repeated requests are served from
|
||||
// pixa's disk cache instead.
|
||||
// image and no operation cache. Process already works on one image per CPU
|
||||
// by default, so more threads per image would only compete for the CPUs.
|
||||
// Each request decodes different source bytes, so the operation cache
|
||||
// would rarely be hit and would hold memory outside MaxConcurrentProcessing;
|
||||
// repeated requests are served from pixa's disk cache instead.
|
||||
func initVips() {
|
||||
vipsOnce.Do(func() {
|
||||
vips.LoggingSettings(nil, vips.LogLevelError)
|
||||
@@ -37,22 +37,46 @@ func initVips() {
|
||||
})
|
||||
}
|
||||
|
||||
// errNoJPEGXL is returned by CheckJPEGXLSupport.
|
||||
var errNoJPEGXL = errors.New("libvips lacks JPEG XL support: install " +
|
||||
"vips-jxl on Alpine, or use a libvips built with libjxl")
|
||||
// Format represents supported output image formats.
|
||||
type Format string
|
||||
|
||||
// CheckJPEGXLSupport returns an error, naming the fix, when libvips
|
||||
// cannot load and save JPEG XL.
|
||||
func CheckJPEGXLSupport() error {
|
||||
initVips()
|
||||
// Supported image output formats.
|
||||
const (
|
||||
FormatOriginal Format = "orig"
|
||||
FormatJPEG Format = "jpeg"
|
||||
FormatPNG Format = "png"
|
||||
FormatWebP Format = "webp"
|
||||
FormatAVIF Format = "avif"
|
||||
FormatGIF Format = "gif"
|
||||
)
|
||||
|
||||
// govips counts a format as supported when libvips has its loader;
|
||||
// libvips builds the JPEG XL loader and saver together.
|
||||
if !vips.IsTypeSupported(vips.ImageTypeJXL) {
|
||||
return errNoJPEGXL
|
||||
}
|
||||
// FitMode represents how to fit an image into requested dimensions.
|
||||
type FitMode string
|
||||
|
||||
return nil
|
||||
// Supported image fit modes.
|
||||
const (
|
||||
FitCover FitMode = "cover"
|
||||
FitContain FitMode = "contain"
|
||||
FitFill FitMode = "fill"
|
||||
FitInside FitMode = "inside"
|
||||
FitOutside FitMode = "outside"
|
||||
)
|
||||
|
||||
// ErrInvalidFitMode is returned when an invalid fit mode is provided.
|
||||
var ErrInvalidFitMode = errors.New("invalid fit mode")
|
||||
|
||||
// Size represents requested image dimensions.
|
||||
type Size struct {
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
// Request holds the parameters for image processing.
|
||||
type Request struct {
|
||||
Size Size
|
||||
Format Format
|
||||
Quality int
|
||||
FitMode FitMode
|
||||
}
|
||||
|
||||
// Result contains the output of image processing.
|
||||
@@ -90,39 +114,10 @@ var ErrInputTooLarge = errors.New("input image dimensions exceed maximum")
|
||||
// configured byte limit.
|
||||
var ErrInputDataTooLarge = errors.New("input data exceeds maximum allowed size")
|
||||
|
||||
// MaxOutputDimension is the maximum width or height of an output image, and
|
||||
// so of a requested size (imagerequest.MaxDimension).
|
||||
const MaxOutputDimension = imagerequest.MaxDimension
|
||||
|
||||
// ErrOutputTooLarge is returned, before resizing, when the output would have a
|
||||
// side above MaxOutputDimension: the side of a size with one side 0, computed
|
||||
// from the input's proportions, or with FitOutside, the side scaled past the
|
||||
// requested size.
|
||||
var ErrOutputTooLarge = errors.New("output image dimensions exceed maximum")
|
||||
|
||||
// MaxAVIFOutputPixels is the most pixels an AVIF output may have, so that an
|
||||
// encode takes at most 40 seconds of the default downstream_timeout of 60
|
||||
// seconds; AVIF is already at the lowest effort govips can set (avifEffort).
|
||||
// With one thread, as pixad runs libvips, saving 4096x4096, 8192x2048 or
|
||||
// 2048x8192 random pixels with an alpha channel takes up to about 34
|
||||
// seconds, the slowest at any quality and at 8 or 16 bits per sample, and
|
||||
// 5120x4096 at 8 bits already takes about 36.
|
||||
const MaxAVIFOutputPixels = 4096 * 4096
|
||||
|
||||
// ErrAVIFOutputTooLarge is returned, before anything is decoded, when an AVIF
|
||||
// output would have more than MaxAVIFOutputPixels pixels.
|
||||
var ErrAVIFOutputTooLarge = errors.New("AVIF output has too many pixels")
|
||||
|
||||
// ErrUnsupportedOutputFormat is returned when the requested output format is
|
||||
// not supported.
|
||||
var ErrUnsupportedOutputFormat = errors.New("unsupported output format")
|
||||
|
||||
// ErrUnsupportedInputFormat is returned, before the input is decoded, when
|
||||
// libvips reads the input's first bytes as another format than the source
|
||||
// formats, or as none: for example an SVG whose first bytes pass the
|
||||
// first-bytes check for AVIF.
|
||||
var ErrUnsupportedInputFormat = errors.New("unsupported input format")
|
||||
|
||||
// ErrTooManyImages is returned when MaxConcurrentProcessing images are being
|
||||
// processed and none finishes within ProcessingWaitTimeout.
|
||||
var ErrTooManyImages = errors.New("too many images being processed at once")
|
||||
@@ -149,7 +144,7 @@ type Params struct {
|
||||
// If <= 0, DefaultMaxInputBytes is used.
|
||||
MaxInputBytes int64
|
||||
// MaxConcurrentProcessing is the most images processed at once.
|
||||
// If <= 0, config.DefaultMaxConcurrentProcessing() is used.
|
||||
// If <= 0, the number of CPUs Go uses (runtime.GOMAXPROCS(0)) is used.
|
||||
MaxConcurrentProcessing int
|
||||
}
|
||||
|
||||
@@ -165,7 +160,7 @@ func New(params Params) *ImageProcessor {
|
||||
|
||||
maxConcurrentProcessing := params.MaxConcurrentProcessing
|
||||
if maxConcurrentProcessing <= 0 {
|
||||
maxConcurrentProcessing = config.DefaultMaxConcurrentProcessing()
|
||||
maxConcurrentProcessing = runtime.GOMAXPROCS(0)
|
||||
}
|
||||
|
||||
return &ImageProcessor{
|
||||
@@ -175,18 +170,14 @@ func New(params Params) *ImageProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
// Process transforms an image according to transform. When
|
||||
// Process transforms an image according to the request. When
|
||||
// MaxConcurrentProcessing images are already being processed, it waits up
|
||||
// to ProcessingWaitTimeout for one to finish, then fails with
|
||||
// ErrTooManyImages. When ctx has ended once Process has a slot, or before it
|
||||
// encodes, it returns ctx's error and gives the slot back. govips gives pixa
|
||||
// no way to stop libvips once an encode has begun, so an encode keeps its
|
||||
// slot until it finishes. An AVIF output of more than MaxAVIFOutputPixels is
|
||||
// refused with ErrAVIFOutputTooLarge before anything is decoded.
|
||||
// ErrTooManyImages.
|
||||
func (p *ImageProcessor) Process(
|
||||
ctx context.Context,
|
||||
input io.Reader,
|
||||
transform *imagerequest.Transform,
|
||||
req *Request,
|
||||
) (*Result, error) {
|
||||
release, err := p.acquireSlot(ctx)
|
||||
if err != nil {
|
||||
@@ -195,20 +186,18 @@ func (p *ImageProcessor) Process(
|
||||
|
||||
defer release()
|
||||
|
||||
data, err := p.readInput(input)
|
||||
// Read input with a size limit to prevent unbounded memory consumption.
|
||||
// We read at most maxInputBytes+1 so we can detect if the input exceeds
|
||||
// the limit without consuming additional memory.
|
||||
limited := io.LimitReader(input, p.maxInputBytes+1)
|
||||
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("failed to read input: %w", err)
|
||||
}
|
||||
|
||||
// govips picks the libvips loader from the input's first bytes with
|
||||
// DetermineImageType. Input whose first bytes it reads as no source
|
||||
// format, such as SVG, which librsvg would render, is refused before
|
||||
// libvips reads it.
|
||||
inputFormat := p.detectFormat(vips.DetermineImageType(data))
|
||||
|
||||
sourceFormat, err := p.formatFromString(inputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if int64(len(data)) > p.maxInputBytes {
|
||||
return nil, ErrInputDataTooLarge
|
||||
}
|
||||
|
||||
// Decode image
|
||||
@@ -229,42 +218,33 @@ func (p *ImageProcessor) Process(
|
||||
origWidth := img.Width()
|
||||
origHeight := img.Height()
|
||||
|
||||
// Detect input format
|
||||
inputFormat := p.detectFormat(img)
|
||||
|
||||
// Validate input dimensions to prevent DoS via decompression bombs
|
||||
if origWidth > MaxInputDimension || origHeight > MaxInputDimension {
|
||||
return nil, ErrInputTooLarge
|
||||
}
|
||||
|
||||
// Determine target dimensions; too large ones are refused before resizing
|
||||
targetWidth, targetHeight, err := targetDimensions(transform.Size,
|
||||
origWidth, origHeight)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// orig is the source's own format; encode refuses an empty format
|
||||
outputFormat := transform.Format
|
||||
if outputFormat == imagerequest.FormatOriginal {
|
||||
outputFormat = sourceFormat
|
||||
}
|
||||
|
||||
// Determine output dimensions; too large ones are refused before
|
||||
// resizing, which with FitCover already decodes the input
|
||||
outputWidth, outputHeight, err := outputDimensions(origWidth, origHeight,
|
||||
targetWidth, targetHeight, transform.FitMode, outputFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Determine target dimensions
|
||||
targetWidth, targetHeight := targetDimensions(req.Size, origWidth, origHeight)
|
||||
|
||||
// Resize if needed
|
||||
if targetWidth != origWidth || targetHeight != origHeight {
|
||||
err := p.resize(img, outputWidth, outputHeight, transform.FitMode)
|
||||
err := p.resize(img, targetWidth, targetHeight, req.FitMode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resize: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine output format
|
||||
outputFormat := req.Format
|
||||
if outputFormat == FormatOriginal || outputFormat == "" {
|
||||
outputFormat = p.formatFromString(inputFormat)
|
||||
}
|
||||
|
||||
// Encode to target format
|
||||
output, err := p.encode(ctx, img, outputFormat, transform.Quality)
|
||||
output, err := p.encode(img, outputFormat, req.Quality)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to encode: %w", err)
|
||||
}
|
||||
@@ -283,33 +263,21 @@ func (p *ImageProcessor) Process(
|
||||
|
||||
// targetDimensions calculates the output dimensions for a requested size,
|
||||
// scaling proportionally when only one dimension is given and keeping the
|
||||
// original dimensions when both are zero. A side scaled proportionally is
|
||||
// rounded down but kept at 1 or more, as for a thin image it would round
|
||||
// down to 0, a side of no pixels. It can come out above MaxOutputDimension,
|
||||
// which is ErrOutputTooLarge.
|
||||
func targetDimensions(
|
||||
size imagerequest.Size, origWidth, origHeight int,
|
||||
) (int, int, error) {
|
||||
width, height := size.Width, size.Height
|
||||
|
||||
// original dimensions when both are zero.
|
||||
func targetDimensions(size Size, origWidth, origHeight int) (int, int) {
|
||||
switch {
|
||||
case width == 0 && height == 0:
|
||||
case size.Width == 0 && size.Height == 0:
|
||||
// Both are 0: keep original size
|
||||
return origWidth, origHeight, nil
|
||||
case width == 0:
|
||||
return origWidth, origHeight
|
||||
case size.Width == 0:
|
||||
// Only height specified: calculate width proportionally
|
||||
width = max(1, origWidth*height/origHeight)
|
||||
case height == 0:
|
||||
return origWidth * size.Height / origHeight, size.Height
|
||||
case size.Height == 0:
|
||||
// Only width specified: calculate height proportionally
|
||||
height = max(1, origHeight*width/origWidth)
|
||||
return size.Width, origHeight * size.Width / origWidth
|
||||
default:
|
||||
return size.Width, size.Height
|
||||
}
|
||||
|
||||
if width > MaxOutputDimension || height > MaxOutputDimension {
|
||||
return 0, 0, fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge,
|
||||
width, height, MaxOutputDimension)
|
||||
}
|
||||
|
||||
return width, height, nil
|
||||
}
|
||||
|
||||
// MIME types for the supported image formats.
|
||||
@@ -319,7 +287,6 @@ const (
|
||||
mimeGIF = "image/gif"
|
||||
mimeWebP = "image/webp"
|
||||
mimeAVIF = "image/avif"
|
||||
mimeJXL = "image/jxl"
|
||||
)
|
||||
|
||||
// SupportedInputFormats returns MIME types this processor can read.
|
||||
@@ -330,115 +297,67 @@ func (p *ImageProcessor) SupportedInputFormats() []string {
|
||||
mimeGIF,
|
||||
mimeWebP,
|
||||
mimeAVIF,
|
||||
mimeJXL,
|
||||
}
|
||||
}
|
||||
|
||||
// SupportedOutputFormats returns formats this processor can write.
|
||||
func (p *ImageProcessor) SupportedOutputFormats() []imagerequest.Format {
|
||||
return []imagerequest.Format{
|
||||
imagerequest.FormatJPEG,
|
||||
imagerequest.FormatPNG,
|
||||
imagerequest.FormatGIF,
|
||||
imagerequest.FormatWebP,
|
||||
imagerequest.FormatAVIF,
|
||||
imagerequest.FormatJXL,
|
||||
func (p *ImageProcessor) SupportedOutputFormats() []Format {
|
||||
return []Format{
|
||||
FormatJPEG,
|
||||
FormatPNG,
|
||||
FormatGIF,
|
||||
FormatWebP,
|
||||
FormatAVIF,
|
||||
}
|
||||
}
|
||||
|
||||
// FormatToMIME converts a Format to its MIME type string.
|
||||
func FormatToMIME(format imagerequest.Format) string {
|
||||
func FormatToMIME(format Format) string {
|
||||
switch format {
|
||||
case imagerequest.FormatJPEG:
|
||||
case FormatJPEG:
|
||||
return mimeJPEG
|
||||
case imagerequest.FormatPNG:
|
||||
case FormatPNG:
|
||||
return mimePNG
|
||||
case imagerequest.FormatWebP:
|
||||
case FormatWebP:
|
||||
return mimeWebP
|
||||
case imagerequest.FormatGIF:
|
||||
case FormatGIF:
|
||||
return mimeGIF
|
||||
case imagerequest.FormatAVIF:
|
||||
case FormatAVIF:
|
||||
return mimeAVIF
|
||||
case imagerequest.FormatJXL:
|
||||
return mimeJXL
|
||||
case imagerequest.FormatOriginal, imagerequest.FormatAuto:
|
||||
case FormatOriginal:
|
||||
return "application/octet-stream"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
// WaitForProcessing waits until no image is being processed, or until ctx
|
||||
// ends, and returns how many images were still being processed then. It
|
||||
// waits by taking each slot in processingSemaphore as it frees up until it
|
||||
// holds them all, or until ctx ends, then gives back the slots it took.
|
||||
func (p *ImageProcessor) WaitForProcessing(ctx context.Context) int {
|
||||
taken := 0
|
||||
|
||||
defer func() {
|
||||
for range taken {
|
||||
<-p.processingSemaphore
|
||||
}
|
||||
}()
|
||||
|
||||
for taken < cap(p.processingSemaphore) {
|
||||
select {
|
||||
case p.processingSemaphore <- struct{}{}:
|
||||
taken++
|
||||
case <-ctx.Done():
|
||||
return len(p.processingSemaphore) - taken
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// acquireSlot takes a slot in processingSemaphore, waiting at most
|
||||
// processingWaitTimeout for one to free up, and returns the func that gives
|
||||
// it back. When ctx has ended by the time it has a slot, it gives the slot
|
||||
// back and returns ctx's error.
|
||||
// it back. A free slot is taken even when ctx has ended; only the wait for
|
||||
// one stops when ctx ends, as the rest of Process does not check ctx.
|
||||
func (p *ImageProcessor) acquireSlot(ctx context.Context) (func(), error) {
|
||||
release := func() { <-p.processingSemaphore }
|
||||
|
||||
select {
|
||||
case p.processingSemaphore <- struct{}{}:
|
||||
return release, nil
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case p.processingSemaphore <- struct{}{}:
|
||||
return release, nil
|
||||
case <-time.After(p.processingWaitTimeout):
|
||||
return nil, ErrTooManyImages
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
release := func() { <-p.processingSemaphore }
|
||||
|
||||
// select takes the slot even when ctx has ended at the same moment
|
||||
err := ctx.Err()
|
||||
if err != nil {
|
||||
release()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return release, nil
|
||||
}
|
||||
|
||||
// readInput reads input with a size limit to prevent unbounded memory
|
||||
// consumption, refusing more than maxInputBytes with ErrInputDataTooLarge. It
|
||||
// reads at most maxInputBytes+1 so it can detect if the input exceeds the
|
||||
// limit without consuming additional memory.
|
||||
func (p *ImageProcessor) readInput(input io.Reader) ([]byte, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(input, p.maxInputBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read input: %w", err)
|
||||
}
|
||||
|
||||
if int64(len(data)) > p.maxInputBytes {
|
||||
return nil, ErrInputDataTooLarge
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// detectFormat returns the format string for a govips image type.
|
||||
func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string {
|
||||
switch imageType {
|
||||
// detectFormat returns the format string from a vips image.
|
||||
func (p *ImageProcessor) detectFormat(img *vips.ImageRef) string {
|
||||
format := img.Format()
|
||||
switch format {
|
||||
case vips.ImageTypeJPEG:
|
||||
return "jpeg"
|
||||
case vips.ImageTypePNG:
|
||||
@@ -448,346 +367,135 @@ func (p *ImageProcessor) detectFormat(imageType vips.ImageType) string {
|
||||
case vips.ImageTypeWEBP:
|
||||
return "webp"
|
||||
case vips.ImageTypeAVIF, vips.ImageTypeHEIF:
|
||||
return string(imagerequest.FormatAVIF)
|
||||
case vips.ImageTypeJXL:
|
||||
return string(imagerequest.FormatJXL)
|
||||
return string(FormatAVIF)
|
||||
case vips.ImageTypeUnknown, vips.ImageTypeMagick, vips.ImageTypePDF,
|
||||
vips.ImageTypeSVG, vips.ImageTypeTIFF, vips.ImageTypeBMP,
|
||||
vips.ImageTypeJP2K:
|
||||
vips.ImageTypeJP2K, vips.ImageTypeJXL:
|
||||
return "unknown"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// outputDimensions returns the dimensions resize gives an image of imgWidth x
|
||||
// imgHeight for the target dimensions and fit mode, without resizing it, and
|
||||
// refuses an output too large for format. With FitOutside one side can come
|
||||
// out larger than the target; when it would be above MaxOutputDimension,
|
||||
// outputDimensions returns ErrOutputTooLarge. An AVIF output of more than
|
||||
// MaxAVIFOutputPixels is ErrAVIFOutputTooLarge.
|
||||
func outputDimensions(
|
||||
imgWidth, imgHeight, width, height int,
|
||||
fit imagerequest.Fit, format imagerequest.Format,
|
||||
) (int, int, error) {
|
||||
scaleW := float64(width) / float64(imgWidth)
|
||||
scaleH := float64(height) / float64(imgHeight)
|
||||
outputWidth, outputHeight := width, height
|
||||
|
||||
switch fit {
|
||||
case imagerequest.FitCover, imagerequest.FitFill, "":
|
||||
// Cover crops, and fill stretches, to exactly the target dimensions
|
||||
|
||||
case imagerequest.FitContain:
|
||||
// Fit within the target dimensions, maintaining aspect ratio. Scaled
|
||||
// by the smaller scale, the short side of a thin image can round
|
||||
// down to 0, which libvips refuses as a width, so each side is at
|
||||
// least 1.
|
||||
scale := min(scaleW, scaleH)
|
||||
outputWidth = max(1, int(float64(imgWidth)*scale))
|
||||
outputHeight = max(1, int(float64(imgHeight)*scale))
|
||||
|
||||
case imagerequest.FitInside:
|
||||
// Same as contain, but only shrink
|
||||
scale := min(scaleW, scaleH, 1)
|
||||
outputWidth = max(1, int(float64(imgWidth)*scale))
|
||||
outputHeight = max(1, int(float64(imgHeight)*scale))
|
||||
|
||||
case imagerequest.FitOutside:
|
||||
// The smallest dimension fits, the other may exceed the target. A
|
||||
// side scaled to 1 can come out just under 1 in floating point, as
|
||||
// 49 times 1/49 does, and round down to 0, which libvips refuses as
|
||||
// a width, so each side is at least 1.
|
||||
scale := max(scaleW, scaleH)
|
||||
outputWidth = max(1, int(float64(imgWidth)*scale))
|
||||
outputHeight = max(1, int(float64(imgHeight)*scale))
|
||||
|
||||
default:
|
||||
return 0, 0, fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, fit)
|
||||
}
|
||||
|
||||
// The side FitOutside scales past the target is held to the maximum too
|
||||
if max(outputWidth, outputHeight) > MaxOutputDimension {
|
||||
return 0, 0, fmt.Errorf("%w: %dx%d is above %d", ErrOutputTooLarge,
|
||||
outputWidth, outputHeight, MaxOutputDimension)
|
||||
}
|
||||
|
||||
if format == imagerequest.FormatAVIF &&
|
||||
outputWidth*outputHeight > MaxAVIFOutputPixels {
|
||||
return 0, 0, fmt.Errorf("%w: %dx%d is above %d pixels",
|
||||
ErrAVIFOutputTooLarge, outputWidth, outputHeight, MaxAVIFOutputPixels)
|
||||
}
|
||||
|
||||
return outputWidth, outputHeight, nil
|
||||
}
|
||||
|
||||
// resize resizes the image to width x height, the dimensions
|
||||
// outputDimensions gives for the fit mode.
|
||||
// resize resizes the image according to the fit mode.
|
||||
func (p *ImageProcessor) resize(
|
||||
img *vips.ImageRef, width, height int, fit imagerequest.Fit,
|
||||
img *vips.ImageRef, width, height int, fit FitMode,
|
||||
) error {
|
||||
switch fit {
|
||||
case imagerequest.FitCover, "":
|
||||
// Crop the centre and resize to fill exact dimensions (default)
|
||||
return cropCentreAndResize(img, width, height)
|
||||
case FitCover, "":
|
||||
// Resize and crop to fill exact dimensions (default)
|
||||
return img.Thumbnail(width, height, vips.InterestingCentre)
|
||||
|
||||
case imagerequest.FitFill:
|
||||
case FitContain:
|
||||
// Resize to fit within dimensions, maintaining aspect ratio
|
||||
imgW, imgH := img.Width(), img.Height()
|
||||
scaleW := float64(width) / float64(imgW)
|
||||
scaleH := float64(height) / float64(imgH)
|
||||
scale := min(scaleW, scaleH)
|
||||
newW := int(float64(imgW) * scale)
|
||||
newH := int(float64(imgH) * scale)
|
||||
|
||||
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
||||
|
||||
case FitFill:
|
||||
// Resize to exact dimensions (may distort)
|
||||
return img.ThumbnailWithSize(width, height, vips.InterestingNone, vips.SizeForce)
|
||||
|
||||
case imagerequest.FitInside:
|
||||
// outputDimensions keeps the dimensions of an image that already fits
|
||||
if img.Width() == width && img.Height() == height {
|
||||
return nil
|
||||
case FitInside:
|
||||
// Same as contain, but only shrink
|
||||
if img.Width() <= width && img.Height() <= height {
|
||||
return nil // Already fits
|
||||
}
|
||||
|
||||
return img.Thumbnail(width, height, vips.InterestingNone)
|
||||
imgW, imgH := img.Width(), img.Height()
|
||||
scaleW := float64(width) / float64(imgW)
|
||||
scaleH := float64(height) / float64(imgH)
|
||||
scale := min(scaleW, scaleH)
|
||||
newW := int(float64(imgW) * scale)
|
||||
newH := int(float64(imgH) * scale)
|
||||
|
||||
case imagerequest.FitContain, imagerequest.FitOutside:
|
||||
return img.Thumbnail(width, height, vips.InterestingNone)
|
||||
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
||||
|
||||
case FitOutside:
|
||||
// Resize so smallest dimension fits, may exceed target on other dimension
|
||||
imgW, imgH := img.Width(), img.Height()
|
||||
scaleW := float64(width) / float64(imgW)
|
||||
scaleH := float64(height) / float64(imgH)
|
||||
scale := max(scaleW, scaleH)
|
||||
newW := int(float64(imgW) * scale)
|
||||
newH := int(float64(imgH) * scale)
|
||||
|
||||
return img.Thumbnail(newW, newH, vips.InterestingNone)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, fit)
|
||||
return fmt.Errorf("%w: %s", ErrInvalidFitMode, fit)
|
||||
}
|
||||
}
|
||||
|
||||
// cropCentreAndResize scales img by the same amount on both sides, the
|
||||
// least that makes it cover width x height, and cuts the centre width x
|
||||
// height out of it, as libvips' thumbnail does with InterestingCentre.
|
||||
// thumbnail alone scales the whole image and copies the result into memory
|
||||
// before it cuts, and for a thin image that is far larger than the image or
|
||||
// the output: 8192x64 at 8192x8192 is 1048576x8192, about 24 GiB. So the
|
||||
// centre is cut from img first, widened to whole pixels of img, which adds
|
||||
// less than one pixel of img on each side, and thumbnail scales and cuts
|
||||
// only that.
|
||||
func cropCentreAndResize(img *vips.ImageRef, width, height int) error {
|
||||
imgW, imgH := img.Width(), img.Height()
|
||||
|
||||
// An image wider than the requested proportions keeps its height and
|
||||
// loses width down to imgH*width/height, any other keeps its width and
|
||||
// loses height down to imgW*height/width. Half of what is lost comes
|
||||
// off each side, rounded down to whole pixels, so the part kept holds
|
||||
// all of the centre.
|
||||
left, top := 0, 0
|
||||
if imgW*height > imgH*width {
|
||||
left = (imgW*height - imgH*width) / (2 * height) //nolint:mnd // each side
|
||||
} else {
|
||||
top = (imgH*width - imgW*height) / (2 * width) //nolint:mnd // each side
|
||||
}
|
||||
|
||||
err := img.ExtractArea(left, top, imgW-2*left, imgH-2*top)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return img.Thumbnail(width, height, vips.InterestingCentre)
|
||||
}
|
||||
|
||||
const defaultQuality = 85
|
||||
|
||||
// encode encodes an image to the specified format. govips gives pixa no way
|
||||
// to stop libvips once an encode has begun, so when ctx has ended encode
|
||||
// returns ctx's error without beginning one.
|
||||
// encode encodes an image to the specified format.
|
||||
func (p *ImageProcessor) encode(
|
||||
ctx context.Context, img *vips.ImageRef, format imagerequest.Format,
|
||||
quality int,
|
||||
img *vips.ImageRef, format Format, quality int,
|
||||
) ([]byte, error) {
|
||||
err := ctx.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if quality <= 0 {
|
||||
quality = defaultQuality
|
||||
}
|
||||
|
||||
var params vips.ExportParams
|
||||
|
||||
switch format {
|
||||
case FormatJPEG:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeJPEG,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatPNG:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypePNG,
|
||||
}
|
||||
|
||||
case FormatGIF:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeGIF,
|
||||
}
|
||||
|
||||
case FormatWebP:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeWEBP,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatAVIF:
|
||||
params = vips.ExportParams{
|
||||
Format: vips.ImageTypeAVIF,
|
||||
Quality: quality,
|
||||
}
|
||||
|
||||
case FormatOriginal:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
}
|
||||
|
||||
// Stripping drops the ICC profile as well, and clients show an image
|
||||
// with no profile as sRGB, so convert to sRGB first. "srgb" names
|
||||
// libvips' built-in profile; govips' own sRGB path variable is set on
|
||||
// first use but read without a lock, so concurrent requests race on it.
|
||||
if img.HasICCProfile() {
|
||||
err = img.TransformICCProfileWithFallback("srgb", "srgb")
|
||||
err := img.TransformICCProfileWithFallback("srgb", "srgb")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert to sRGB: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
switch format {
|
||||
case imagerequest.FormatJPEG:
|
||||
return exportJPEG(img, quality)
|
||||
// Drop EXIF, XMP, IPTC and the ICC profile. govips ignores this for
|
||||
// GIF, which carries none of them.
|
||||
params.StripMetadata = true
|
||||
|
||||
case imagerequest.FormatPNG:
|
||||
return exportPNG(img)
|
||||
|
||||
case imagerequest.FormatGIF:
|
||||
return exportGIF(img)
|
||||
|
||||
case imagerequest.FormatWebP:
|
||||
return exportWebP(img, quality)
|
||||
|
||||
case imagerequest.FormatAVIF:
|
||||
return exportAVIF(img, quality)
|
||||
|
||||
case imagerequest.FormatJXL:
|
||||
return exportJXL(img, quality)
|
||||
|
||||
case imagerequest.FormatOriginal, imagerequest.FormatAuto:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedOutputFormat, format)
|
||||
}
|
||||
}
|
||||
|
||||
// govips sends libvips Go's zero value for some settings an export leaves
|
||||
// out, such as no compression at all for PNG, so each export below sets
|
||||
// every setting whose zero value is not what pixa wants. Stripping metadata
|
||||
// drops EXIF, XMP, IPTC and the ICC profile.
|
||||
|
||||
// exportJPEG encodes img as JPEG at quality, without metadata. The settings
|
||||
// it leaves out are at libvips' defaults.
|
||||
func exportJPEG(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportJpeg(&vips.JpegExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// pngCompression is libvips' default PNG compression, from 0 (none) to 9.
|
||||
const pngCompression = 6
|
||||
|
||||
// exportPNG encodes img as PNG at libvips' default compression and row
|
||||
// filter, without metadata.
|
||||
func exportPNG(img *vips.ImageRef) ([]byte, error) {
|
||||
output, _, err := img.ExportPng(&vips.PngExportParams{
|
||||
StripMetadata: true,
|
||||
Compression: pngCompression,
|
||||
Filter: vips.PngFilterNone,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// gifEffort is libvips' default GIF effort, from 1 to 10.
|
||||
const gifEffort = 7
|
||||
|
||||
// exportGIF encodes img as GIF at libvips' default effort. govips cannot
|
||||
// have libvips strip metadata from GIF, which carries none.
|
||||
func exportGIF(img *vips.ImageRef) ([]byte, error) {
|
||||
output, _, err := img.ExportGIF(&vips.GifExportParams{Effort: gifEffort})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// webpEffort is the WebP effort, from 0 (fastest) to 6. It is below libvips'
|
||||
// default of 4 so that an encode takes at most 40 seconds of the default
|
||||
// downstream_timeout of 60 seconds. With one thread, as pixad runs libvips,
|
||||
// saving an 8192x8192 image of random pixels with an alpha channel takes up
|
||||
// to about 23 seconds at 2, and up to about 16 without one, at any quality;
|
||||
// with an alpha channel, 3 takes about 50 seconds and 4 about 61.
|
||||
const webpEffort = 2
|
||||
|
||||
// exportWebP encodes img as lossy WebP at quality and webpEffort, without
|
||||
// metadata.
|
||||
func exportWebP(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportWebp(&vips.WebpExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
ReductionEffort: webpEffort,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// avifEffort is the AVIF effort, from 0 (fastest) to 9; 1 is the lowest
|
||||
// govips can set. With one thread, as pixad runs libvips, 1 takes about 52
|
||||
// seconds to save an 8192x8192 RGB image of random pixels at quality 85,
|
||||
// and about 96 with an alpha channel, against the default
|
||||
// downstream_timeout of 60 seconds, so an AVIF output is held to
|
||||
// MaxAVIFOutputPixels. On an image of milder noise, which 1 saves in about
|
||||
// 12 seconds, 2 takes nearly a minute and libvips' default, 4, takes
|
||||
// minutes.
|
||||
const avifEffort = 1
|
||||
|
||||
// avifBitdepth is the AVIF bit depth, 8 bits per sample for every image.
|
||||
// libvips would save a 16-bit image with 12, but at avifEffort that takes
|
||||
// about 54 seconds for a 16-bit 8192x8192 image of milder noise, nearly all
|
||||
// of the default downstream_timeout, and about 12 seconds with 8.
|
||||
const avifBitdepth = 8
|
||||
|
||||
// exportAVIF encodes img as lossy AVIF at quality, avifEffort and
|
||||
// avifBitdepth, without metadata.
|
||||
func exportAVIF(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
output, _, err := img.ExportAvif(&vips.AvifExportParams{
|
||||
StripMetadata: true,
|
||||
Quality: quality,
|
||||
Effort: avifEffort,
|
||||
Bitdepth: avifBitdepth,
|
||||
})
|
||||
|
||||
return output, err
|
||||
}
|
||||
|
||||
// jxlResolution is the resolution every JPEG XL image is saved with, in
|
||||
// pixels per millimetre as libvips counts it: 72 dpi, what libvips gives a
|
||||
// JPEG that names none.
|
||||
const jxlResolution = 72 / 25.4
|
||||
|
||||
// jxlEffort is the JPEG XL effort, from 1 (fastest) to 9. It is below
|
||||
// libvips' default of 7 so that an encode takes at most 40 seconds of the
|
||||
// default downstream_timeout of 60 seconds. With one thread, as pixad runs
|
||||
// libvips, saving an 8192x8192 image of random pixels with an alpha channel
|
||||
// takes up to about 27 seconds at 4, and up to about 13 without one, at any
|
||||
// quality; with an alpha channel, 5 takes about 43 seconds and 7 about 81.
|
||||
const jxlEffort = 4
|
||||
|
||||
// exportJXL encodes img as JPEG XL at quality and jxlEffort, without
|
||||
// metadata. govips sends libvips a distance, the JPEG XL encoder's own
|
||||
// measure of quality, along with the quality, and libvips then uses the
|
||||
// distance alone, so the quality is also given as a distance.
|
||||
func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
// libvips converts a CMYK image to sRGB before it saves WebP, AVIF or
|
||||
// PNG, but cannot save one as JPEG XL. encode has already converted
|
||||
// any image with an ICC profile to sRGB, so this is CMYK with none.
|
||||
if img.Interpretation() == vips.InterpretationCMYK {
|
||||
err := img.ToColorSpace(vips.InterpretationSRGB)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert CMYK to sRGB: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// govips cannot make libvips strip metadata from JPEG XL, so it is
|
||||
// removed from the image itself. RemoveMetadata removes EXIF, XMP and
|
||||
// IPTC but keeps the ICC profile.
|
||||
err := img.RemoveMetadata()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = img.RemoveICCProfile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// libvips 8.16 and later still write an EXIF block of their own, from
|
||||
// the image's size, orientation and resolution, and fixed values. The
|
||||
// image is upright, so its orientation is 1, but its resolution is still
|
||||
// the source's.
|
||||
toSave, err := img.CopyChangingResolution(jxlResolution, jxlResolution)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer toSave.Close()
|
||||
|
||||
params := vips.NewJxlExportParams()
|
||||
params.Quality = quality
|
||||
params.Distance = jxlDistance(quality)
|
||||
params.Effort = jxlEffort
|
||||
|
||||
output, _, err := toSave.ExportJxl(params)
|
||||
output, _, err := img.Export(¶ms)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -795,41 +503,20 @@ func exportJXL(img *vips.ImageRef, quality int) ([]byte, error) {
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// jxlDistance turns a quality from 1 to 100 into the JPEG XL encoder's
|
||||
// distance, with the formula libvips and libjxl use for their own quality
|
||||
// setting, except that 100 stays lossy (distance 0.1) where libjxl makes it
|
||||
// lossless.
|
||||
//
|
||||
//nolint:mnd // the constants of that formula
|
||||
func jxlDistance(quality int) float64 {
|
||||
q := float64(quality)
|
||||
|
||||
if quality >= 30 {
|
||||
return 0.1 + (100-q)*0.09
|
||||
}
|
||||
|
||||
return 53.0/3000.0*q*q - 23.0/20.0*q + 25.0
|
||||
}
|
||||
|
||||
// formatFromString converts a format string from detectFormat to Format. A
|
||||
// format detectFormat could not name is ErrUnsupportedInputFormat.
|
||||
func (p *ImageProcessor) formatFromString(
|
||||
format string,
|
||||
) (imagerequest.Format, error) {
|
||||
// formatFromString converts a format string to Format.
|
||||
func (p *ImageProcessor) formatFromString(format string) Format {
|
||||
switch format {
|
||||
case "jpeg":
|
||||
return imagerequest.FormatJPEG, nil
|
||||
return FormatJPEG
|
||||
case "png":
|
||||
return imagerequest.FormatPNG, nil
|
||||
return FormatPNG
|
||||
case "gif":
|
||||
return imagerequest.FormatGIF, nil
|
||||
return FormatGIF
|
||||
case "webp":
|
||||
return imagerequest.FormatWebP, nil
|
||||
case string(imagerequest.FormatAVIF):
|
||||
return imagerequest.FormatAVIF, nil
|
||||
case string(imagerequest.FormatJXL):
|
||||
return imagerequest.FormatJXL, nil
|
||||
return FormatWebP
|
||||
case string(FormatAVIF):
|
||||
return FormatAVIF
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %s", ErrUnsupportedInputFormat, format)
|
||||
return FormatJPEG
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -88,7 +87,7 @@ func isAVIF(data []byte) bool {
|
||||
|
||||
brand := string(data[8:12])
|
||||
|
||||
return brand == string(imagerequest.FormatAVIF) || brand == "avis"
|
||||
return brand == string(FormatAVIF) || brand == "avis"
|
||||
}
|
||||
|
||||
// detectMIME is a minimal magic-byte detector for test assertions.
|
||||
@@ -124,14 +123,14 @@ func TestImageProcessor_ResizeJPEG(t *testing.T) {
|
||||
|
||||
input := createTestJPEG(t, 800, 600)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 400, Height: 300},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 400, Height: 300},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
@@ -170,13 +169,13 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) {
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 200, Height: 150},
|
||||
Format: imagerequest.FormatPNG,
|
||||
FitMode: imagerequest.FitCover,
|
||||
req := &Request{
|
||||
Size: Size{Width: 200, Height: 150},
|
||||
Format: FormatPNG,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
@@ -197,7 +196,7 @@ func TestImageProcessor_ConvertToPNG(t *testing.T) {
|
||||
// processAndCheckSize processes a test JPEG of the given input dimensions
|
||||
// with the requested size and asserts the resulting dimensions.
|
||||
func processAndCheckSize(
|
||||
t *testing.T, inputW, inputH int, size imagerequest.Size, wantW, wantH int,
|
||||
t *testing.T, inputW, inputH int, size Size, wantW, wantH int,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
@@ -206,14 +205,14 @@ func processAndCheckSize(
|
||||
|
||||
input := createTestJPEG(t, inputW, inputH)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
req := &Request{
|
||||
Size: size,
|
||||
Format: imagerequest.FormatJPEG,
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
@@ -233,7 +232,7 @@ func TestImageProcessor_OriginalSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Width and height 0: keep original size
|
||||
processAndCheckSize(t, 640, 480, imagerequest.Size{Width: 0, Height: 0}, 640, 480)
|
||||
processAndCheckSize(t, 640, 480, Size{Width: 0, Height: 0}, 640, 480)
|
||||
}
|
||||
|
||||
func TestImageProcessor_FitContain(t *testing.T) {
|
||||
@@ -246,14 +245,14 @@ func TestImageProcessor_FitContain(t *testing.T) {
|
||||
// Should result in 400x200 (maintaining aspect ratio)
|
||||
input := createTestJPEG(t, 800, 400)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 400, Height: 400},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 400, Height: 400},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitContain,
|
||||
FitMode: FitContain,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
@@ -271,7 +270,7 @@ func TestImageProcessor_ProportionalScale_WidthOnly(t *testing.T) {
|
||||
|
||||
// 800x600 image, request width=400 height=0
|
||||
// Should scale proportionally to 400x300
|
||||
processAndCheckSize(t, 800, 600, imagerequest.Size{Width: 400, Height: 0}, 400, 300)
|
||||
processAndCheckSize(t, 800, 600, Size{Width: 400, Height: 0}, 400, 300)
|
||||
}
|
||||
|
||||
func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) {
|
||||
@@ -279,7 +278,7 @@ func TestImageProcessor_ProportionalScale_HeightOnly(t *testing.T) {
|
||||
|
||||
// 800x600 image, request width=0 height=300
|
||||
// Should scale proportionally to 400x300
|
||||
processAndCheckSize(t, 800, 600, imagerequest.Size{Width: 0, Height: 300}, 400, 300)
|
||||
processAndCheckSize(t, 800, 600, Size{Width: 0, Height: 300}, 400, 300)
|
||||
}
|
||||
|
||||
func TestImageProcessor_ProcessPNG(t *testing.T) {
|
||||
@@ -290,13 +289,13 @@ func TestImageProcessor_ProcessPNG(t *testing.T) {
|
||||
|
||||
input := createTestPNG(t, 400, 300)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 200, Height: 150},
|
||||
Format: imagerequest.FormatPNG,
|
||||
FitMode: imagerequest.FitCover,
|
||||
req := &Request{
|
||||
Size: Size{Width: 200, Height: 150},
|
||||
Format: FormatPNG,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
@@ -350,14 +349,14 @@ func TestImageProcessor_RejectsOversizedInput(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
input := createTestJPEG(t, tt.width, tt.height)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 100, Height: 100},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 100, Height: 100},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
_, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
_, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err == nil {
|
||||
t.Error("Process() should reject oversized input images")
|
||||
}
|
||||
@@ -378,14 +377,14 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) {
|
||||
// Create an image at exactly MaxInputDimension - should be accepted
|
||||
input := createTestJPEG(t, MaxInputDimension, 100)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 100, Height: 100},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 100, Height: 100},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf(
|
||||
"Process() should accept images at MaxInputDimension, got error: %v",
|
||||
@@ -398,9 +397,7 @@ func TestImageProcessor_AcceptsMaxDimensionInput(t *testing.T) {
|
||||
|
||||
// encodeAndCheck processes a 200x150 test JPEG into a 100x75 output of the
|
||||
// given format and asserts the output MIME type and dimensions.
|
||||
func encodeAndCheck(
|
||||
t *testing.T, format imagerequest.Format, quality int, wantMIME string,
|
||||
) {
|
||||
func encodeAndCheck(t *testing.T, format Format, quality int, wantMIME string) {
|
||||
t.Helper()
|
||||
|
||||
proc := New(Params{})
|
||||
@@ -408,14 +405,14 @@ func encodeAndCheck(
|
||||
|
||||
input := createTestJPEG(t, 200, 150)
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 100, Height: 75},
|
||||
req := &Request{
|
||||
Size: Size{Width: 100, Height: 75},
|
||||
Format: format,
|
||||
Quality: quality,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v, want nil", err)
|
||||
}
|
||||
@@ -446,7 +443,7 @@ func encodeAndCheck(
|
||||
func TestImageProcessor_EncodeWebP(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
encodeAndCheck(t, imagerequest.FormatWebP, 80, mimeWebP)
|
||||
encodeAndCheck(t, FormatWebP, 80, mimeWebP)
|
||||
}
|
||||
|
||||
func TestImageProcessor_DecodeAVIF(t *testing.T) {
|
||||
@@ -462,14 +459,14 @@ func TestImageProcessor_DecodeAVIF(t *testing.T) {
|
||||
}
|
||||
|
||||
// Request resize and convert to JPEG
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 2, Height: 2},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 2, Height: 2},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v, want nil (AVIF decoding should work)", err)
|
||||
}
|
||||
@@ -503,14 +500,14 @@ func TestImageProcessor_RejectsOversizedInputData(t *testing.T) {
|
||||
t.Fatalf("test JPEG must exceed %d bytes, got %d", limit, len(input))
|
||||
}
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 100, Height: 75},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 100, Height: 75},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
_, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
_, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err == nil {
|
||||
t.Fatal("Process() should reject input exceeding maxInputBytes")
|
||||
}
|
||||
@@ -530,14 +527,14 @@ func TestImageProcessor_AcceptsInputWithinLimit(t *testing.T) {
|
||||
proc := New(Params{MaxInputBytes: limit})
|
||||
ctx := context.Background()
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 10, Height: 10},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
req := &Request{
|
||||
Size: Size{Width: 10, Height: 10},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), transform)
|
||||
result, err := proc.Process(ctx, bytes.NewReader(input), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v, want nil", err)
|
||||
}
|
||||
@@ -564,18 +561,16 @@ func TestImageProcessor_DefaultMaxInputBytes(t *testing.T) {
|
||||
func TestImageProcessor_EncodeAVIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
encodeAndCheck(t, imagerequest.FormatAVIF, 85, mimeAVIF)
|
||||
encodeAndCheck(t, FormatAVIF, 85, mimeAVIF)
|
||||
}
|
||||
|
||||
// processAndDecode runs input through Process and decodes the output with
|
||||
// vips, so a test can inspect the image a client would receive.
|
||||
func processAndDecode(
|
||||
t *testing.T, input []byte, transform *imagerequest.Transform,
|
||||
) *vips.ImageRef {
|
||||
func processAndDecode(t *testing.T, input []byte, req *Request) *vips.ImageRef {
|
||||
t.Helper()
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
context.Background(), bytes.NewReader(input), transform,
|
||||
context.Background(), bytes.NewReader(input), req,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
@@ -619,17 +614,15 @@ func TestImageProcessor_StripsEXIF(t *testing.T) {
|
||||
t.Fatal("testdata/gps-exif.jpg has no GPS latitude")
|
||||
}
|
||||
|
||||
formats := []imagerequest.Format{
|
||||
imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatWebP,
|
||||
imagerequest.FormatAVIF, imagerequest.FormatGIF, imagerequest.FormatOriginal,
|
||||
formats := []Format{
|
||||
FormatJPEG, FormatPNG, FormatWebP, FormatAVIF, FormatGIF, FormatOriginal,
|
||||
}
|
||||
|
||||
for _, format := range formats {
|
||||
t.Run(string(format), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
output := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: format})
|
||||
output := processAndDecode(t, input, &Request{Format: format})
|
||||
|
||||
if output.HasExif() {
|
||||
t.Errorf("output has EXIF: %v", output.GetExif())
|
||||
@@ -651,21 +644,21 @@ func TestImageProcessor_AppliesEXIFOrientation(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
size imagerequest.Size
|
||||
size Size
|
||||
wantW int
|
||||
wantH int
|
||||
}{
|
||||
{name: "original size", size: imagerequest.Size{}, wantW: 8, wantH: 16},
|
||||
{name: "width only", size: imagerequest.Size{Width: 4}, wantW: 4, wantH: 8},
|
||||
{name: "original size", size: Size{}, wantW: 8, wantH: 16},
|
||||
{name: "width only", size: Size{Width: 4}, wantW: 4, wantH: 8},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
output := processAndDecode(t, input, &imagerequest.Transform{
|
||||
output := processAndDecode(t, input, &Request{
|
||||
Size: tt.size,
|
||||
Format: imagerequest.FormatPNG,
|
||||
Format: FormatPNG,
|
||||
})
|
||||
|
||||
if output.Width() != tt.wantW || output.Height() != tt.wantH {
|
||||
@@ -701,8 +694,7 @@ func TestImageProcessor_ConvertsWideGamutToSRGB(t *testing.T) {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatPNG})
|
||||
output := processAndDecode(t, input, &Request{Format: FormatPNG})
|
||||
|
||||
if output.HasICCProfile() {
|
||||
t.Error("output has an ICC profile")
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
// TestCheckJPEGXLSupport fails when libvips lacks JPEG XL support, as on
|
||||
// Alpine without the vips-jxl package.
|
||||
func TestCheckJPEGXLSupport(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := CheckJPEGXLSupport()
|
||||
if err != nil {
|
||||
t.Fatalf("CheckJPEGXLSupport() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLibvipsSavesAndLoadsJPEGXL saves an image as JPEG XL with govips and
|
||||
// loads it back. It fails when libvips lacks JPEG XL support, as on Alpine
|
||||
// without the vips-jxl package.
|
||||
func TestLibvipsSavesAndLoadsJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(createTestJPEG(t, 64, 48))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load test JPEG: %v", err)
|
||||
}
|
||||
|
||||
defer img.Close()
|
||||
|
||||
jxl, _, err := img.ExportJxl(vips.NewJxlExportParams())
|
||||
if err != nil {
|
||||
t.Fatalf("ExportJxl() error = %v", err)
|
||||
}
|
||||
|
||||
loaded, err := vips.NewImageFromBuffer(jxl)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the JPEG XL image: %v", err)
|
||||
}
|
||||
|
||||
defer loaded.Close()
|
||||
|
||||
if loaded.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("loaded format = %s, want jxl", vips.ImageTypes[loaded.Format()])
|
||||
}
|
||||
|
||||
if loaded.Width() != 64 || loaded.Height() != 48 {
|
||||
t.Errorf("loaded size = %dx%d, want 64x48", loaded.Width(), loaded.Height())
|
||||
}
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_EncodeJPEGXL converts a JPEG to a smaller JPEG XL and
|
||||
// checks the content type, and the format and size the output loads as.
|
||||
func TestImageProcessor_EncodeJPEGXL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
transform := &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 100, Height: 75},
|
||||
Format: imagerequest.FormatJXL,
|
||||
}
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(createTestJPEG(t, 200, 150)), transform,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Process() error = %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = result.Content.Close() }()
|
||||
|
||||
if result.ContentType != "image/jxl" {
|
||||
t.Errorf("ContentType = %q, want image/jxl", result.ContentType)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(result.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read result: %v", err)
|
||||
}
|
||||
|
||||
output, err := vips.NewImageFromBuffer(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load the output: %v", err)
|
||||
}
|
||||
|
||||
defer output.Close()
|
||||
|
||||
if output.Format() != vips.ImageTypeJXL {
|
||||
t.Errorf("output format = %s, want jxl", vips.ImageTypes[output.Format()])
|
||||
}
|
||||
|
||||
if output.Width() != 100 || output.Height() != 75 {
|
||||
t.Errorf("output size = %dx%d, want 100x75", output.Width(), output.Height())
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLQuality verifies that the quality reaches the JPEG
|
||||
// XL encoder: the same image comes out smaller at quality 30 than at 90.
|
||||
func TestImageProcessor_JPEGXLQuality(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := createTestJPEG(t, 400, 300)
|
||||
outputBytes := make(map[int]int64)
|
||||
|
||||
for _, quality := range []int{30, 90} {
|
||||
result, err := New(Params{}).Process(t.Context(),
|
||||
bytes.NewReader(input),
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL, Quality: quality})
|
||||
if err != nil {
|
||||
t.Fatalf("Process() at quality %d error = %v", quality, err)
|
||||
}
|
||||
|
||||
_ = result.Content.Close()
|
||||
outputBytes[quality] = result.ContentLength
|
||||
}
|
||||
|
||||
if outputBytes[30] >= outputBytes[90] {
|
||||
t.Errorf("%d bytes at quality 30, %d at 90, want fewer at 30",
|
||||
outputBytes[30], outputBytes[90])
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLDropsSourceEXIF verifies that none of the source's
|
||||
// EXIF reaches a JPEG XL output. libvips 8.16 and later write an EXIF block of
|
||||
// their own into JPEG XL (orientation, resolution, size, colour space and
|
||||
// fixed defaults), and govips cannot ask them to leave it out, so the test
|
||||
// looks for the source's fields rather than for no EXIF at all.
|
||||
func TestImageProcessor_JPEGXLDropsSourceEXIF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input, err := os.ReadFile("testdata/gps-exif.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
exif := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL}).GetExif()
|
||||
|
||||
for _, field := range []string{
|
||||
"exif-ifd0-Make", "exif-ifd0-Model", "exif-ifd2-BodySerialNumber",
|
||||
"exif-ifd2-DateTimeOriginal", "exif-ifd3-GPSLatitude",
|
||||
"exif-ifd3-GPSLongitude",
|
||||
} {
|
||||
if value, found := exif[field]; found {
|
||||
t.Errorf("output has %s: %s", field, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLDropsSourceResolution verifies that the source's
|
||||
// resolution does not reach the EXIF block libvips 8.16 and later write into
|
||||
// JPEG XL. A JPEG XL image holds its resolution in that block alone, so the
|
||||
// resolution the output loads with is the block's.
|
||||
func TestImageProcessor_JPEGXLDropsSourceResolution(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// dpi-300.jpg is a flat 8x8 grey image with a resolution of 300 dpi.
|
||||
input, err := os.ReadFile("testdata/dpi-300.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL})
|
||||
|
||||
// libvips gives the resolution in pixels per millimetre.
|
||||
xDPI := math.Round(output.ResX() * 25.4)
|
||||
yDPI := math.Round(output.ResY() * 25.4)
|
||||
|
||||
if xDPI == 300 || yDPI == 300 {
|
||||
t.Errorf("output resolution = %vx%v dpi, the source's", xDPI, yDPI)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLAppliesEXIFOrientation verifies that a JPEG XL
|
||||
// output is turned upright, as TestImageProcessor_AppliesEXIFOrientation does
|
||||
// for PNG.
|
||||
func TestImageProcessor_JPEGXLAppliesEXIFOrientation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// orientation-6.jpg is stored 16x8, red on the left and blue on the
|
||||
// right, with EXIF orientation 6 (turn 90 degrees clockwise to view).
|
||||
// Upright it is 8x16, red on top and blue below.
|
||||
input, err := os.ReadFile("testdata/orientation-6.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL})
|
||||
|
||||
if output.Width() != 8 || output.Height() != 16 {
|
||||
t.Fatalf("output is %dx%d, want 8x16", output.Width(), output.Height())
|
||||
}
|
||||
|
||||
top, err := output.GetPoint(4, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
bottom, err := output.GetPoint(4, 15)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
if top[0] <= top[2] || bottom[2] <= bottom[0] {
|
||||
t.Errorf("top pixel = %v, bottom pixel = %v, want red above blue",
|
||||
top, bottom)
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLConvertsWideGamutToSRGB verifies that a JPEG XL
|
||||
// output is converted to sRGB, as TestImageProcessor_ConvertsWideGamutToSRGB
|
||||
// does for PNG. It does not check for an ICC profile, as libvips reports one
|
||||
// for every JPEG XL image it loads.
|
||||
func TestImageProcessor_JPEGXLConvertsWideGamutToSRGB(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// display-p3.jpg is a flat 8x8 image with the Display P3 profile
|
||||
// embedded, filled with Display P3 (234, 51, 35), which is sRGB red.
|
||||
input, err := os.ReadFile("testdata/display-p3.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL})
|
||||
|
||||
pixel, err := output.GetPoint(4, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
want := []float64{255, 0, 0}
|
||||
for i := range want {
|
||||
if math.Abs(pixel[i]-want[i]) > 5 {
|
||||
t.Fatalf("pixel = %v, want within 5 of %v", pixel, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestImageProcessor_JPEGXLFromCMYK verifies that a CMYK JPEG with no ICC
|
||||
// profile can be served as JPEG XL, in sRGB.
|
||||
func TestImageProcessor_JPEGXLFromCMYK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// cmyk.jpg is a flat 8x8 CMYK image with no ICC profile, filled with
|
||||
// full cyan and no magenta, yellow or black.
|
||||
input, err := os.ReadFile("testdata/cmyk.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test JPEG: %v", err)
|
||||
}
|
||||
|
||||
output := processAndDecode(t, input,
|
||||
&imagerequest.Transform{Format: imagerequest.FormatJXL})
|
||||
|
||||
if output.Bands() != 3 {
|
||||
t.Fatalf("output has %d bands, want 3", output.Bands())
|
||||
}
|
||||
|
||||
pixel, err := output.GetPoint(4, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPoint() error = %v", err)
|
||||
}
|
||||
|
||||
// Cyan in sRGB: little red, much green and blue.
|
||||
if pixel[0] > 50 || pixel[1] < 100 || pixel[2] < 200 {
|
||||
t.Errorf("pixel = %v, want cyan", pixel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJXLDistance verifies the JPEG XL distance for a few qualities: 90 is
|
||||
// distance 1, the encoder's own default, and 100 stays lossy.
|
||||
func TestJXLDistance(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
quality int
|
||||
want float64
|
||||
}{
|
||||
{quality: 100, want: 0.1},
|
||||
{quality: 90, want: 1},
|
||||
{quality: 30, want: 6.4},
|
||||
{quality: 20, want: 9.0667},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := jxlDistance(tt.quality)
|
||||
if math.Abs(got-tt.want) > 0.0001 {
|
||||
t.Errorf("jxlDistance(%d) = %v, want %v", tt.quality, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// errTestReadFailed is the error the unreadable test input returns.
|
||||
@@ -78,13 +76,13 @@ func (r *gatedReader) Read(p []byte) (int, error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// smallJPEGTransform asks for a 5x5 JPEG.
|
||||
func smallJPEGTransform() *imagerequest.Transform {
|
||||
return &imagerequest.Transform{
|
||||
Size: imagerequest.Size{Width: 5, Height: 5},
|
||||
Format: imagerequest.FormatJPEG,
|
||||
// smallJPEGRequest asks for a 5x5 JPEG.
|
||||
func smallJPEGRequest() *Request {
|
||||
return &Request{
|
||||
Size: Size{Width: 5, Height: 5},
|
||||
Format: FormatJPEG,
|
||||
Quality: 85,
|
||||
FitMode: imagerequest.FitCover,
|
||||
FitMode: FitCover,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,7 +92,7 @@ func processInBackground(
|
||||
proc *ImageProcessor, reader *gatedReader, results chan<- error,
|
||||
) {
|
||||
go func() {
|
||||
result, err := proc.Process(context.Background(), reader, smallJPEGTransform())
|
||||
result, err := proc.Process(context.Background(), reader, smallJPEGRequest())
|
||||
if err == nil {
|
||||
_ = result.Content.Close()
|
||||
}
|
||||
@@ -117,14 +115,14 @@ func waitForEntries(t *testing.T, entered <-chan struct{}, count int) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewDefaultsMaxConcurrentProcessingToHalfTheCPUs(t *testing.T) {
|
||||
func TestNewDefaultsMaxConcurrentProcessingToCPUs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, limit := range []int{0, -1} {
|
||||
proc := New(Params{MaxConcurrentProcessing: limit})
|
||||
if got := cap(proc.processingSemaphore); got != max(runtime.GOMAXPROCS(0)/2, 1) {
|
||||
t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, half the CPUs",
|
||||
limit, got, max(runtime.GOMAXPROCS(0)/2, 1))
|
||||
if got := cap(proc.processingSemaphore); got != runtime.GOMAXPROCS(0) {
|
||||
t.Errorf("MaxConcurrentProcessing %d: %d slots, want %d, one per CPU",
|
||||
limit, got, runtime.GOMAXPROCS(0))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,30 +132,6 @@ func TestNewDefaultsMaxConcurrentProcessingToHalfTheCPUs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewDefaultIsHalfTheCPUsAndAtLeastOne checks that New, given no
|
||||
// MaxConcurrentProcessing, makes one slot for every two CPUs Go uses,
|
||||
// rounded down, and one slot when Go uses one CPU.
|
||||
//
|
||||
//nolint:paralleltest // it sets GOMAXPROCS, which other tests read
|
||||
func TestNewDefaultIsHalfTheCPUsAndAtLeastOne(t *testing.T) {
|
||||
cpusBefore := runtime.GOMAXPROCS(0)
|
||||
|
||||
t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) })
|
||||
|
||||
for _, tc := range []struct{ cpus, want int }{
|
||||
{1, 1}, {2, 1}, {3, 1}, {8, 4},
|
||||
} {
|
||||
runtime.GOMAXPROCS(tc.cpus)
|
||||
|
||||
got := cap(New(Params{}).processingSemaphore)
|
||||
t.Logf("GOMAXPROCS %d: %d slots", tc.cpus, got)
|
||||
|
||||
if got != tc.want {
|
||||
t.Errorf("GOMAXPROCS %d: %d slots, want %d", tc.cpus, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessNeverExceedsMaxConcurrentProcessing starts more Process calls
|
||||
// than MaxConcurrentProcessing allows and holds the first ones inside
|
||||
// Process until the test lets them go. No more than the limit may be
|
||||
@@ -239,7 +213,7 @@ func TestProcessWaitsThenFailsWhenNoSlotFrees(t *testing.T) {
|
||||
start := time.Now()
|
||||
|
||||
_, err := proc.Process(context.Background(), bytes.NewReader(input),
|
||||
smallJPEGTransform())
|
||||
smallJPEGRequest())
|
||||
if !errors.Is(err, ErrTooManyImages) {
|
||||
t.Fatalf("Process() error = %v, want ErrTooManyImages", err)
|
||||
}
|
||||
@@ -265,38 +239,38 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
|
||||
|
||||
valid := createTestJPEG(t, 10, 10)
|
||||
|
||||
unsupported := smallJPEGTransform()
|
||||
unsupported := smallJPEGRequest()
|
||||
unsupported.Format = "bmp"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
input io.Reader
|
||||
transform *imagerequest.Transform
|
||||
name string
|
||||
input io.Reader
|
||||
req *Request
|
||||
// want is the error Process must return; nil means any error.
|
||||
want error
|
||||
}{
|
||||
{
|
||||
name: "input cannot be read",
|
||||
input: iotest.ErrReader(errTestReadFailed),
|
||||
transform: smallJPEGTransform(),
|
||||
want: errTestReadFailed,
|
||||
name: "input cannot be read",
|
||||
input: iotest.ErrReader(errTestReadFailed),
|
||||
req: smallJPEGRequest(),
|
||||
want: errTestReadFailed,
|
||||
},
|
||||
{
|
||||
name: "input over the byte limit",
|
||||
input: bytes.NewReader(createTestJPEG(t, 800, 600)),
|
||||
transform: smallJPEGTransform(),
|
||||
want: ErrInputDataTooLarge,
|
||||
name: "input over the byte limit",
|
||||
input: bytes.NewReader(createTestJPEG(t, 800, 600)),
|
||||
req: smallJPEGRequest(),
|
||||
want: ErrInputDataTooLarge,
|
||||
},
|
||||
{
|
||||
name: "input not an image",
|
||||
input: strings.NewReader("not an image"),
|
||||
transform: smallJPEGTransform(),
|
||||
name: "input not an image",
|
||||
input: strings.NewReader("not an image"),
|
||||
req: smallJPEGRequest(),
|
||||
},
|
||||
{
|
||||
name: "output format not supported",
|
||||
input: bytes.NewReader(valid),
|
||||
transform: unsupported,
|
||||
want: ErrUnsupportedOutputFormat,
|
||||
name: "output format not supported",
|
||||
input: bytes.NewReader(valid),
|
||||
req: unsupported,
|
||||
want: ErrUnsupportedOutputFormat,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -307,7 +281,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
|
||||
proc := New(Params{MaxInputBytes: 4096, MaxConcurrentProcessing: 1})
|
||||
proc.processingWaitTimeout = 100 * time.Millisecond
|
||||
|
||||
_, err := proc.Process(context.Background(), tc.input, tc.transform)
|
||||
_, err := proc.Process(context.Background(), tc.input, tc.req)
|
||||
if err == nil || (tc.want != nil && !errors.Is(err, tc.want)) {
|
||||
t.Fatalf("Process() error = %v, want %v", err, tc.want)
|
||||
}
|
||||
@@ -317,7 +291,7 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
|
||||
}
|
||||
|
||||
result, err := proc.Process(context.Background(), bytes.NewReader(valid),
|
||||
smallJPEGTransform())
|
||||
smallJPEGRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("Process() after the error = %v, want nil", err)
|
||||
}
|
||||
@@ -326,69 +300,3 @@ func TestProcessReleasesSlotOnError(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWaitForProcessing holds a processing slot with a Process call that
|
||||
// cannot finish reading its input. WaitForProcessing must report that image
|
||||
// when its context ends first, wait for it otherwise, return 0 once it has
|
||||
// finished, and give back the slots it took while waiting.
|
||||
func TestWaitForProcessing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{MaxConcurrentProcessing: 2})
|
||||
|
||||
gate := make(chan struct{})
|
||||
entered := make(chan struct{}, 1)
|
||||
results := make(chan error, 1)
|
||||
|
||||
openGate := sync.OnceFunc(func() { close(gate) })
|
||||
t.Cleanup(openGate)
|
||||
|
||||
processInBackground(proc, &gatedReader{
|
||||
data: bytes.NewReader(createTestJPEG(t, 10, 10)), gate: gate,
|
||||
entered: entered, counter: &readingCounter{},
|
||||
}, results)
|
||||
waitForEntries(t, entered, 1)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
stillProcessing := proc.WaitForProcessing(ctx)
|
||||
t.Logf("WaitForProcessing() after its context ended: %d", stillProcessing)
|
||||
|
||||
if stillProcessing != 1 {
|
||||
t.Errorf("WaitForProcessing() after its context ended = %d, want 1",
|
||||
stillProcessing)
|
||||
}
|
||||
|
||||
waited := make(chan int, 1)
|
||||
|
||||
go func() { waited <- proc.WaitForProcessing(t.Context()) }()
|
||||
|
||||
select {
|
||||
case got := <-waited:
|
||||
t.Fatalf("WaitForProcessing() = %d while an image was being processed",
|
||||
got)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
|
||||
openGate()
|
||||
|
||||
err := <-results
|
||||
if err != nil {
|
||||
t.Errorf("Process() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case got := <-waited:
|
||||
if got != 0 {
|
||||
t.Errorf("WaitForProcessing() once processing finished = %d, want 0",
|
||||
got)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("WaitForProcessing() did not return once processing finished")
|
||||
}
|
||||
|
||||
if held := len(proc.processingSemaphore); held != 0 {
|
||||
t.Errorf("%d slots still held after WaitForProcessing() returned", held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_OrigOfUnnamedFormatRefused verifies that orig of a TIFF
|
||||
// input, which is not a source format, is refused with
|
||||
// ErrUnsupportedInputFormat before it is decoded, and not saved as JPEG.
|
||||
func TestImageProcessor_OrigOfUnnamedFormatRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
img, err := vips.NewImageFromBuffer(createTestPNG(t, 20, 20))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decode test PNG: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(img.Close)
|
||||
|
||||
tiff, _, err := img.ExportTiff(vips.NewTiffExportParams())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to save test TIFF: %v", err)
|
||||
}
|
||||
|
||||
result, err := New(Params{}).Process(
|
||||
t.Context(), bytes.NewReader(tiff),
|
||||
&imagerequest.Transform{Format: imagerequest.FormatOriginal},
|
||||
)
|
||||
if result != nil {
|
||||
t.Logf("Process() returned %s", result.ContentType)
|
||||
}
|
||||
|
||||
if !errors.Is(err, ErrUnsupportedInputFormat) {
|
||||
t.Errorf("Process() error = %v, want %v", err, ErrUnsupportedInputFormat)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package imageprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestImageProcessor_LoadsEachSourceFormat verifies that an image in each of
|
||||
// the six source formats, saved by the processor itself, is loaded and saved
|
||||
// again in its own format (orig).
|
||||
func TestImageProcessor_LoadsEachSourceFormat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proc := New(Params{})
|
||||
|
||||
for _, format := range []imagerequest.Format{
|
||||
imagerequest.FormatJPEG, imagerequest.FormatPNG, imagerequest.FormatGIF,
|
||||
imagerequest.FormatWebP, imagerequest.FormatAVIF, imagerequest.FormatJXL,
|
||||
} {
|
||||
saved, err := proc.Process(t.Context(),
|
||||
bytes.NewReader(createTestPNG(t, 20, 20)),
|
||||
&imagerequest.Transform{Format: format})
|
||||
if err != nil {
|
||||
t.Fatalf("saving the test image as %s: %v", format, err)
|
||||
}
|
||||
|
||||
source, err := io.ReadAll(saved.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the %s image: %v", format, err)
|
||||
}
|
||||
|
||||
result, err := proc.Process(t.Context(), bytes.NewReader(source),
|
||||
&imagerequest.Transform{Format: imagerequest.FormatOriginal})
|
||||
if err != nil {
|
||||
t.Errorf("orig of the %s image: %v", format, err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
t.Logf("orig of the %s image: %s", format, result.ContentType)
|
||||
|
||||
if result.ContentType != FormatToMIME(format) {
|
||||
t.Errorf("orig of the %s image is %s, want %s",
|
||||
format, result.ContentType, FormatToMIME(format))
|
||||
}
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 352 B |
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 799 B |
@@ -1,147 +0,0 @@
|
||||
// Package imagerequest holds the image request type, its parts, its checks,
|
||||
// the parsing of the /v1/image/ path, the variant key and the source key. It
|
||||
// imports no other package of this repo, so every package that handles an
|
||||
// image request can import it.
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrInvalidFitMode is returned when an invalid fit mode is provided.
|
||||
var ErrInvalidFitMode = errors.New("invalid fit mode")
|
||||
|
||||
// Format represents supported output image formats.
|
||||
type Format string
|
||||
|
||||
// Supported image output formats.
|
||||
const (
|
||||
FormatOriginal Format = "orig"
|
||||
FormatJPEG Format = "jpeg"
|
||||
FormatPNG Format = "png"
|
||||
FormatWebP Format = "webp"
|
||||
FormatAVIF Format = "avif"
|
||||
FormatJXL Format = "jxl"
|
||||
FormatGIF Format = "gif"
|
||||
|
||||
// FormatAuto stands for JPEG XL, AVIF, WebP or JPEG, chosen for each
|
||||
// request from its Accept header once the URL's signature or token has
|
||||
// been checked; it is never processed or cached as itself.
|
||||
FormatAuto Format = "auto"
|
||||
)
|
||||
|
||||
// Size represents requested image dimensions
|
||||
type Size struct {
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
// OriginalSize returns true if this represents "keep original size"
|
||||
func (s Size) OriginalSize() bool {
|
||||
return s.Width == 0 && s.Height == 0
|
||||
}
|
||||
|
||||
// Fit represents how to fit image into requested dimensions.
|
||||
type Fit string
|
||||
|
||||
// Supported image fit modes.
|
||||
const (
|
||||
FitCover Fit = "cover"
|
||||
FitContain Fit = "contain"
|
||||
FitFill Fit = "fill"
|
||||
FitInside Fit = "inside"
|
||||
FitOutside Fit = "outside"
|
||||
)
|
||||
|
||||
// ValidateFitMode checks if the given fit mode is valid.
|
||||
// Returns ErrInvalidFitMode for unrecognized fit modes.
|
||||
func ValidateFitMode(fit Fit) error {
|
||||
switch fit {
|
||||
case FitCover, FitContain, FitFill, FitInside, FitOutside, "":
|
||||
return nil
|
||||
default:
|
||||
return ErrInvalidFitMode
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateImageRequest checks a request's width and height with
|
||||
// ValidateDimension and its fit mode with ValidateFitMode. Both the plain
|
||||
// /v1/image/ route and the encrypted /v1/e/ route validate through this
|
||||
// function so a request from either source enforces identical bounds.
|
||||
func ValidateImageRequest(req *Request) error {
|
||||
err := ValidateDimension("width", req.Transform.Size.Width)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = ValidateDimension("height", req.Transform.Size.Height)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ValidateFitMode(req.Transform.FitMode)
|
||||
}
|
||||
|
||||
// Request represents a request for a processed image: the source to fetch
|
||||
// and the transform to apply to it.
|
||||
type Request struct {
|
||||
// Source is what to fetch
|
||||
Source Source
|
||||
// Transform is what to do to it
|
||||
Transform Transform
|
||||
// Signature is the HMAC signature for non-allowlisted hosts
|
||||
Signature string
|
||||
// Expires is when the URL expires: the exp of a signed URL, or the expiry
|
||||
// of an encrypted URL; the zero time if it has none
|
||||
Expires time.Time
|
||||
// AllowHTTP indicates whether HTTP (non-TLS) is allowed for this request
|
||||
AllowHTTP bool
|
||||
}
|
||||
|
||||
// Source is the image a request fetches from the origin.
|
||||
type Source struct {
|
||||
// Host is the origin host (e.g., "cdn.example.com")
|
||||
Host string
|
||||
// Path is the path on the origin with its percent-escapes decoded, as
|
||||
// url.URL's Path holds it (e.g., "/photos/my cat.jpg")
|
||||
Path string
|
||||
// Query is the optional query string for the origin URL as the URL
|
||||
// writes it, percent-escapes included, as url.URL's RawQuery holds it
|
||||
Query string
|
||||
}
|
||||
|
||||
// Transform is what a request does to its source; the image processor
|
||||
// takes it alone.
|
||||
type Transform struct {
|
||||
// Size is the requested output dimensions
|
||||
Size Size
|
||||
// Format is the requested output format
|
||||
Format Format
|
||||
// Quality is the output quality (1-100) for lossy formats
|
||||
Quality int
|
||||
// FitMode is how to fit the image into requested dimensions
|
||||
FitMode Fit
|
||||
}
|
||||
|
||||
// SourceURL returns the full upstream URL to fetch: the source path
|
||||
// percent-encoded again, as net/url writes a path, so each source path is one
|
||||
// upstream path, and the source query as it is, with the characters
|
||||
// escapeQuery encodes percent-encoded. Uses http:// scheme when AllowHTTP is
|
||||
// true, otherwise https://.
|
||||
func (r *Request) SourceURL() string {
|
||||
scheme := "https"
|
||||
if r.AllowHTTP {
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
sourceURL := url.URL{
|
||||
Scheme: scheme,
|
||||
Host: r.Source.Host,
|
||||
Path: r.Source.Path,
|
||||
RawQuery: escapeQuery(r.Source.Query),
|
||||
}
|
||||
|
||||
return sourceURL.String()
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SourceKey is a SHA256 hash identifying a cached source image (hex-encoded).
|
||||
type SourceKey string
|
||||
|
||||
// Key returns the key the source is cached under, made as CacheKey makes a
|
||||
// variant key.
|
||||
// Format: sha256(host:path:query)
|
||||
// The path and query go through escapePercentAndColon, so that a ":" in
|
||||
// either cannot be taken for the one between them.
|
||||
func (s Source) Key() SourceKey {
|
||||
data := fmt.Sprintf("%s:%s:%s",
|
||||
s.Host,
|
||||
escapePercentAndColon(s.Path),
|
||||
escapePercentAndColon(s.Query),
|
||||
)
|
||||
h := sha256.Sum256([]byte(data))
|
||||
|
||||
return SourceKey(hex.EncodeToString(h[:]))
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package imagerequest
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestSourceKey_DiffersForEachSource checks that each source gets a key of
|
||||
// 64 hex characters, the same each time, and that no two of these sources
|
||||
// share one, also when they differ only by a ":" or "%" in the path or query.
|
||||
func TestSourceKey_DiffersForEachSource(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sources := []Source{
|
||||
{Host: testHostCDN, Path: testPathCat},
|
||||
{Host: testHostCDN, Path: testPathCat, Query: "v=2"},
|
||||
{Host: "other.example.com", Path: testPathCat},
|
||||
{Host: testHostCDN, Path: "/c:d"},
|
||||
{Host: testHostCDN, Path: "/c", Query: "d:"},
|
||||
{Host: testHostCDN, Path: "/c%3Ad"},
|
||||
}
|
||||
|
||||
seen := make(map[SourceKey]Source)
|
||||
|
||||
for _, source := range sources {
|
||||
key := source.Key()
|
||||
t.Logf("Key() of %+v = %s", source, key)
|
||||
|
||||
if len(key) != 64 {
|
||||
t.Errorf("Key() of %+v length = %d, want 64", source, len(key))
|
||||
}
|
||||
|
||||
if again := source.Key(); again != key {
|
||||
t.Errorf("Key() of %+v not deterministic: %s vs %s", source, key, again)
|
||||
}
|
||||
|
||||
if earlier, found := seen[key]; found {
|
||||
t.Errorf("Key() gave %+v and %+v the same key", earlier, source)
|
||||
}
|
||||
|
||||
seen[key] = source
|
||||
}
|
||||
}
|
||||
@@ -1,357 +0,0 @@
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// URL parsing errors.
|
||||
var (
|
||||
ErrInvalidPath = errors.New("invalid image path")
|
||||
ErrMissingHost = errors.New("missing source host")
|
||||
ErrInvalidHost = errors.New("invalid source host")
|
||||
ErrMissingSize = errors.New("missing size specification")
|
||||
ErrInvalidSize = errors.New("invalid size format")
|
||||
ErrInvalidFormat = errors.New("invalid or unsupported format")
|
||||
ErrDimensionTooLarge = errors.New("dimension exceeds maximum")
|
||||
ErrPathTraversal = errors.New("path traversal detected")
|
||||
)
|
||||
|
||||
// MaxDimension is the maximum allowed width or height, the same as the image
|
||||
// processor's maximum for an output image (imageprocessor.MaxOutputDimension).
|
||||
const MaxDimension = 8192
|
||||
|
||||
// ValidateDimension checks one requested width or height; name ("width" or
|
||||
// "height") appears in the error. 0 is valid: that side is then worked out from
|
||||
// the source's proportions and the other requested side, rounded down but at
|
||||
// least 1, or is the source's own side when the other side is 0 too.
|
||||
func ValidateDimension(name string, value int) error {
|
||||
if value < 0 {
|
||||
return fmt.Errorf("%w: %s is negative", ErrInvalidSize, name)
|
||||
}
|
||||
|
||||
if value > MaxDimension {
|
||||
return fmt.Errorf("%w: %s is above %d",
|
||||
ErrDimensionTooLarge, name, MaxDimension)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sizeFormatRegex matches patterns like "800x600.webp", "0x0.jpeg", "orig.png",
|
||||
// and a size with no format, such as "800x600" or "orig"
|
||||
var sizeFormatRegex = regexp.MustCompile(
|
||||
`^(\d+)x(\d+)(?:\.(\w+))?$|^(orig)(?:\.(\w+))?$`)
|
||||
|
||||
// percentEscapeRegex matches one percent-escape, such as "%2e"
|
||||
var percentEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}`)
|
||||
|
||||
// queryEscapeRegex matches a percent-escape, or one character that a browser
|
||||
// percent-encodes in the query of an http or https URL: one outside the
|
||||
// printable ASCII characters "!" to "~", such as a space, a control character
|
||||
// or a character outside ASCII, or one of `"#'<>`. A "%" that does not begin a
|
||||
// percent-escape matches too.
|
||||
var queryEscapeRegex = regexp.MustCompile(`%[0-9A-Fa-f]{2}|[^!-~]|["#'<>%]`)
|
||||
|
||||
// hostRegex matches the forms the host of an image URL may take: a host name
|
||||
// or an IPv4 address (letters, digits, "-", "_" and "."), or an IPv6 address
|
||||
// in brackets (hex digits, ":" and "."), either one optionally followed by ":"
|
||||
// and a port, which may be empty. Its one group holds the text in brackets,
|
||||
// which parseImageComponents also checks is an IPv6 address.
|
||||
var hostRegex = regexp.MustCompile(
|
||||
`^(?:[A-Za-z0-9_.-]+|\[([0-9A-Fa-f:.]+)\])(?::[0-9]*)?$`)
|
||||
|
||||
// ParseImagePath parses an image URL's path after /v1/image/, decoded once:
|
||||
// <host>/<path>/<size>.<format>, or <host>/<path>/<size> for JPEG XL.
|
||||
// Examples:
|
||||
// - cdn.example.com/photos/cat.jpg/800x600.webp
|
||||
// - cdn.example.com/photos/cat.jpg/0x0.jpeg
|
||||
// - cdn.example.com/photos/cat.jpg/orig.png
|
||||
// - cdn.example.com/photos/cat.jpg/800x600
|
||||
func ParseImagePath(path string) (*Request, error) {
|
||||
// Strip leading slash if present (chi may include it)
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
if path == "" {
|
||||
return nil, ErrMissingHost
|
||||
}
|
||||
|
||||
return parseImageComponents(path)
|
||||
}
|
||||
|
||||
// ParseImageURL parses an image URL's path, decoded once as r.URL.Path holds
|
||||
// it: /v1/image/<host>/<path>/<size>.<format>, or /v1/image/<host>/<path>/<size>
|
||||
// for JPEG XL. A source query follows the source path after a "?", which the
|
||||
// URL writes as %3F.
|
||||
func ParseImageURL(urlPath string) (*Request, error) {
|
||||
// Remove the /v1/image/ prefix
|
||||
const prefix = "/v1/image/"
|
||||
if !strings.HasPrefix(urlPath, prefix) {
|
||||
return nil, ErrInvalidPath
|
||||
}
|
||||
|
||||
remainder := strings.TrimPrefix(urlPath, prefix)
|
||||
if remainder == "" {
|
||||
return nil, ErrMissingHost
|
||||
}
|
||||
|
||||
return parseImageComponents(remainder)
|
||||
}
|
||||
|
||||
// parseImageComponents parses <host>/<path>/<size>.<format>, or
|
||||
// <host>/<path>/<size> for JPEG XL.
|
||||
func parseImageComponents(remainder string) (*Request, error) {
|
||||
// Find the last path segment, which holds "size" or "size.format"
|
||||
lastSlash := strings.LastIndex(remainder, "/")
|
||||
if lastSlash == -1 {
|
||||
return nil, ErrMissingSize
|
||||
}
|
||||
|
||||
sizeFormat := remainder[lastSlash+1:]
|
||||
hostAndPath := remainder[:lastSlash]
|
||||
|
||||
if hostAndPath == "" {
|
||||
return nil, ErrMissingHost
|
||||
}
|
||||
|
||||
// Parse size and format from the last segment
|
||||
size, format, err := parseSizeFormat(sizeFormat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Split host from path
|
||||
// The first segment is the host, everything after is the path
|
||||
firstSlash := strings.Index(hostAndPath, "/")
|
||||
|
||||
var host, path, query string
|
||||
|
||||
if firstSlash == -1 {
|
||||
// No path, just host (unusual but valid)
|
||||
host = hostAndPath
|
||||
path = "/"
|
||||
} else {
|
||||
host = hostAndPath[:firstSlash]
|
||||
path = hostAndPath[firstSlash:]
|
||||
}
|
||||
|
||||
if host == "" {
|
||||
return nil, ErrMissingHost
|
||||
}
|
||||
|
||||
// Only the forms hostRegex describes are accepted. A "?", "#" or "@" in
|
||||
// the host would make Go's URL parser, which the allowlist check and the
|
||||
// fetcher use, read a different host from the one allowlisted and cached
|
||||
hostMatch := hostRegex.FindStringSubmatch(host)
|
||||
if hostMatch == nil {
|
||||
return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host)
|
||||
}
|
||||
|
||||
// Go's URL parser refuses brackets around anything but an IPv6 address
|
||||
if inBrackets := hostMatch[1]; inBrackets != "" {
|
||||
addr, err := netip.ParseAddr(inBrackets)
|
||||
if err != nil || !addr.Is6() {
|
||||
return nil, fmt.Errorf("%w: %q", ErrInvalidHost, host)
|
||||
}
|
||||
}
|
||||
|
||||
// Host names do not depend on letter case, so the allowlist, the upstream
|
||||
// connection limits, the cache and the signature see one spelling of each
|
||||
// host
|
||||
host = strings.ToLower(host)
|
||||
|
||||
// Go's URL parser, which the fetcher uses, reads a "#" as the start of a
|
||||
// fragment and drops the rest, so each text after it would be another
|
||||
// spelling, cached under its own keys, of one upstream request
|
||||
if strings.Contains(path, "#") {
|
||||
return nil, fmt.Errorf(
|
||||
`%w: the source path or query holds a "#"`, ErrInvalidPath)
|
||||
}
|
||||
|
||||
// Extract query string if present in path. It is the source URL's query
|
||||
// as that URL writes it. A character a URL's query cannot hold as it is,
|
||||
// such as a space written %20 in the image URL, is percent-encoded here,
|
||||
// so the query is signed and cached as it is sent upstream
|
||||
if qIndex := strings.Index(path, "?"); qIndex != -1 {
|
||||
query = escapeQuery(path[qIndex+1:])
|
||||
path = path[:qIndex]
|
||||
}
|
||||
|
||||
// The source query names no file on the upstream, so only the host and
|
||||
// the source path are checked for path traversal
|
||||
err = checkPathTraversal(host + path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Ensure path starts with /
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
path = "/" + path
|
||||
}
|
||||
|
||||
return &Request{
|
||||
Source: Source{Host: host, Path: path, Query: query},
|
||||
Transform: Transform{Size: size, Format: format},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkPathTraversal detects path traversal attempts in a URL path.
|
||||
// It checks for various attack vectors including:
|
||||
// - Direct ../ sequences
|
||||
// - URL-encoded variants (%2e%2e, %252e%252e)
|
||||
// - Backslash variants (..\)
|
||||
// - Null byte injection (%00)
|
||||
func checkPathTraversal(path string) error {
|
||||
// First, URL-decode the path to catch encoded attacks
|
||||
// Decode multiple times to catch double-encoding
|
||||
decoded := path
|
||||
for range 3 {
|
||||
newDecoded := decodePercentEscapes(decoded)
|
||||
if newDecoded == decoded {
|
||||
break
|
||||
}
|
||||
|
||||
decoded = newDecoded
|
||||
}
|
||||
|
||||
// Normalize backslashes to forward slashes
|
||||
normalized := strings.ReplaceAll(decoded, "\\", "/")
|
||||
|
||||
// Check for null bytes
|
||||
if strings.Contains(normalized, "\x00") {
|
||||
return ErrPathTraversal
|
||||
}
|
||||
|
||||
// Check for parent directory traversal
|
||||
// Look for "/.." or "../" patterns
|
||||
if strings.Contains(normalized, "/../") ||
|
||||
strings.Contains(normalized, "/..") ||
|
||||
strings.HasPrefix(normalized, "../") ||
|
||||
strings.HasSuffix(normalized, "/..") ||
|
||||
normalized == ".." {
|
||||
return ErrPathTraversal
|
||||
}
|
||||
|
||||
// Also check for ".." as a path segment in the original path
|
||||
// This catches cases where the path hasn't been normalized
|
||||
for seg := range strings.SplitSeq(path, "/") {
|
||||
// URL decode the segment
|
||||
decodedSeg, _ := url.PathUnescape(seg)
|
||||
decodedSeg = strings.ReplaceAll(decodedSeg, "\\", "/")
|
||||
|
||||
if decodedSeg == ".." {
|
||||
return ErrPathTraversal
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// decodePercentEscapes decodes every percent-escape in s, such as %2e, and
|
||||
// keeps a "%" that does not start one as it is, where url.PathUnescape would
|
||||
// refuse the whole string. A source path holding %25, a literal "%", has such
|
||||
// a "%" after one more decoding.
|
||||
func decodePercentEscapes(s string) string {
|
||||
return percentEscapeRegex.ReplaceAllStringFunc(s, func(escape string) string {
|
||||
decoded, _ := url.PathUnescape(escape)
|
||||
|
||||
return decoded
|
||||
})
|
||||
}
|
||||
|
||||
// escapeQuery returns a source query as a URL's query holds it: each
|
||||
// character that queryEscapeRegex names, such as a space, percent-encoded,
|
||||
// and the rest, the query's percent-escapes included, kept as they are. The
|
||||
// result is unchanged by a second escapeQuery.
|
||||
func escapeQuery(query string) string {
|
||||
return queryEscapeRegex.ReplaceAllStringFunc(query, func(match string) string {
|
||||
if percentEscapeRegex.MatchString(match) {
|
||||
return match
|
||||
}
|
||||
|
||||
return url.PathEscape(match)
|
||||
})
|
||||
}
|
||||
|
||||
// parseSizeFormat parses strings like "800x600.webp", "orig.png" or "800x600"
|
||||
func parseSizeFormat(s string) (Size, Format, error) {
|
||||
matches := sizeFormatRegex.FindStringSubmatch(s)
|
||||
if matches == nil {
|
||||
return Size{}, "", ErrInvalidSize
|
||||
}
|
||||
|
||||
var (
|
||||
size Size
|
||||
formatStr string
|
||||
)
|
||||
|
||||
if matches[4] == "orig" {
|
||||
// "orig" or "orig.format" pattern
|
||||
size = Size{Width: 0, Height: 0}
|
||||
formatStr = matches[5]
|
||||
} else {
|
||||
// "WxH" or "WxH.format" pattern
|
||||
width, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return Size{}, "", ErrInvalidSize
|
||||
}
|
||||
|
||||
height, err := strconv.Atoi(matches[2])
|
||||
if err != nil {
|
||||
return Size{}, "", ErrInvalidSize
|
||||
}
|
||||
|
||||
size = Size{Width: width, Height: height}
|
||||
formatStr = matches[3]
|
||||
}
|
||||
|
||||
err := ValidateDimension("width", size.Width)
|
||||
if err != nil {
|
||||
return Size{}, "", err
|
||||
}
|
||||
|
||||
err = ValidateDimension("height", size.Height)
|
||||
if err != nil {
|
||||
return Size{}, "", err
|
||||
}
|
||||
|
||||
// A URL that names no format is served, and signed, as JPEG XL
|
||||
if formatStr == "" {
|
||||
return size, FormatJXL, nil
|
||||
}
|
||||
|
||||
format, err := parseFormat(formatStr)
|
||||
if err != nil {
|
||||
return Size{}, "", err
|
||||
}
|
||||
|
||||
return size, format, nil
|
||||
}
|
||||
|
||||
// parseFormat converts a format string to Format.
|
||||
func parseFormat(s string) (Format, error) {
|
||||
switch strings.ToLower(s) {
|
||||
case "orig", "original":
|
||||
return FormatOriginal, nil
|
||||
case "jpg", "jpeg":
|
||||
return FormatJPEG, nil
|
||||
case "png":
|
||||
return FormatPNG, nil
|
||||
case "webp":
|
||||
return FormatWebP, nil
|
||||
case "avif":
|
||||
return FormatAVIF, nil
|
||||
case "jxl":
|
||||
return FormatJXL, nil
|
||||
case "gif":
|
||||
return FormatGIF, nil
|
||||
case "auto":
|
||||
return FormatAuto, nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: %s", ErrInvalidFormat, s)
|
||||
}
|
||||
}
|
||||
@@ -1,486 +0,0 @@
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Shared test fixture strings.
|
||||
const (
|
||||
testHostCDN = "cdn.example.com"
|
||||
testHostExample = "example.com"
|
||||
testPathCat = "/photos/cat.jpg"
|
||||
)
|
||||
|
||||
// assertRequest compares the source host, path and query and the size and
|
||||
// format of got against want.
|
||||
func assertRequest(t *testing.T, got, want *Request) {
|
||||
t.Helper()
|
||||
|
||||
if got.Source.Host != want.Source.Host {
|
||||
t.Errorf("Host = %q, want %q", got.Source.Host, want.Source.Host)
|
||||
}
|
||||
|
||||
if got.Source.Path != want.Source.Path {
|
||||
t.Errorf("Path = %q, want %q", got.Source.Path, want.Source.Path)
|
||||
}
|
||||
|
||||
if got.Source.Query != want.Source.Query {
|
||||
t.Errorf("Query = %q, want %q", got.Source.Query, want.Source.Query)
|
||||
}
|
||||
|
||||
if got.Transform.Size != want.Transform.Size {
|
||||
t.Errorf("Size = %v, want %v", got.Transform.Size, want.Transform.Size)
|
||||
}
|
||||
|
||||
if got.Transform.Format != want.Transform.Format {
|
||||
t.Errorf("Format = %q, want %q",
|
||||
got.Transform.Format, want.Transform.Format)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImageURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want *Request
|
||||
}{
|
||||
{
|
||||
name: "basic path with size",
|
||||
input: "/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat},
|
||||
Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "original size with 0x0",
|
||||
input: "/v1/image/cdn.example.com/photos/cat.jpg/0x0.jpeg",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat},
|
||||
Transform: Transform{Size: Size{Width: 0, Height: 0}, Format: FormatJPEG},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "original size with orig keyword",
|
||||
input: "/v1/image/cdn.example.com/photos/cat.jpg/orig.png",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat},
|
||||
Transform: Transform{Size: Size{Width: 0, Height: 0}, Format: FormatPNG},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "path with query string",
|
||||
input: "/v1/image/cdn.example.com/photos/cat.jpg?arg1=val1&arg2=val2/800x600.webp",
|
||||
want: &Request{
|
||||
Source: Source{
|
||||
Host: testHostCDN, Path: testPathCat, Query: "arg1=val1&arg2=val2",
|
||||
},
|
||||
Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "deep nested path",
|
||||
input: "/v1/image/cdn.example.com/a/b/c/d/image.jpg/1920x1080.avif",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostCDN, Path: "/a/b/c/d/image.jpg"},
|
||||
Transform: Transform{Size: Size{Width: 1920, Height: 1080}, Format: FormatAVIF},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "jpg alias for jpeg",
|
||||
input: "/v1/image/example.com/img.png/100x100.jpg",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostExample, Path: "/img.png"},
|
||||
Transform: Transform{Size: Size{Width: 100, Height: 100}, Format: FormatJPEG},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gif format",
|
||||
input: "/v1/image/example.com/animated.gif/200x200.gif",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostExample, Path: "/animated.gif"},
|
||||
Transform: Transform{Size: Size{Width: 200, Height: 200}, Format: FormatGIF},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImageURL(tt.input)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImageURL() unexpected error = %v", err)
|
||||
}
|
||||
|
||||
assertRequest(t, got, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImageURL_AutoFormat verifies that the format auto in an image URL
|
||||
// parses as FormatAuto.
|
||||
func TestParseImageURL_AutoFormat(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImageURL("/v1/image/example.com/photo.jpg/200x200.auto")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImageURL() error = %v", err)
|
||||
}
|
||||
|
||||
if got.Transform.Format != FormatAuto {
|
||||
t.Errorf("Format = %q, want %q", got.Transform.Format, FormatAuto)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImageURL_Errors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "missing prefix",
|
||||
input: "/image/cdn.example.com/photo.jpg/800x600.webp",
|
||||
wantErr: ErrInvalidPath,
|
||||
},
|
||||
{
|
||||
name: "empty path after prefix",
|
||||
input: "/v1/image/",
|
||||
wantErr: ErrMissingHost,
|
||||
},
|
||||
{
|
||||
name: "host only, no path or size",
|
||||
input: "/v1/image/cdn.example.com",
|
||||
wantErr: ErrMissingSize,
|
||||
},
|
||||
{
|
||||
name: "invalid size format",
|
||||
input: "/v1/image/cdn.example.com/photo.jpg/invalid.webp",
|
||||
wantErr: ErrInvalidSize,
|
||||
},
|
||||
{
|
||||
name: "unsupported format",
|
||||
input: "/v1/image/cdn.example.com/photo.jpg/800x600.bmp",
|
||||
wantErr: ErrInvalidFormat,
|
||||
},
|
||||
{
|
||||
name: "dimension too large",
|
||||
input: "/v1/image/cdn.example.com/photo.jpg/10000x600.webp",
|
||||
wantErr: ErrDimensionTooLarge,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := ParseImageURL(tt.input)
|
||||
if err == nil {
|
||||
t.Fatalf("ParseImageURL() error = nil, wantErr %v", tt.wantErr)
|
||||
}
|
||||
|
||||
if !errorIs(err, tt.wantErr) {
|
||||
t.Errorf("ParseImageURL() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImagePath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// ParseImagePath is for chi wildcard capture (no /v1/image/ prefix)
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want *Request
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "chi wildcard capture",
|
||||
input: "cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
want: &Request{
|
||||
Source: Source{
|
||||
Host: testHostCDN,
|
||||
Path: testPathCat,
|
||||
},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600},
|
||||
Format: FormatWebP,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with leading slash from chi",
|
||||
input: "/cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
want: &Request{
|
||||
Source: Source{
|
||||
Host: testHostCDN,
|
||||
Path: testPathCat,
|
||||
},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600},
|
||||
Format: FormatWebP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImagePath(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ParseImagePath() error = %v, wantErr %v", err, tt.wantErr)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assertRequest(t, got, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImagePath_LowerCasesHost verifies that the host is lower-cased, so
|
||||
// that every spelling of its letter case is the same host to the allowlist,
|
||||
// the upstream connection limits and the cache, and that the path keeps its
|
||||
// case.
|
||||
func TestParseImagePath_LowerCasesHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImagePath("CDN.Example.COM/Photos/Cat.jpg/800x600.webp")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImagePath() error = %v", err)
|
||||
}
|
||||
|
||||
t.Logf("parsed host %q, path %q", got.Source.Host, got.Source.Path)
|
||||
|
||||
if got.Source.Host != testHostCDN {
|
||||
t.Errorf("Host = %q, want %q", got.Source.Host, testHostCDN)
|
||||
}
|
||||
|
||||
if got.Source.Path != "/Photos/Cat.jpg" {
|
||||
t.Errorf("Path = %q, want %q", got.Source.Path, "/Photos/Cat.jpg")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImagePath_RefusesUserinfo verifies that a host holding "@" is
|
||||
// refused: a URL reads what comes before the "@" as a user name and password
|
||||
// and the rest as the host.
|
||||
func TestParseImagePath_RefusesUserinfo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, input := range []string{
|
||||
"user@cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
"user:password@cdn.example.com/photos/cat.jpg/800x600.webp",
|
||||
} {
|
||||
got, err := ParseImagePath(input)
|
||||
t.Logf("ParseImagePath(%q) = %+v, error = %v", input, got, err)
|
||||
|
||||
if !errors.Is(err, ErrInvalidHost) {
|
||||
t.Errorf("ParseImagePath(%q) error = %v, want %v",
|
||||
input, err, ErrInvalidHost)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImageURL_PathTraversal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// All path traversal attempts should be rejected
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
}{
|
||||
{
|
||||
name: "simple parent directory",
|
||||
input: "/v1/image/cdn.example.com/../etc/passwd/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "double parent directory",
|
||||
input: "/v1/image/cdn.example.com/../../etc/passwd/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "parent in middle of path",
|
||||
input: "/v1/image/cdn.example.com/photos/../../../etc/passwd/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "encoded parent directory",
|
||||
input: "/v1/image/cdn.example.com/photos/%2e%2e/secret/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "double encoded parent",
|
||||
input: "/v1/image/cdn.example.com/photos/%252e%252e/secret/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "backslash traversal",
|
||||
input: "/v1/image/cdn.example.com/photos/..\\secret/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "mixed slashes",
|
||||
input: "/v1/image/cdn.example.com/photos/../\\../secret/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "null byte injection",
|
||||
input: "/v1/image/cdn.example.com/photos/image.jpg%00.png/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "parent at start of path",
|
||||
input: "/v1/image/cdn.example.com/../800x600.jpeg",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := ParseImageURL(tt.input)
|
||||
if err == nil {
|
||||
t.Error("ParseImageURL() should reject path traversal attempts")
|
||||
}
|
||||
|
||||
if !errors.Is(err, ErrPathTraversal) {
|
||||
t.Errorf("ParseImageURL() error = %v, want ErrPathTraversal", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseImagePath_PathTraversal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test path traversal via ParseImagePath (chi wildcard)
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
}{
|
||||
{
|
||||
name: "parent directory in path",
|
||||
input: "cdn.example.com/photos/../secret/800x600.jpeg",
|
||||
},
|
||||
{
|
||||
name: "encoded traversal",
|
||||
input: "cdn.example.com/photos/%2e%2e/secret/800x600.jpeg",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := ParseImagePath(tt.input)
|
||||
if err == nil {
|
||||
t.Error("ParseImagePath() should reject path traversal attempts")
|
||||
}
|
||||
|
||||
if !errors.Is(err, ErrPathTraversal) {
|
||||
t.Errorf("ParseImagePath() error = %v, want ErrPathTraversal", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImageURL_SourceURLHoldingPercent25 verifies that a source path and
|
||||
// a source query holding %25, the percent-escape of a literal "%", are
|
||||
// accepted as they are, although decoding them once more leaves a "%" that
|
||||
// does not start a percent-escape.
|
||||
func TestParseImageURL_SourceURLHoldingPercent25(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want *Request
|
||||
}{
|
||||
{
|
||||
name: "path",
|
||||
input: "/v1/image/cdn.example.com/sale/50%25off.jpg/800x600.webp",
|
||||
want: &Request{
|
||||
Source: Source{Host: testHostCDN, Path: "/sale/50%25off.jpg"},
|
||||
Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "query",
|
||||
input: "/v1/image/cdn.example.com/photos/cat.jpg?discount=50%25/800x600.webp",
|
||||
want: &Request{
|
||||
Source: Source{
|
||||
Host: testHostCDN, Path: testPathCat, Query: "discount=50%25",
|
||||
},
|
||||
Transform: Transform{Size: Size{Width: 800, Height: 600}, Format: FormatWebP},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseImageURL(tt.input)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseImageURL() unexpected error = %v", err)
|
||||
}
|
||||
|
||||
assertRequest(t, got, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseImageURL_DoubleEncodedTraversalNextToPercent verifies that a
|
||||
// double-encoded ".." is still refused when the path also holds a "%" that
|
||||
// does not start a percent-escape, either as written or after one more
|
||||
// decoding.
|
||||
func TestParseImageURL_DoubleEncodedTraversalNextToPercent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, input := range []string{
|
||||
"/v1/image/cdn.example.com/photos/%252e%252e/secret/50%25off.jpg/800x600.jpeg",
|
||||
"/v1/image/cdn.example.com/photos/%252e%252e/secret/100%/800x600.jpeg",
|
||||
} {
|
||||
_, err := ParseImageURL(input)
|
||||
t.Logf("ParseImageURL(%q) error = %v", input, err)
|
||||
|
||||
if !errors.Is(err, ErrPathTraversal) {
|
||||
t.Errorf("ParseImageURL(%q) error = %v, want ErrPathTraversal",
|
||||
input, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// errorIs checks if err matches target (handles wrapped errors).
|
||||
func errorIs(err, target error) bool {
|
||||
if errors.Is(err, target) {
|
||||
return true
|
||||
}
|
||||
// Check if error message contains target message for wrapped errors
|
||||
if err != nil && target != nil {
|
||||
return contains(err.Error(), target.Error())
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsAt(s, substr))
|
||||
}
|
||||
|
||||
func containsAt(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package imagerequest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// VariantKey is a SHA256 hash identifying a specific image variant (hex-encoded).
|
||||
type VariantKey string
|
||||
|
||||
// CacheKey generates a unique key for a request variant.
|
||||
// Format: sha256(host:path:query:width:height:format:quality:fit_mode)
|
||||
// The path and query go through escapePercentAndColon, so that a ":" in
|
||||
// either cannot be taken for the one between them.
|
||||
func CacheKey(req *Request) VariantKey {
|
||||
data := fmt.Sprintf("%s:%s:%s:%d:%d:%s:%d:%s",
|
||||
req.Source.Host,
|
||||
escapePercentAndColon(req.Source.Path),
|
||||
escapePercentAndColon(req.Source.Query),
|
||||
req.Transform.Size.Width,
|
||||
req.Transform.Size.Height,
|
||||
req.Transform.Format,
|
||||
req.Transform.Quality,
|
||||
req.Transform.FitMode,
|
||||
)
|
||||
h := sha256.Sum256([]byte(data))
|
||||
|
||||
return VariantKey(hex.EncodeToString(h[:]))
|
||||
}
|
||||
|
||||
// escapePercentAndColon returns s with each "%" written as "%25" and each
|
||||
// ":" as "%3A", so that the result holds no ":".
|
||||
func escapePercentAndColon(s string) string {
|
||||
s = strings.ReplaceAll(s, "%", "%25")
|
||||
|
||||
return strings.ReplaceAll(s, ":", "%3A")
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package imagerequest
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCacheKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
req1 := &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600}, Format: FormatWebP,
|
||||
Quality: 85, FitMode: FitCover,
|
||||
},
|
||||
}
|
||||
|
||||
req2 := &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600}, Format: FormatWebP,
|
||||
Quality: 85, FitMode: FitCover,
|
||||
},
|
||||
}
|
||||
|
||||
// Same request should produce same key
|
||||
key1 := CacheKey(req1)
|
||||
key2 := CacheKey(req2)
|
||||
|
||||
if key1 != key2 {
|
||||
t.Errorf("CacheKey() not deterministic: %s vs %s", key1, key2)
|
||||
}
|
||||
|
||||
// Key should be 64 hex chars
|
||||
if len(string(key1)) != 64 {
|
||||
t.Errorf("CacheKey() length = %d, want 64", len(string(key1)))
|
||||
}
|
||||
|
||||
// Different size should produce different key
|
||||
req3 := &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 400, Height: 300}, Format: FormatWebP, // Different size
|
||||
Quality: 85, FitMode: FitCover,
|
||||
},
|
||||
}
|
||||
|
||||
key3 := CacheKey(req3)
|
||||
if key1 == key3 {
|
||||
t.Error("CacheKey() produced same key for different sizes")
|
||||
}
|
||||
|
||||
// Different format should produce different key
|
||||
req4 := &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600}, Format: FormatPNG, // Different format
|
||||
Quality: 85, FitMode: FitCover,
|
||||
},
|
||||
}
|
||||
|
||||
key4 := CacheKey(req4)
|
||||
if key1 == key4 {
|
||||
t.Error("CacheKey() produced same key for different formats")
|
||||
}
|
||||
|
||||
// Different quality should produce different key
|
||||
req5 := &Request{
|
||||
Source: Source{Host: testHostCDN, Path: testPathCat, Query: ""},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600}, Format: FormatWebP,
|
||||
Quality: 50, FitMode: FitCover, // Different quality
|
||||
},
|
||||
}
|
||||
|
||||
key5 := CacheKey(req5)
|
||||
if key1 == key5 {
|
||||
t.Error("CacheKey() produced same key for different quality")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheKey_ColonOrPercentInPathOrQuery checks that two different source
|
||||
// URLs whose path or query holds ":" or "%" get different keys.
|
||||
func TestCacheKey_ColonOrPercentInPathOrQuery(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
path1, query1 string
|
||||
path2, query2 string
|
||||
}{
|
||||
{
|
||||
name: "colon moved from path to query",
|
||||
path1: "/a:b", query1: "",
|
||||
path2: "/a", query2: "b:",
|
||||
},
|
||||
{
|
||||
name: "colon in path and its percent-encoding",
|
||||
path1: "/a:b", query1: "",
|
||||
path2: "/a%3Ab", query2: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key1 := CacheKey(&Request{
|
||||
Source: Source{
|
||||
Host: testHostCDN,
|
||||
Path: tt.path1,
|
||||
Query: tt.query1,
|
||||
},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600},
|
||||
Format: FormatWebP,
|
||||
Quality: 85,
|
||||
FitMode: FitCover,
|
||||
},
|
||||
})
|
||||
key2 := CacheKey(&Request{
|
||||
Source: Source{
|
||||
Host: testHostCDN,
|
||||
Path: tt.path2,
|
||||
Query: tt.query2,
|
||||
},
|
||||
Transform: Transform{
|
||||
Size: Size{Width: 800, Height: 600},
|
||||
Format: FormatWebP,
|
||||
Quality: 85,
|
||||
FitMode: FitCover,
|
||||
},
|
||||
})
|
||||
|
||||
t.Logf("CacheKey() of path %q query %q = %s", tt.path1, tt.query1, key1)
|
||||
t.Logf("CacheKey() of path %q query %q = %s", tt.path2, tt.query2, key2)
|
||||
|
||||
if key1 == key2 {
|
||||
t.Errorf("CacheKey() gave path %q query %q and path %q query %q "+
|
||||
"the same key", tt.path1, tt.query1, tt.path2, tt.query2)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user