diff --git a/.dockerignore b/.dockerignore
index 8fa00fc..d711d2c 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,8 +1,82 @@
-.git
-.gitignore
-.DS_Store
-.env*
+# .dockerignore does NOT use .gitignore semantics. Docker matches with
+# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
+# `/` and an unprefixed pattern is anchored at the context root. Every
+# depth-independent pattern therefore needs `**/`, or `config/.env` and
+# `certs/server.key` still ship while this file reads as solved. Only
+# genuinely root-anchored entries go unprefixed. Never transplant these
+# into .gitignore, where `**/` is wrong.
+#
+# Matching is case-sensitive, so secrets use character ranges rather
+# than an ALL-CAPS twin, which would still miss `Server.Key`.
+#
+# Extend with this repo's own host-built artifacts, written anchored:
+# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
+# deletes the package directory from the context.
+
+# .git is sent without its config. Without a VERSION build argument the
+# stage that compiles runs `git describe --tags --always` on .git, which
+# does not need .git/config; that file can hold a credential, such as a
+# password in a remote URL or the token the CI checkout step stores there.
+# Each submodule keeps a config with the same exposure in its git directory
+# under .git/modules/, nested again for a submodule's own submodules, or in
+# its own .git directory when it keeps one.
+# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
+# `deploy/config`, `config/lib`) loses its whole git directory, because
+# `**/.git/modules/**/config` also matches that segment's directory
+# under .git/modules/. Go's version stamping then fails the build;
+# nothing leaks. Name such a submodule without that segment:
+# `git submodule add --name`.
+**/.git/config
+**/.git/modules/**/config
+
+# Agent scratch: one full checkout of the repo per in-flight agent.
+# Anchored because it occurs once where agents run at the repo root.
+# KNOWN GAP: a repo running agents in subdirectories still ships
+# `services/api/.claude/` and must add its own anchored entry.
.claude
-node_modules
-bin/
-data/
+
+# Environment files. `*.env` covers bare `.env` and the `prod.env`
+# convention. Re-include a committed template with a negation if the
+# build needs one: `!docs/example.env`.
+**/*.[eE][nN][vV]
+**/.[eE][nN][vV].*
+**/.[eE][nN][vV][rR][cC]
+
+# Private keys and the bundles carrying them. Public certificates
+# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
+**/*.[pP][eE][mM]
+**/*.[kK][eE][yY]
+**/*.[pP]12
+**/*.[pP][fF][xX]
+**/[iI][dD]_[rR][sS][aA]
+**/[iI][dD]_[dD][sS][aA]
+**/[iI][dD]_[eE][cC][dD][sS][aA]
+**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
+**/[iI][dD]_[eE][dD]25519
+**/[iI][dD]_[eE][dD]25519_[sS][kK]
+
+# Dependencies: restored inside the image, never copied in.
+**/node_modules
+
+# OS metadata.
+**/.DS_Store
+**/Thumbs.db
+
+# Editor state: never a build input, and it churns COPY.
+**/*.swp
+**/*.swo
+**/*~
+**/*.bak
+**/.idea
+**/.vscode
+**/*.sublime-*
+
+# pixa's own entries. On the host, `make build` writes bin/pixad, and the
+# example config keeps its state directory in data/.
+/bin
+/data
+
+# Local config files, kept out of git because they can hold the signing key.
+**/[cC][oO][nN][fF][iI][gG].[yY][mM][lL]
+**/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL]
+**/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL]
diff --git a/.editorconfig b/.editorconfig
index 92ec261..de84fae 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -13,3 +13,6 @@ indent_style = tab
[*.go]
indent_style = tab
+
+# This repository's own sections, such as one for another language it
+# uses, go below this comment, and a re-vendor keeps them.
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..0e4c6f8
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,4 @@
+# Every PR adds an entry at the top of TODO.md's Completed Steps; union keeps
+# both sides instead of conflicting. Git never reports a conflict here: read
+# the merged entries after every merge or rebase.
+TODO.md merge=union
diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml
index decade9..2e96965 100644
--- a/.gitea/workflows/check.yml
+++ b/.gitea/workflows/check.yml
@@ -1,10 +1,20 @@
name: check
on: [push]
+# Free the shared runner: a new push cancels only the same branch's older run.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
jobs:
check:
runs-on: ubuntu-latest
+ # Free the shared runner from a hung build.
+ timeout-minutes: 20
steps:
# actions/checkout v4.2.2, 2026-02-22
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+ # script/cibuild needs no token, so none is left in .git/config.
+ with:
+ persist-credentials: false
+ # All history and tags, so git describe finds the version tag.
+ fetch-depth: 0
- run: script/cibuild
- - run: script/docker-smoke
diff --git a/.gitignore b/.gitignore
index 50fad0e..c8e02b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,15 +11,52 @@ Thumbs.db
.vscode/
*.sublime-*
-# Environment / secrets
-.env
-.env.*
-*.pem
-*.key
+# Agent scratch (worktrees of this repo, created and destroyed by
+# in-flight tooling). Unanchored: .gitignore patterns already match at
+# every depth, so no prefix is wanted here. This is not a .dockerignore
+# entry and must not be given a `**/` prefix on the way into one.
+.claude/
+
+# Node
+node_modules/
+
+# Secrets. Unanchored like every entry above, so each matches at every
+# depth. Matching is case-sensitive on Linux, so names use character
+# ranges rather than a lowercase form that misses `Server.Key`.
+
+# Environment files. `*.env` covers bare `.env` and the `prod.env`
+# convention. Only the templates `example.env` and `sample.env` are
+# re-included below. A repository that commits any other template adds
+# its own negation at the end of this file, for example `!.env.example`.
+*.[eE][nN][vV]
+.[eE][nN][vV].*
+.[eE][nN][vV][rR][cC]
+!example.env
+!sample.env
+
+# Private keys and the bundles carrying them.
+*.[pP][eE][mM]
+*.[kK][eE][yY]
+*.[pP]12
+*.[pP][fF][xX]
+[iI][dD]_[rR][sS][aA]
+[iI][dD]_[dD][sS][aA]
+[iI][dD]_[eE][cC][dD][sS][aA]
+[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
+[iI][dD]_[eE][dD]25519
+[iI][dD]_[eE][dD]25519_[sS][kK]
+
+# This repository's own entries, such as its build outputs, go below
+# this comment, and a re-vendor keeps them. Anchor a binary built at the
+# root: `/myapp`, never `myapp`, which also ignores `cmd/myapp/`.
+
+# Go
+*.log
+*.out
+*.test
# Dependencies
vendor/
-node_modules/
# Build output
/bin/
@@ -31,5 +68,5 @@ node_modules/
*.sqlite3
# Local dev configs
+config.yml
config.yaml
-config.dev.yml
diff --git a/.golangci.yml b/.golangci.yml
index 26b1610..32db9fe 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -10,14 +10,23 @@ run:
linters:
default: all
+ enable:
+ # Successor to the deprecated gomodguard. Named explicitly, rather than
+ # left to `default: all`, because it carries the module policy below.
+ - gomodguard_v2
disable:
# Genuinely incompatible with project patterns
- exhaustruct # Requires all struct fields
- - depguard # Dependency allow/block lists
+ - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
- godot # Requires comments to end with periods
- - wsl # Deprecated, replaced by wsl_v5
- wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go
+ # Deprecated: the warning is attached to the old name, so it is
+ # silenced by disabling that name, not by enabling the successor.
+ - wsl # Deprecated, replaced by wsl_v5
+ - gomodguard # Deprecated, replaced by gomodguard_v2
+ # Misses findings at random in v2.14.0; back once a pinned release fixes it
+ - canonicalheader
settings:
lll:
line-length: 88
@@ -28,6 +37,64 @@ linters:
max-complexity: 15
dupl:
threshold: 100
+ depguard:
+ # Test-support code must not be compiled into the shipped binary. A
+ # test-support package exists to hand a test privileges the program
+ # itself must never have, so a file that is not a test must not import
+ # one. Test files, and the files inside a package whose directory name
+ # ends in `test`, are where that code belongs, and are exempt.
+ #
+ # The deny list below is the one part of this file a repository is
+ # expected to extend, and the only part it may. depguard matches an
+ # import path against a list of prefixes, so it cannot be told "any path
+ # whose last segment ends in test"; a repository's own test-support
+ # packages have to be named here one at a time, by full import path,
+ # under a module path that differs from repository to repository. Add
+ # them; change nothing else.
+ rules:
+ test-support:
+ list-mode: lax
+ files:
+ - "$all"
+ - "!$test"
+ - "!**/*test/**"
+ deny:
+ - pkg: net/http/httptest
+ desc: >-
+ Test-support code belongs in test files and in packages whose
+ directory name ends in test, not in the shipped binary.
+ # Only decisions already recorded in the Go package defaults are
+ # listed here. Every entry matches the module path exactly.
+ gomodguard_v2:
+ blocked:
+ - module: github.com/rs/zerolog
+ recommendations:
+ - log/slog
+ reason: "Structured logging is stdlib log/slog."
+ # One entry per pre-fork module path, because the later releases
+ # are separate paths. A prefix match would be shorter but would
+ # also reach github.com/go-redis/redismock, the test double for
+ # the successor these entries recommend.
+ - module: github.com/go-redis/redis
+ recommendations:
+ - github.com/redis/go-redis/v9
+ reason: "Pre-fork module; use the maintained go-redis v9."
+ - module: github.com/go-redis/redis/v7
+ recommendations:
+ - github.com/redis/go-redis/v9
+ reason: "Pre-fork module; use the maintained go-redis v9."
+ - module: github.com/go-redis/redis/v8
+ recommendations:
+ - github.com/redis/go-redis/v9
+ reason: "Pre-fork module; use the maintained go-redis v9."
+ - module: github.com/sergi/go-diff
+ recommendations:
+ - github.com/aymanbagabas/go-udiff
+ reason: "No unified diff output; use go-udiff."
+ - module: github.com/hexops/gotextdiff
+ recommendations:
+ - github.com/aymanbagabas/go-udiff
+ reason: "Unmaintained fork; use go-udiff."
issues:
max-issues-per-linter: 0
diff --git a/.prettierignore b/.prettierignore
new file mode 100644
index 0000000..23d67fc
--- /dev/null
+++ b/.prettierignore
@@ -0,0 +1,2 @@
+node_modules/
+yarn.lock
diff --git a/.prettierrc b/.prettierrc
new file mode 100644
index 0000000..8af31cd
--- /dev/null
+++ b/.prettierrc
@@ -0,0 +1,4 @@
+{
+ "tabWidth": 4,
+ "proseWrap": "always"
+}
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..d50c321
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,76 @@
+# Repository Rules
+
+Last Updated 2026-10-27
+
+These rules MUST be followed at all times. This is very important.
+
+- Never use `git add -A`. Stage specific changes for a deliberate commit. A
+ commit holds one change. After each change, make a commit with a good one-line
+ summary.
+
+- NEVER modify the linter config without asking first.
+
+- NEVER modify a test to exclude special cases or otherwise make it pass without
+ asking first. In almost all cases, change the code, NOT the test. You need
+ explicit user approval to modify an existing test. If you think a test needs
+ to change, make your case and ask for permission to proceed. Then stop. A bug
+ fix may change an existing test without asking only when that test pins
+ exactly the behavior the filed bug calls wrong. The change must then move the
+ test to the fixed behavior and nothing else. Any other change to an existing
+ test needs explicit user approval. Writing NEW tests needs no approval.
+
+- When linting, assume the linter config is CORRECT. Treat each item the linter
+ reports as something that legitimately needs fixing in the code.
+
+- When running tests, use `make test`.
+
+- Before committing, run `make check`. It runs `make test`, `make lint` and
+ `make fmt-check`. Any issue it finds MUST be resolved before committing,
+ unless you are explicitly told otherwise.
+
+- When fixing a bug, write a failing test for the bug FIRST. Add appropriate
+ logging to the test, to make sure the test is written correctly. Commit that.
+ Then work on the fix until the test passes, without changing the test further.
+ Then commit the fix.
+
+- When adding a new feature, do the same: write a test first (test-driven
+ development, TDD). The test does not need to be very complex. Commit the test,
+ then commit the feature.
+
+- When adding a new feature, use a feature branch. The feature branch reaches
+ `next` through a PR that passes review. It is merged only when the feature is
+ completely finished and the code is up to standards. The code is up to
+ standards when it passes `make check`. The branch is deleted after the merge.
+ Only the user merges `next` into `main`.
+
+- Write `godoc` comments for all exported types and functions as you go along.
+
+- ALWAYS be consistent in naming. If you name something one way in one place,
+ use the EXACT SAME name in every other place.
+
+- Be descriptive and specific in naming. `al` is bad; `SourceHostAllowlist` is
+ good. `ConnsPerHost` is bad; `MaxConnectionsPerHost` is good.
+
+- This is not prototype or teaching code. It is designed for production. Any
+ security issue (such as denial of service) or other web vulnerability is a P1
+ bug. It must be added at the top of `TODO.md`.
+
+- Do not stub an implementation unless you are specifically instructed to. This
+ is production code, and we need working implementations.
+
+- NEVER silently fall back to a different setting when a user's parameter
+ explicitly specifies a value. For example, if a user requests `format=webp`
+ and WebP encoding is not supported, return an error. Do NOT silently output
+ PNG instead. If a user specifies `fit=invalid` and that fit mode does not
+ exist, return an error. Do NOT silently default to `cover`. A silent fallback
+ violates the principle of least surprise and masks bugs. Defaults are
+ acceptable only for OMITTED parameters, never for INVALID explicit values.
+
+- Avoid vendoring dependencies unless specifically instructed to. NEVER commit
+ the `vendor/` directory. NEVER commit compiled binaries. If such a directory
+ or file exists and `.gitignore` does not list it, add it to `.gitignore`.
+ Commit that `.gitignore` change. Keep the entire git repository, history
+ included, under 20 MiB. Go over 20 MiB only when you specifically must commit
+ a large file, such as a media file used as a test fixture. Only OUR source
+ code and its immediately supporting files, such as test examples, go into the
+ repository and its history.
diff --git a/CLAUDE.md b/CLAUDE.md
deleted file mode 100644
index c9041c8..0000000
--- a/CLAUDE.md
+++ /dev/null
@@ -1,76 +0,0 @@
-# Repository Rules
-
-Last Updated 2026-01-08
-
-These rules MUST be followed at all times, it is very important.
-
-* Never use `git add -A` - add specific changes to a deliberate commit. A
- commit should contain one change. After each change, make a commit with a
- good one-line summary.
-
-* NEVER modify the linter config without asking first.
-
-* NEVER modify tests to exclude special cases or otherwise get them to pass
- without asking first. In almost all cases, the code should be changed,
- NOT the tests. If you think the test needs to be changed, make your case
- for that and ask for permission to proceed, then stop. You need explicit
- user approval to modify existing tests. (You do not need user approval
- for writing NEW tests.)
-
-* When linting, assume the linter config is CORRECT, and that each item
- output by the linter is something that legitimately needs fixing in the
- code.
-
-* When running tests, use `make test`.
-
-* Before commits, run `make check`. This runs `make lint` and `make test`
- and `make check-fmt`. Any issues discovered MUST be resolved before
- committing unless explicitly told otherwise.
-
-* When fixing a bug, write a failing test for the bug FIRST. Add
- appropriate logging to the test to ensure it is written correctly. Commit
- that. Then go about fixing the bug until the test passes (without
- modifying the test further). Then commit that.
-
-* When adding a new feature, do the same - implement a test first (TDD). It
- doesn't have to be super complex. Commit the test, then commit the
- feature.
-
-* When adding a new feature, use a feature branch. When the feature is
- completely finished and the code is up to standards (passes `make check`)
- then and only then can the feature branch be merged into `main` and the
- branch deleted.
-
-* Write godoc documentation comments for all exported types and functions as
- you go along.
-
-* ALWAYS be consistent in naming. If you name something one thing in one
- place, name it the EXACT SAME THING in another place.
-
-* Be descriptive and specific in naming. `wl` is bad;
- `SourceHostWhitelist` is good. `ConnsPerHost` is bad;
- `MaxConnectionsPerHost` is good.
-
-* This is not prototype or teaching code - this is designed for production.
- Any security issues (such as denial of service) or other web
- vulnerabilities are P1 bugs and must be added to TODO.md at the top.
-
-* As this is production code, no stubbing of implementations unless
- specifically instructed. We need working implementations.
-
-* NEVER silently fall back to a different setting when a user's parameter
- explicitly specifies a value. If a user requests format=webp and WebP
- encoding is not supported, return an error - do NOT silently output PNG
- instead. If a user specifies fit=invalid and that fit mode doesn't exist,
- return an error - do NOT silently default to "cover". Silent fallbacks
- violate the principle of least surprise and mask bugs. The only acceptable
- defaults are for OMITTED parameters, never for INVALID explicit values.
-
-* Avoid vendoring deps unless specifically instructed to. NEVER commit
- the vendor directory, NEVER commit compiled binaries. If these
- directories or files exist, add them to .gitignore (and commit the
- .gitignore) if they are not already in there. Keep the entire git
- repository (with history) small - under 20MiB, unless you specifically
- must commit larger files (e.g. test fixture example media files). Only
- OUR source code and immediately supporting files (such as test examples)
- goes into the repo/history.
diff --git a/CONVENTIONS.md b/CONVENTIONS.md
deleted file mode 100644
index 6a62434..0000000
--- a/CONVENTIONS.md
+++ /dev/null
@@ -1,1267 +0,0 @@
-# Go HTTP Server Conventions
-
-This document defines the architectural patterns, design decisions, and conventions for building Go HTTP servers. All new projects must follow these standards.
-
-## Table of Contents
-
-1. [Required Libraries](#1-required-libraries)
-2. [Project Structure](#2-project-structure)
-3. [Dependency Injection (Uber fx)](#3-dependency-injection-uber-fx)
-4. [Server Architecture](#4-server-architecture)
-5. [Routing (go-chi)](#5-routing-go-chi)
-6. [Handler Conventions](#6-handler-conventions)
-7. [Middleware Conventions](#7-middleware-conventions)
-8. [Configuration (Viper)](#8-configuration-viper)
-9. [Logging (slog)](#9-logging-slog)
-10. [Database Wrapper](#10-database-wrapper)
-11. [Globals Package](#11-globals-package)
-12. [Static Assets & Templates](#12-static-assets--templates)
-13. [Health Check](#13-health-check)
-14. [External Integrations](#14-external-integrations)
-
----
-
-## 1. Required Libraries
-
-These libraries are **mandatory** for all new projects:
-
-| Purpose | Library | Import Path |
-| -------------------- | --------------- | ------------------------------------- |
-| Dependency Injection | Uber fx | `go.uber.org/fx` |
-| HTTP Router | go-chi | `github.com/go-chi/chi` |
-| Logging | slog (stdlib) | `log/slog` |
-| Configuration | Viper | `github.com/spf13/viper` |
-| Environment Loading | godotenv | `github.com/joho/godotenv/autoload` |
-| CORS | go-chi/cors | `github.com/go-chi/cors` |
-| Error Reporting | Sentry | `github.com/getsentry/sentry-go` |
-| Metrics | Prometheus | `github.com/prometheus/client_golang` |
-| Metrics Middleware | go-http-metrics | `github.com/slok/go-http-metrics` |
-| Basic Auth | basicauth-go | `github.com/99designs/basicauth-go` |
-
----
-
-## 2. Project Structure
-
-```
-project-root/
-├── cmd/
-│ └── {appname}/
-│ └── main.go # Entry point
-├── internal/
-│ ├── config/
-│ │ └── config.go # Configuration loading
-│ ├── database/
-│ │ └── database.go # Database wrapper
-│ ├── globals/
-│ │ └── globals.go # Build-time variables
-│ ├── handlers/
-│ │ ├── handlers.go # Base handler struct and helpers
-│ │ ├── index.go # Individual handlers...
-│ │ ├── healthcheck.go
-│ │ └── {feature}.go
-│ ├── healthcheck/
-│ │ └── healthcheck.go # Health check service
-│ ├── logger/
-│ │ └── logger.go # Logger setup
-│ ├── middleware/
-│ │ └── middleware.go # All middleware definitions
-│ └── server/
-│ ├── server.go # Server struct and lifecycle
-│ ├── http.go # HTTP server setup
-│ └── routes.go # Route definitions
-├── static/
-│ ├── static.go # Embed directive
-│ ├── css/
-│ └── js/
-├── templates/
-│ ├── templates.go # Embed and parse
-│ └── *.html
-├── go.mod
-├── go.sum
-├── Makefile
-└── Dockerfile
-```
-
-### Key Principles
-
-- **`cmd/{appname}/`**: Only the entry point. Minimal logic, just bootstrapping.
-- **`internal/`**: All application packages. Not importable by external projects.
-- **One package per concern**: config, database, handlers, middleware, etc.
-- **Flat handler files**: One file per handler or logical group of handlers.
-
----
-
-## 3. Dependency Injection (Uber fx)
-
-### Entry Point Pattern
-
-```go
-// cmd/httpd/main.go
-package main
-
-import (
- "yourproject/internal/config"
- "yourproject/internal/database"
- "yourproject/internal/globals"
- "yourproject/internal/handlers"
- "yourproject/internal/healthcheck"
- "yourproject/internal/logger"
- "yourproject/internal/middleware"
- "yourproject/internal/server"
- "go.uber.org/fx"
-)
-
-var (
- Appname string = "CHANGEME"
- Version string
- Buildarch string
-)
-
-func main() {
- globals.Appname = Appname
- globals.Version = Version
- globals.Buildarch = Buildarch
-
- fx.New(
- fx.Provide(
- config.New,
- database.New,
- globals.New,
- handlers.New,
- logger.New,
- server.New,
- middleware.New,
- healthcheck.New,
- ),
- fx.Invoke(func(*server.Server) {}),
- ).Run()
-}
-```
-
-### Params Struct Pattern
-
-Every component that receives dependencies uses a params struct with `fx.In`:
-
-```go
-type HandlersParams struct {
- fx.In
- Logger *logger.Logger
- Globals *globals.Globals
- Database *database.Database
- Healthcheck *healthcheck.Healthcheck
-}
-
-type Handlers struct {
- params *HandlersParams
- log *slog.Logger
- hc *healthcheck.Healthcheck
-}
-```
-
-### Factory Function Pattern
-
-All components expose a `New` function with this signature:
-
-```go
-func New(lc fx.Lifecycle, params SomeParams) (*Something, error) {
- s := &Something{
- log: params.Logger.Get(),
- config: params.Config,
- db: params.Database,
- }
-
- lc.Append(fx.Hook{
- OnStart: func(ctx context.Context) error {
- // Initialize resources
- return nil
- },
- OnStop: func(ctx context.Context) error {
- // Cleanup resources
- return nil
- },
- })
- return s, nil
-}
-```
-
-### Params vs Struct Fields
-
-The `Params` struct is **only for dependency injection in the constructor**. Dependencies should be copied to struct fields for use at runtime:
-
-```go
-// WRONG - accessing dependencies via params
-type Server struct {
- params ServerParams // Don't store params
-}
-
-func (s *Server) SomeMethod() {
- port := s.params.Config.Port // Bad: indirect access
-}
-
-// CORRECT - store dependencies directly on struct
-type Server struct {
- log *slog.Logger
- config *config.Config
- db *database.Database
-}
-
-func (s *Server) SomeMethod() {
- port := s.config.Port // Good: direct access
-}
-```
-
-This pattern:
-- Makes dependencies explicit in the struct definition
-- Avoids unnecessary indirection at runtime
-- Clearly separates DI concerns (params) from runtime state (struct fields)
-
-### Dependency Order
-
-Providers are resolved automatically by fx, but conceptually follow this order:
-
-1. `globals.New` - Build-time variables (no dependencies)
-2. `logger.New` - Logger (depends on Globals)
-3. `config.New` - Configuration (depends on Globals, Logger)
-4. `database.New` - Database (depends on Logger, Config)
-5. `healthcheck.New` - Health check (depends on Globals, Config, Logger, Database)
-6. `middleware.New` - Middleware (depends on Logger, Globals, Config)
-7. `handlers.New` - Handlers (depends on Logger, Globals, Database, Healthcheck)
-8. `server.New` - Server (depends on all above)
-
----
-
-## 4. Server Architecture
-
-### Server Struct
-
-The Server struct is the central orchestrator:
-
-```go
-// internal/server/server.go
-type ServerParams struct {
- fx.In
- Logger *logger.Logger
- Globals *globals.Globals
- Config *config.Config
- Middleware *middleware.Middleware
- Handlers *handlers.Handlers
-}
-
-type Server struct {
- startupTime time.Time
- port int
- exitCode int
- sentryEnabled bool
- log *slog.Logger
- ctx context.Context
- cancelFunc context.CancelFunc
- httpServer *http.Server
- router *chi.Mux
- params ServerParams
- mw *middleware.Middleware
- h *handlers.Handlers
-}
-```
-
-### Server Factory
-
-```go
-func New(lc fx.Lifecycle, params ServerParams) (*Server, error) {
- s := new(Server)
- s.params = params
- s.mw = params.Middleware
- s.h = params.Handlers
- s.log = params.Logger.Get()
-
- lc.Append(fx.Hook{
- OnStart: func(ctx context.Context) error {
- s.startupTime = time.Now()
- go s.Run()
- return nil
- },
- OnStop: func(ctx context.Context) error {
- // Server shutdown logic
- return nil
- },
- })
- return s, nil
-}
-```
-
-### HTTP Server Setup
-
-```go
-// internal/server/http.go
-func (s *Server) serveUntilShutdown() {
- listenAddr := fmt.Sprintf(":%d", s.params.Config.Port)
- s.httpServer = &http.Server{
- Addr: listenAddr,
- ReadTimeout: 10 * time.Second,
- WriteTimeout: 10 * time.Second,
- MaxHeaderBytes: 1 << 20,
- Handler: s,
- }
-
- s.SetupRoutes()
-
- s.log.Info("http begin listen", "listenaddr", listenAddr)
- if err := s.httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
- s.log.Error("listen error", "error", err)
- if s.cancelFunc != nil {
- s.cancelFunc()
- }
- }
-}
-
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- s.router.ServeHTTP(w, r)
-}
-```
-
-### Signal Handling and Graceful Shutdown
-
-```go
-func (s *Server) serve() int {
- s.ctx, s.cancelFunc = context.WithCancel(context.Background())
-
- // Signal watcher
- go func() {
- c := make(chan os.Signal, 1)
- signal.Ignore(syscall.SIGPIPE)
- signal.Notify(c, os.Interrupt, syscall.SIGTERM)
- sig := <-c
- s.log.Info("signal received", "signal", sig)
- if s.cancelFunc != nil {
- s.cancelFunc()
- }
- }()
-
- go s.serveUntilShutdown()
-
- for range s.ctx.Done() {
- }
- s.cleanShutdown()
- return s.exitCode
-}
-
-func (s *Server) cleanShutdown() {
- s.exitCode = 0
- ctxShutdown, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
- if err := s.httpServer.Shutdown(ctxShutdown); err != nil {
- s.log.Error("server clean shutdown failed", "error", err)
- }
- if shutdownCancel != nil {
- shutdownCancel()
- }
- s.cleanupForExit()
- if s.sentryEnabled {
- sentry.Flush(2 * time.Second)
- }
-}
-```
-
----
-
-## 5. Routing (go-chi)
-
-### Route Setup Pattern
-
-```go
-// internal/server/routes.go
-func (s *Server) SetupRoutes() {
- s.router = chi.NewRouter()
-
- // Global middleware (applied to all routes)
- s.router.Use(middleware.Recoverer)
- s.router.Use(middleware.RequestID)
- s.router.Use(s.mw.Logging())
-
- // Conditional middleware
- if viper.GetString("METRICS_USERNAME") != "" {
- s.router.Use(s.mw.Metrics())
- }
-
- s.router.Use(s.mw.CORS())
- s.router.Use(middleware.Timeout(60 * time.Second))
-
- if s.sentryEnabled {
- sentryHandler := sentryhttp.New(sentryhttp.Options{
- Repanic: true,
- })
- s.router.Use(sentryHandler.Handle)
- }
-
- // Routes
- s.router.Get("/", s.h.HandleIndex())
-
- // Static files
- s.router.Mount("/s", http.StripPrefix("/s", http.FileServer(http.FS(static.Static))))
-
- // API versioning
- s.router.Route("/api/v1", func(r chi.Router) {
- r.Get("/now", s.h.HandleNow())
- })
-
- // Routes with specific middleware
- auth := s.mw.Auth()
- s.router.Get("/login", auth(s.h.HandleLoginGET()).ServeHTTP)
-
- // Health check (standard path)
- s.router.Get("/.well-known/healthcheck.json", s.h.HandleHealthCheck())
-
- // Protected route groups
- if viper.GetString("METRICS_USERNAME") != "" {
- s.router.Group(func(r chi.Router) {
- r.Use(s.mw.MetricsAuth())
- r.Get("/metrics", http.HandlerFunc(promhttp.Handler().ServeHTTP))
- })
- }
-}
-```
-
-### Middleware Ordering (Critical)
-
-1. `middleware.Recoverer` - Panic recovery (must be first)
-2. `middleware.RequestID` - Generate request IDs
-3. `s.mw.Logging()` - Request logging
-4. `s.mw.Metrics()` - Prometheus metrics (if enabled)
-5. `s.mw.CORS()` - CORS headers
-6. `middleware.Timeout(60s)` - Request timeout
-7. `sentryhttp.Handler` - Sentry error reporting (if enabled)
-
-### API Versioning
-
-Use route groups for API versioning:
-
-```go
-s.router.Route("/api/v1", func(r chi.Router) {
- r.Get("/resource", s.h.HandleResource())
-})
-```
-
-### Static File Serving
-
-Static files are served at `/s/` prefix:
-
-```go
-s.router.Mount("/s", http.StripPrefix("/s", http.FileServer(http.FS(static.Static))))
-```
-
----
-
-## 6. Handler Conventions
-
-### Handler Base Struct
-
-```go
-// internal/handlers/handlers.go
-type HandlersParams struct {
- fx.In
- Logger *logger.Logger
- Globals *globals.Globals
- Database *database.Database
- Healthcheck *healthcheck.Healthcheck
-}
-
-type Handlers struct {
- params *HandlersParams
- log *slog.Logger
- hc *healthcheck.Healthcheck
-}
-
-func New(lc fx.Lifecycle, params HandlersParams) (*Handlers, error) {
- s := new(Handlers)
- s.params = ¶ms
- s.log = params.Logger.Get()
- s.hc = params.Healthcheck
- lc.Append(fx.Hook{
- OnStart: func(ctx context.Context) error {
- // Compile templates or other initialization
- return nil
- },
- })
- return s, nil
-}
-```
-
-### Closure-Based Handler Pattern
-
-For JSON route handlers, both the request and the response structures are
-defined in the scope of the method that returns the HandlerFunc. They can
-be called simply `Request` and `Response` or slightly more descriptive
-names.
-
-All handlers return `http.HandlerFunc` using the closure pattern. This allows initialization logic to run once when the handler is created:
-
-```go
-// internal/handlers/index.go
-func (s *Handlers) HandleIndex() http.HandlerFunc {
- // Initialization runs once
- t := templates.GetParsed()
-
- // Handler runs per-request
- return func(w http.ResponseWriter, r *http.Request) {
- err := t.ExecuteTemplate(w, "index.html", nil)
- if err != nil {
- s.log.Error("template execution failed", "error", err)
- http.Error(w, http.StatusText(500), 500)
- }
- }
-}
-```
-
-### JSON Handler Pattern
-
-```go
-// internal/handlers/now.go
-func (s *Handlers) HandleNow() http.HandlerFunc {
-
- type request struct {
- // request format
- }
-
- // Response struct defined in closure scope
- type response struct {
- Now time.Time `json:"now"`
- }
- return func(w http.ResponseWriter, r *http.Request) {
- s.respondJSON(w, r, &response{Now: time.Now()}, 200)
- }
-}
-```
-
-### Response Helpers
-
-```go
-// internal/handlers/handlers.go
-func (s *Handlers) respondJSON(w http.ResponseWriter, r *http.Request, data interface{}, status int) {
- w.WriteHeader(status)
- w.Header().Set("Content-Type", "application/json")
- if data != nil {
- err := json.NewEncoder(w).Encode(data)
- if err != nil {
- s.log.Error("json encode error", "error", err)
- }
- }
-}
-
-func (s *Handlers) decodeJSON(w http.ResponseWriter, r *http.Request, v interface{}) error {
- return json.NewDecoder(r.Body).Decode(v)
-}
-```
-
-### Handler Naming Convention
-
-- `HandleIndex()` - Main page
-- `HandleLoginGET()` / `HandleLoginPOST()` - Form handlers with HTTP method suffix
-- `HandleNow()` - API endpoints
-- `HandleHealthCheck()` - System endpoints
-
----
-
-## 7. Middleware Conventions
-
-### Middleware Struct
-
-```go
-// internal/middleware/middleware.go
-type MiddlewareParams struct {
- fx.In
- Logger *logger.Logger
- Globals *globals.Globals
- Config *config.Config
-}
-
-type Middleware struct {
- log *slog.Logger
- params *MiddlewareParams
-}
-
-func New(lc fx.Lifecycle, params MiddlewareParams) (*Middleware, error) {
- s := new(Middleware)
- s.params = ¶ms
- s.log = params.Logger.Get()
- return s, nil
-}
-```
-
-### Middleware Signature
-
-All custom middleware methods return `func(http.Handler) http.Handler`:
-
-```go
-func (s *Middleware) Auth() func(http.Handler) http.Handler {
- return func(next http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- // Before request
- s.log.Info("AUTH: before request")
-
- next.ServeHTTP(w, r)
-
- // After request (optional)
- })
- }
-}
-```
-
-### Logging Middleware with Status Capture
-
-```go
-type loggingResponseWriter struct {
- http.ResponseWriter
- statusCode int
-}
-
-func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
- return &loggingResponseWriter{w, http.StatusOK}
-}
-
-func (lrw *loggingResponseWriter) WriteHeader(code int) {
- lrw.statusCode = code
- lrw.ResponseWriter.WriteHeader(code)
-}
-
-func (s *Middleware) Logging() func(http.Handler) http.Handler {
- return func(next http.Handler) http.Handler {
- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- start := time.Now()
- lrw := NewLoggingResponseWriter(w)
- ctx := r.Context()
- defer func() {
- latency := time.Since(start)
- s.log.InfoContext(ctx, "request",
- "request_start", start,
- "method", r.Method,
- "url", r.URL.String(),
- "useragent", r.UserAgent(),
- "request_id", ctx.Value(middleware.RequestIDKey).(string),
- "referer", r.Referer(),
- "proto", r.Proto,
- "remoteIP", ipFromHostPort(r.RemoteAddr),
- "status", lrw.statusCode,
- "latency_ms", latency.Milliseconds(),
- )
- }()
- next.ServeHTTP(lrw, r)
- })
- }
-}
-```
-
-### CORS Middleware
-
-```go
-func (s *Middleware) CORS() func(http.Handler) http.Handler {
- return cors.Handler(cors.Options{
- AllowedOrigins: []string{"*"},
- AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
- AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
- ExposedHeaders: []string{"Link"},
- AllowCredentials: false,
- MaxAge: 300,
- })
-}
-```
-
-### Metrics Middleware
-
-```go
-func (s *Middleware) Metrics() func(http.Handler) http.Handler {
- mdlw := ghmm.New(ghmm.Config{
- Recorder: metrics.NewRecorder(metrics.Config{}),
- })
- return func(next http.Handler) http.Handler {
- return std.Handler("", mdlw, next)
- }
-}
-
-func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
- return basicauth.New(
- "metrics",
- map[string][]string{
- viper.GetString("METRICS_USERNAME"): {
- viper.GetString("METRICS_PASSWORD"),
- },
- },
- )
-}
-```
-
----
-
-## 8. Configuration (Viper)
-
-### Config Struct
-
-```go
-// internal/config/config.go
-type ConfigParams struct {
- fx.In
- Globals *globals.Globals
- Logger *logger.Logger
-}
-
-type Config struct {
- DBURL string
- Debug bool
- MaintenanceMode bool
- MetricsPassword string
- MetricsUsername string
- Port int
- SentryDSN string
- params *ConfigParams
- log *slog.Logger
-}
-```
-
-### Configuration Loading
-
-```go
-func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
- log := params.Logger.Get()
- name := params.Globals.Appname
-
- // Config file settings
- viper.SetConfigName(name)
- viper.SetConfigType("yaml")
- viper.AddConfigPath(fmt.Sprintf("/etc/%s", name))
- viper.AddConfigPath(fmt.Sprintf("$HOME/.config/%s", name))
-
- // Environment variables override everything
- viper.AutomaticEnv()
-
- // Defaults
- viper.SetDefault("DEBUG", "false")
- viper.SetDefault("MAINTENANCE_MODE", "false")
- viper.SetDefault("PORT", "8080")
- viper.SetDefault("DBURL", "")
- viper.SetDefault("SENTRY_DSN", "")
- viper.SetDefault("METRICS_USERNAME", "")
- viper.SetDefault("METRICS_PASSWORD", "")
-
- // Read config file (optional)
- if err := viper.ReadInConfig(); err != nil {
- if _, ok := err.(viper.ConfigFileNotFoundError); ok {
- // Config file not found is OK
- } else {
- log.Error("config file malformed", "error", err)
- panic(err)
- }
- }
-
- // Build config struct
- s := &Config{
- DBURL: viper.GetString("DBURL"),
- Debug: viper.GetBool("debug"),
- Port: viper.GetInt("PORT"),
- SentryDSN: viper.GetString("SENTRY_DSN"),
- MaintenanceMode: viper.GetBool("MAINTENANCE_MODE"),
- MetricsUsername: viper.GetString("METRICS_USERNAME"),
- MetricsPassword: viper.GetString("METRICS_PASSWORD"),
- log: log,
- params: ¶ms,
- }
-
- // Enable debug logging if configured
- if s.Debug {
- params.Logger.EnableDebugLogging()
- s.log = params.Logger.Get()
- }
-
- return s, nil
-}
-```
-
-### Configuration Precedence
-
-1. **Environment variables** (highest priority via `AutomaticEnv()`)
-2. **`.env` file** (loaded via `godotenv/autoload` import)
-3. **Config files**: `/etc/{appname}/{appname}.yaml`, `~/.config/{appname}/{appname}.yaml`
-4. **Defaults** (lowest priority)
-
-### Environment Loading
-
-Import godotenv with autoload to automatically load `.env` files:
-
-```go
-import (
- _ "github.com/joho/godotenv/autoload"
-)
-```
-
----
-
-## 9. Logging (slog)
-
-### Logger Struct
-
-```go
-// internal/logger/logger.go
-type LoggerParams struct {
- fx.In
- Globals *globals.Globals
-}
-
-type Logger struct {
- log *slog.Logger
- level *slog.LevelVar
- params LoggerParams
-}
-```
-
-### Logger Setup with TTY Detection
-
-```go
-func New(lc fx.Lifecycle, params LoggerParams) (*Logger, error) {
- l := new(Logger)
- l.level = new(slog.LevelVar)
- l.level.Set(slog.LevelInfo)
-
- // TTY detection for dev vs prod output
- tty := false
- if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) != 0 {
- tty = true
- }
-
- var handler slog.Handler
- if tty {
- // Text output for development
- handler = slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
- Level: l.level,
- AddSource: true,
- })
- } else {
- // JSON output for production
- handler = slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
- Level: l.level,
- AddSource: true,
- })
- }
-
- l.log = slog.New(handler)
- return l, nil
-}
-```
-
-### Logger Methods
-
-```go
-func (l *Logger) EnableDebugLogging() {
- l.level.Set(slog.LevelDebug)
- l.log.Debug("debug logging enabled", "debug", true)
-}
-
-func (l *Logger) Get() *slog.Logger {
- return l.log
-}
-
-func (l *Logger) Identify() {
- l.log.Info("starting",
- "appname", l.params.Globals.Appname,
- "version", l.params.Globals.Version,
- "buildarch", l.params.Globals.Buildarch,
- )
-}
-```
-
-### Logging Patterns
-
-```go
-// Info with fields
-s.log.Info("message", "key", "value")
-
-// Error with error object
-s.log.Error("operation failed", "error", err)
-
-// With context
-s.log.InfoContext(ctx, "processing request", "request_id", reqID)
-
-// Structured request logging
-s.log.Info("request completed",
- "request_start", start,
- "method", r.Method,
- "url", r.URL.String(),
- "status", statusCode,
- "latency_ms", latency.Milliseconds(),
-)
-
-// Using slog.Group for nested attributes
-s.log.Info("request",
- slog.Group("http",
- "method", r.Method,
- "url", r.URL.String(),
- ),
- slog.Group("timing",
- "start", start,
- "latency_ms", latency.Milliseconds(),
- ),
-)
-```
-
----
-
-## 10. Database Wrapper
-
-### Database Struct
-
-```go
-// internal/database/database.go
-type DatabaseParams struct {
- fx.In
- Logger *logger.Logger
- Config *config.Config
-}
-
-type Database struct {
- URL string
- log *slog.Logger
- params *DatabaseParams
-}
-```
-
-### Database Factory with Lifecycle
-
-```go
-func New(lc fx.Lifecycle, params DatabaseParams) (*Database, error) {
- s := new(Database)
- s.params = ¶ms
- s.log = params.Logger.Get()
-
- s.log.Info("Database instantiated")
-
- lc.Append(fx.Hook{
- OnStart: func(ctx context.Context) error {
- s.log.Info("Database OnStart Hook")
- // Connect to database here
- // Example: s.db, err = sql.Open("postgres", s.params.Config.DBURL)
- return nil
- },
- OnStop: func(ctx context.Context) error {
- // Disconnect from database here
- // Example: s.db.Close()
- return nil
- },
- })
- return s, nil
-}
-```
-
-### Usage Pattern
-
-The Database struct is injected into handlers and other services:
-
-```go
-type HandlersParams struct {
- fx.In
- Database *database.Database
- // ...
-}
-
-// Access in handler
-func (s *Handlers) HandleSomething() http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- // Use s.params.Database
- }
-}
-```
-
----
-
-## 11. Globals Package
-
-### Package Variables and Struct
-
-```go
-// internal/globals/globals.go
-package globals
-
-import "go.uber.org/fx"
-
-// Package-level variables (set from main)
-var (
- Appname string
- Version string
- Buildarch string
-)
-
-// Struct for DI
-type Globals struct {
- Appname string
- Version string
- Buildarch string
-}
-
-func New(lc fx.Lifecycle) (*Globals, error) {
- n := &Globals{
- Appname: Appname,
- Buildarch: Buildarch,
- Version: Version,
- }
- return n, nil
-}
-```
-
-### Setting Globals in Main
-
-```go
-// cmd/httpd/main.go
-var (
- Appname string = "CHANGEME" // Default, overridden by build
- Version string // Set at build time
- Buildarch string // Set at build time
-)
-
-func main() {
- globals.Appname = Appname
- globals.Version = Version
- globals.Buildarch = Buildarch
- // ...
-}
-```
-
-### Build-Time Variable Injection
-
-Use ldflags to inject version information at build time:
-
-```makefile
-VERSION := $(shell git describe --tags --always)
-BUILDARCH := $(shell go env GOARCH)
-
-build:
- go build -ldflags "-X main.Version=$(VERSION) -X main.Buildarch=$(BUILDARCH)" ./cmd/httpd
-```
-
----
-
-## 12. Static Assets & Templates
-
-### Static File Embedding
-
-```go
-// static/static.go
-package static
-
-import "embed"
-
-//go:embed css js
-var Static embed.FS
-```
-
-Directory structure:
-
-```
-static/
-├── static.go
-├── css/
-│ ├── bootstrap-4.5.3.min.css
-│ └── style.css
-└── js/
- ├── bootstrap-4.5.3.bundle.min.js
- └── jquery-3.5.1.slim.min.js
-```
-
-### Template Embedding and Lazy Parsing
-
-```go
-// templates/templates.go
-package templates
-
-import (
- "embed"
- "text/template"
-)
-
-//go:embed *.html
-var TemplatesRaw embed.FS
-var TemplatesParsed *template.Template
-
-func GetParsed() *template.Template {
- if TemplatesParsed == nil {
- TemplatesParsed = template.Must(template.ParseFS(TemplatesRaw, "*"))
- }
- return TemplatesParsed
-}
-```
-
-### Template Composition
-
-Templates use Go's template composition:
-
-```html
-
-{{ template "htmlheader.html" . }} {{ template "navbar.html" . }}
-
-
-
-
-
-{{ template "pagefooter.html" . }} {{ template "htmlfooter.html" . }}
-```
-
-### Static Asset References
-
-Reference static files with `/s/` prefix:
-
-```html
-
-
-
-```
-
----
-
-## 13. Health Check
-
-### Health Check Service
-
-```go
-// internal/healthcheck/healthcheck.go
-type HealthcheckParams struct {
- fx.In
- Globals *globals.Globals
- Config *config.Config
- Logger *logger.Logger
- Database *database.Database
-}
-
-type Healthcheck struct {
- StartupTime time.Time
- log *slog.Logger
- params *HealthcheckParams
-}
-
-func New(lc fx.Lifecycle, params HealthcheckParams) (*Healthcheck, error) {
- s := new(Healthcheck)
- s.params = ¶ms
- s.log = params.Logger.Get()
-
- lc.Append(fx.Hook{
- OnStart: func(ctx context.Context) error {
- s.StartupTime = time.Now()
- return nil
- },
- OnStop: func(ctx context.Context) error {
- return nil
- },
- })
- return s, nil
-}
-```
-
-### Health Check Response
-
-```go
-type HealthcheckResponse struct {
- Status string `json:"status"`
- Now string `json:"now"`
- UptimeSeconds int64 `json:"uptime_seconds"`
- UptimeHuman string `json:"uptime_human"`
- Version string `json:"version"`
- Appname string `json:"appname"`
- Maintenance bool `json:"maintenance_mode"`
-}
-
-func (s *Healthcheck) uptime() time.Duration {
- return time.Since(s.StartupTime)
-}
-
-func (s *Healthcheck) Healthcheck() *HealthcheckResponse {
- resp := &HealthcheckResponse{
- Status: "ok",
- Now: time.Now().UTC().Format(time.RFC3339Nano),
- UptimeSeconds: int64(s.uptime().Seconds()),
- UptimeHuman: s.uptime().String(),
- Appname: s.params.Globals.Appname,
- Version: s.params.Globals.Version,
- }
- return resp
-}
-```
-
-### Standard Endpoint
-
-Health check is served at the standard `.well-known` path:
-
-```go
-s.router.Get("/.well-known/healthcheck.json", s.h.HandleHealthCheck())
-```
-
----
-
-## 14. External Integrations
-
-### Sentry Error Reporting
-
-Sentry is conditionally enabled based on `SENTRY_DSN` environment variable:
-
-```go
-func (s *Server) enableSentry() {
- s.sentryEnabled = false
-
- if s.params.Config.SentryDSN == "" {
- return
- }
-
- err := sentry.Init(sentry.ClientOptions{
- Dsn: s.params.Config.SentryDSN,
- Release: fmt.Sprintf("%s-%s", s.params.Globals.Appname, s.params.Globals.Version),
- })
- if err != nil {
- s.log.Error("sentry init failure", "error", err)
- os.Exit(1)
- return
- }
- s.log.Info("sentry error reporting activated")
- s.sentryEnabled = true
-}
-```
-
-Sentry middleware with repanic (bubbles panics to chi's Recoverer):
-
-```go
-if s.sentryEnabled {
- sentryHandler := sentryhttp.New(sentryhttp.Options{
- Repanic: true,
- })
- s.router.Use(sentryHandler.Handle)
-}
-```
-
-Flush Sentry on shutdown:
-
-```go
-if s.sentryEnabled {
- sentry.Flush(2 * time.Second)
-}
-```
-
-### Prometheus Metrics
-
-Metrics are conditionally enabled and protected by basic auth:
-
-```go
-// Only enable if credentials are configured
-if viper.GetString("METRICS_USERNAME") != "" {
- s.router.Use(s.mw.Metrics())
-}
-
-// Protected /metrics endpoint
-if viper.GetString("METRICS_USERNAME") != "" {
- s.router.Group(func(r chi.Router) {
- r.Use(s.mw.MetricsAuth())
- r.Get("/metrics", http.HandlerFunc(promhttp.Handler().ServeHTTP))
- })
-}
-```
-
-### Environment Variables Summary
-
-| Variable | Description | Default |
-| ------------------ | -------------------------------- | ------- |
-| `PORT` | HTTP listen port | 8080 |
-| `DEBUG` | Enable debug logging | false |
-| `DBURL` | Database connection URL | "" |
-| `SENTRY_DSN` | Sentry DSN for error reporting | "" |
-| `MAINTENANCE_MODE` | Enable maintenance mode | false |
-| `METRICS_USERNAME` | Basic auth username for /metrics | "" |
-| `METRICS_PASSWORD` | Basic auth password for /metrics | "" |
diff --git a/Dockerfile b/Dockerfile
index 77510a0..9cc15d5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,75 +1,138 @@
-# Lint stage
-# Same image as Dockerfile.lint: change both pins together.
-# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
-FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
+# In every stage ARG CACHEBUST comes right after the steps that install
+# system packages and download Go's modules, or right after FROM in a
+# stage with neither, and every step that lints, tests or compiles
+# comes after it. script/ passes a new value on every build, which
+# reruns every RUN step after it. Docker keeps the layers above it
+# until the base image, their own lines or go.mod and go.sum change.
+# The steps that compile mount Go's build cache, and golangci-lint run
+# also mounts golangci-lint's; both survive from build to build.
-RUN apk add --no-cache make build-base vips-dev libheif-dev pkgconfig
+# Lint phase. script/lint builds it alone. The linter is run directly:
+# `make lint` and script/lint are themselves a docker build.
+# golangci/golangci-lint:v2.14.0, 2026-10-15
+FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
+
+# The linter compiles every package, and govips needs the libvips
+# headers for that. REPO_POLICIES.md has the lint phase install them
+# itself; this image is Debian, so with apt-get rather than apk.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends libvips-dev \
+ && rm -rf /var/lib/apt/lists/*
WORKDIR /src
-
-# Copy go mod files first for better layer caching
COPY go.mod go.sum ./
RUN go mod download
-
-# Copy source code
+ARG CACHEBUST
COPY . .
+RUN --mount=type=cache,target=/root/.cache/go-build \
+ --mount=type=cache,target=/root/.cache/golangci-lint \
+ golangci-lint run --config .golangci.yml ./...
-# Tells script/lint it is inside a container, so it runs the linter.
-ENV container=docker
+# Test phase. script/test builds it alone. -race needs cgo and so a C
+# compiler, which the Debian Go image ships and the alpine one does not.
+# golang:1.25.4-trixie, 2026-10-15
+FROM golang@sha256:a02d35efc036053fdf0da8c15919276bf777a80cbfda6a35c5e9f087e652adfc AS test
-# Run formatting check and linter
-RUN make fmt-check
-RUN make lint
-
-# Build stage
-# golang:1.25.4-alpine, 2026-02-25
-FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
-
-# Depend on lint stage passing
-COPY --from=lint /src/go.sum /dev/null
-
-ARG VERSION=dev
-
-# Install build dependencies for CGO image libraries
-RUN apk add --no-cache \
- build-base \
- vips-dev \
- libheif-dev \
- pkgconfig
+# govips compiles against the libvips and libheif headers, and the tests
+# need libvips' JPEG XL support; libvips-dev brings all three. The tests
+# also encode AVIF, which needs libheif's AVIF encoder,
+# libheif-plugin-aomenc: Debian only recommends it, so it is named here.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends libvips-dev libheif-plugin-aomenc \
+ && rm -rf /var/lib/apt/lists/*
WORKDIR /src
-
-# Copy go mod files first for better layer caching
COPY go.mod go.sum ./
-RUN GOTOOLCHAIN=auto go mod download
+RUN go mod download
+ARG CACHEBUST
+COPY . .
+
+# Without -v first; on a failure, again with -v for the details, and
+# the step fails even if the second run passes. -count=1 because the
+# mounted cache also holds test results, which go test would otherwise
+# replay. -parallel 4: by default a package runs as many of its tests at
+# once as the host has CPUs, and on a busy host they then wait so long
+# to be scheduled that a test that times a request can see it return a
+# second late.
+RUN --mount=type=cache,target=/root/.cache/go-build \
+ go test -count=1 -timeout 90s -race -parallel 4 -cover ./... || \
+ { echo "--- Rerunning with -v for details ---"; \
+ go test -count=1 -timeout 90s -race -parallel 4 -v ./...; exit 1; }
+
+# Build stage. Nothing is wanted from the two phases above: these copies
+# make BuildKit build them first, so nothing after them runs unless lint
+# and test passed.
+# golang:1.25.4-alpine3.22, 2026-02-25; the runtime stage uses Alpine 3.22 too
+FROM golang:1.25.4-alpine3.22@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
+
+# git for the version, and for govips a C compiler, pkg-config and the
+# libvips and libheif headers
+RUN apk add --no-cache git build-base pkgconfig vips-dev libheif-dev
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+ARG CACHEBUST
+
+COPY --from=lint /src/go.sum /dev/null
+COPY --from=test /src/go.sum /dev/null
+
+# A tar-stream context keeps the sender's file owners, which git refuses.
+RUN git config --system --add safe.directory /src
# Copy source code
COPY . .
-# Run tests
-RUN make test
+# The VERSION build arg when one is given, otherwise
+# `git describe --tags --always` on the .git in the build context: the
+# tag on a tagged commit, tag-N-gHASH after one, the short commit when
+# no tag is reachable. With .git present, a version that is still empty,
+# dev or unknown fails the build: git is missing or could not read the
+# checkout. A context without .git, as from a source tarball, stamps an
+# empty version. CGO stays enabled for govips; -trimpath keeps build
+# paths out of the binary, and -s -w leave out the symbol table and
+# debug information.
+ARG VERSION
+RUN --mount=type=cache,target=/root/.cache/go-build \
+ VERSION="${VERSION:-$(git describe --tags --always)}"; \
+ if [ -e .git ]; then \
+ case "$VERSION" in ""|dev|unknown) \
+ echo "version is '$VERSION' although .git is present" >&2; \
+ exit 1 ;; \
+ esac; \
+ fi; \
+ CGO_ENABLED=1 GOTOOLCHAIN=auto go build -trimpath \
+ -ldflags="-s -w -X main.Version=${VERSION}" \
+ -o /pixad ./cmd/pixad
-# Build with CGO enabled
-RUN CGO_ENABLED=1 GOTOOLCHAIN=auto go build -ldflags "-X main.Version=${VERSION}" -o /pixad ./cmd/pixad
+# Runtime stage, and the last one: a plain `docker build .` builds this
+# stage and what it depends on, and nothing else. It must use the Alpine
+# release the golang image above is based on, so that pixad runs against
+# the libvips and musl it was built with.
+# alpine:3.22, 2026-10-08
+FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8
-# Runtime stage
-# alpine:3.21, 2026-02-25
-FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
-
-# Install runtime dependencies only
+# Install runtime dependencies only. vips-jxl is libvips' JPEG XL
+# support, without which pixad does not start.
RUN apk add --no-cache \
vips \
+ vips-jxl \
libheif \
ca-certificates \
tzdata \
su-exec
+ARG CACHEBUST
# Copy binary from builder
COPY --from=builder /pixad /usr/local/bin/pixad
COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
-# Create non-root user, config directory, and data directory
-RUN adduser -D -H -s /sbin/nologin pixad && \
+# Create non-root user, config directory, and data directory. pixad
+# gets uid and gid 65532, which host login and system accounts do not
+# use: a bind-mounted /var/lib/pixa is given to pixad, and on the host
+# it must not belong to a person's account.
+RUN addgroup -g 65532 pixad && \
+ adduser -D -H -s /sbin/nologin -u 65532 -G pixad pixad && \
mkdir -p /var/lib/pixa /etc/pixa && \
chown pixad:pixad /var/lib/pixa
diff --git a/Dockerfile.lint b/Dockerfile.lint
deleted file mode 100644
index a07d8ca..0000000
--- a/Dockerfile.lint
+++ /dev/null
@@ -1,34 +0,0 @@
-# Dockerfile.lint: the container script/lint builds to run golangci-lint,
-# which is never installed on the host. Pinned to the same image as the
-# Dockerfile lint stage: change both pins together, or the two run
-# different linter versions.
-#
-# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
-FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60
-
-# pixa is CGO/libvips: the type-aware linters compile every package, so
-# this image needs the same C libraries the build does.
-RUN apk add --no-cache build-base vips-dev libheif-dev pkgconfig
-
-WORKDIR /src
-
-# Modules first for layer caching; go.mod/go.sum settle this layer's
-# result, so it may safely be reused between runs.
-COPY go.mod go.sum ./
-RUN go mod download
-
-COPY . .
-
-# Tells script/lint it is inside a container, so it runs the linter.
-ENV container=docker
-
-# script/lint passes a different CACHEBUST on every run, and BuildKit
-# keys every RUN after this ARG on its value, so the lint step always
-# runs instead of returning a cached success that linted nothing.
-#
-# Go's and golangci-lint's caches (/root/.cache, hundreds of MB) go on a
-# tmpfs that is discarded after the step. Written into the layer, they
-# would pile up as build cache on every run, since no later run, with
-# its new CACHEBUST, can reuse that layer.
-ARG CACHEBUST
-RUN --mount=type=tmpfs,target=/root/.cache script/lint
diff --git a/Makefile b/Makefile
index 708d5e5..a074315 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,10 @@
-.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks
+.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest bench
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
LDFLAGS := -X main.Version=$(VERSION)
# Use nix-shell to provide CGO dependencies unless they are already available
-# (e.g. inside a Docker build or an existing nix-shell).
+# (e.g. inside an existing nix-shell).
HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null)
ifdef HAS_PKGCONFIG
NIX_RUN_PREFIX =
@@ -32,11 +32,11 @@ fmt-check:
fmt:
@script/fmt
-# Run linter
+# Run linter (the lint phase of the Dockerfile)
lint:
@script/lint
-# Run tests (30-second timeout)
+# Run tests (the test phase of the Dockerfile)
test:
@script/test
@@ -59,21 +59,35 @@ docker:
docker-smoke:
@script/docker-smoke
-# Build Docker image tagged pixad:$(VERSION) and pixad:latest
+# Measure throughput, latency and peak memory with the default duration and
+# number of clients (needs Docker and Go; a benchmark, not part of check)
+loadtest:
+ @script/loadtest
+
+# Serve cache hits in parallel through the image handler and report hits per
+# second (needs Go on the host after `script/bootstrap --cgo`, not Docker; a
+# benchmark, not part of check)
+bench:
+ @script/bench
+
+# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION)
docker-versioned:
- docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
+ @script/docker
+ docker tag pixa pixa:$(VERSION)
-# Run tests in Docker (needed for CGO/libvips)
+# Run tests in Docker, as `make test` does
docker-test:
- docker build --target builder --build-arg VERSION=$(VERSION) -t pixad-builder .
- docker run --rm pixad-builder sh -c "CGO_ENABLED=1 GOTOOLCHAIN=auto go test -v ./..."
+ @script/test
-# Run local dev server in Docker
+# Run local dev server in Docker, with debug on and a new random signing
+# key each run. Log in to the page with the key it prints.
devserver: docker-versioned devserver-stop
+ @key="$$(head -c 32 /dev/urandom | base64)" && \
docker run -d --name pixad-dev -p 8080:8080 \
- -v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
- pixad:latest
- @echo "pixad running at http://localhost:8080"
+ -e PIXA_SIGNING_KEY="$$key" -e PIXA_DEBUG=true \
+ pixa:latest >/dev/null && \
+ echo "pixad running at http://localhost:8080" && \
+ echo "signing key: $$key"
# Stop dev server
devserver-stop:
diff --git a/README.md b/README.md
index 4765703..e089fc7 100644
--- a/README.md
+++ b/README.md
@@ -1,298 +1,1422 @@
# pixa
-pixa is a GPL-3.0-licensed Go web server by
-[@sneak](https://sneak.berlin) that proxies images from upstream
-sources, optionally resizing or transforming them, and serves the
-results. Both source and transformed images are cached to disk so that
-subsequent requests are served without origin fetches or additional
-processing.
+pixa is a GPL-3.0-licensed Go web server by [@sneak](https://sneak.berlin) that
+fetches images from other servers, resizes and converts them, and serves the
+results.
+
+A web page asks pixa for an image at a size and in a format. pixa fetches the
+original image, makes the requested image from it, and serves that. It keeps
+both on disk. Later requests for the same image are served from disk. After 24
+hours pixa first asks the server that holds the original whether it changed (see
+Cache TTL).
+
+In this document, the upstream host is the server that holds an original image.
## Getting Started
```bash
-# clone and build
+# clone, install what compiling needs, and build
git clone https://git.eeqj.de/sneak/pixa.git
cd pixa
+script/bootstrap --cgo
make build
-# run with a config file: copy the example and set a real signing key
-# (the example placeholder is refused at startup), e.g. with
-# openssl rand -base64 32
-cp config.example.yml config.yml
-$EDITOR config.yml # replace the signing_key placeholder
+# make a config file from the example and put a real signing key in it
+cp configs/config.example.yml config.yml
+openssl rand -base64 32 # prints a new signing key
+$EDITOR config.yml # replace the signing_key placeholder with it
./bin/pixad --config config.yml
-# or build and run via Docker
+# or build and run the Docker image with that signing key
make docker
-docker run -p 8080:8080 -e PIXA_SIGNING_KEY="$(openssl rand -base64 32)" pixa:latest
+docker run -p 8080:8080 -e PIXA_SIGNING_KEY='the new signing key' pixa:latest
```
-A container takes its settings from environment variables (see
-Configuration below for the list). Only `PIXA_SIGNING_KEY` is required; if
-it is unset the container exits at startup naming the variable. Everything
-else has a built-in default. A config file mounted at `/etc/pixa/config.yml`
-is optional: it is read when present, and an environment variable wins over
-the same setting in it.
+`script/bootstrap --cgo` installs Go, a C compiler and the image libraries pixa
+needs (see Deployment). `make build` then compiles pixa on your machine into
+`bin/pixad`. `pixad` is the server program.
+
+The signing key is pixa's one required secret. It must be 32 characters or
+longer. The placeholder in the example file is refused at startup.
+
+A container takes its settings from environment variables (see Configuration).
+Only `PIXA_SIGNING_KEY` is required. Without it, the container exits at startup
+and names the variable. Every other setting has a built-in default. A config
+file mounted at `/etc/pixa/config.yml` is read when present. An environment
+variable wins over the same setting in the file.
+
+pixa's login page is at `http://localhost:8080/`. Logging in there works only
+with `debug: true` in `config.yml`, or `-e PIXA_DEBUG=true` for the container.
+Otherwise pixa refuses the login with 403, because the page was opened over
+plain HTTP. Use `debug` only on your own machine. A pixa that others use serves
+its pages over HTTPS through a reverse proxy (see Deployment).
+
+Once pixa runs, it serves images through URLs of this form:
+
+```
+/v1/image///.
+```
+
+For example, this URL serves `https://cdn.example.com/photos/cat.jpg` at 800x600
+as WebP:
+
+```
+/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp
+```
+
+A URL for a host in `allowlist_hosts` needs nothing more. A URL for any other
+host needs a signature (see Signed URLs). Someone who knows the signing key can
+also make an encrypted URL on pixa's own web page (see Encrypted URLs). An
+encrypted URL works for any host and hides the original image's URL.
+
+The example config allowlists no host. Every image on an allowlisted host is
+served to anyone, without a signature, at any size and in any format. So
+allowlist only hosts you control, where nobody else can put images.
+
+To see pixa serve an image, sign a URL for an image on any public HTTPS server.
+Run this in a second terminal while pixa runs, with your own signing key and
+image:
+
+```bash
+key='the new signing key'
+image_host=cdn.example.com # a public HTTPS server
+image_path=/photos/cat.jpg # an image on it
+exp=$(($(date +%s) + 3600)) # the URL expires in one hour
+sig=$(printf '%s' "$image_host:$image_path::400:300:webp:$exp:85:cover" |
+ openssl dgst -sha256 -hmac "$key" -binary | base64 | tr '+/' '-_')
+curl --fail-with-body -o cat.webp \
+ "http://localhost:8080/v1/image/$image_host$image_path/400x300.webp?sig=$sig&exp=$exp"
+```
+
+`cat.webp` is then the image at 400x300. If pixa refuses the request, curl
+reports an error and `cat.webp` holds pixa's error message instead. The image
+path must not hold a `%`, a `:` or a query. Signed URLs explains how to sign
+those.
+
+## Deployment
+
+pixa listens on plain HTTP. It runs behind a reverse proxy: a server in front of
+it that receives the public requests and passes them on. The reverse proxy
+terminates TLS, the encryption behind `https`.
+
+The reverse proxy must:
+
+- terminate TLS. The login and generator pages work only over HTTPS (see
+ Routes).
+- pass the `Host`, `Origin` and `Referer` headers on unchanged. pixa refuses a
+ form from its pages unless `Origin` or `Referer` names the host in `Host`. It
+ builds encrypted URLs from `Host`. It checks `Referer` against
+ `referer_blocklist`.
+- set `X-Forwarded-For` to the client's address. `trusted_proxies` must hold the
+ proxy's address as pixa sees it. The login limit and
+ `image_requests_per_minute` then count each client by its own address.
+- wait for pixa's answer for at least `downstream_timeout` (default `60s`). That
+ is the longest pixa takes to fetch, convert and send an image.
+
+[`configs/Caddyfile`](configs/Caddyfile) is an example for Caddy, a web server.
+Caddy does everything in this list without further settings, and gets its own
+TLS certificate.
+
+The reverse proxy may also refuse `/metrics`, as the example does. Only a
+scraper that reaches pixa directly can then read it. pixa itself asks for
+`metrics.username` and `metrics.password` there.
+
+pixa does the rest itself:
+
+- it checks signatures and encrypted URLs;
+- it applies the allowlist;
+- it refuses upstream hosts with private or local addresses;
+- it limits login attempts and each client's image requests;
+- it limits the size of upstream responses and the dimensions of images;
+- it sends the security headers, `Strict-Transport-Security` included, with
+ every response.
+
+The state directory (`state_dir`, `/var/lib/pixa` in the container) holds the
+database and the disk cache:
+
+- It needs a persistent volume. Without one, each new container starts with an
+ empty cache, as after every redeploy. Losing the directory also ends every
+ login session (see Encrypted URLs).
+- In the container, the startup script gives the directory to the user pixa runs
+ as, uid 65532, and sets its mode to `750`. Outside the container, the user
+ running pixa must be able to write it.
+- `cache_max_bytes` limits the cached images. The limit does not count the
+ database, or files still being written. Eviction runs in the background, so
+ the cache can pass the limit for a while. Leave room on the volume beyond it.
+- Set `cache_max_bytes` for a lasting deployment. Its default is worked out at
+ each start from the free space on the volume (see Configuration). Anything
+ else that fills or frees space on the volume moves that default.
+
+A load balancer's health check can request `/.well-known/healthcheck.json`. It
+answers 200 whenever pixa is running, in maintenance mode too (see
+`maintenance_mode`).
+
+On `SIGTERM` or `SIGINT`, pixa stops accepting connections. The requests in
+progress and the images being processed get 5 seconds to finish. A request not
+finished by then is cut off. pixa then saves the last-used times it has not yet
+written (see Storage), and exits. It exits with status 1 when:
+
+- images were still being processed after those 5 seconds;
+- some of those times could not be saved;
+- another part of pixa failed to stop.
+
+Otherwise it exits with 0. `docker stop` waits 10 seconds before it kills the
+container.
+
+Outside Docker, pixa needs these at run time:
+
+- libvips, the image library pixa converts images with, including its JPEG XL
+ support. pixa does not start without that support. On Alpine it is the
+ `vips-jxl` package. The Nix and Homebrew packages of libvips include it. So do
+ the Debian and Ubuntu packages, from Debian 12 and Ubuntu 24.04 on.
+- libheif, which libvips needs for AVIF.
+- the system's CA certificates, which HTTPS fetches need.
+
+Compiling pixa needs the development files of libvips and libheif, `pkg-config`
+and a C compiler. `script/bootstrap --cgo` installs them, and libvips and
+libheif with them. The `Dockerfile` installs its own packages.
## Running under upaas
-What the [upaas](https://git.eeqj.de/sneak/upaas) app for pixa needs:
+[upaas](https://git.eeqj.de/sneak/upaas) is the deployment tool pixa is meant to
+run under. Its app for pixa needs:
- **Port:** pixa listens on container port `8080`.
-- **Volume:** container path `/var/lib/pixa`, where pixa keeps its
- database and cache. upaas bind-mounts the host path it is given and
- does not create it, so the host directory must exist before the first
- deploy.
+- **Volume:** container path `/var/lib/pixa`, where pixa keeps its database and
+ cache.
- **Environment variables:**
- - `PIXA_SIGNING_KEY` (required): secret for signed and encrypted URLs
- and login, 32+ characters, for example from
- `openssl rand -base64 32`
+ - `PIXA_SIGNING_KEY` (required): the secret for signed URLs, encrypted URLs
+ and login. It is 32 characters or more, for example from
+ `openssl rand -base64 32`.
- `PIXA_ALLOWLIST_HOSTS`: upstream hosts served without a signature,
- comma-separated
- - `PIXA_CACHE_MAX_BYTES`: disk cache limit in bytes; `0` disables it;
- default 75% of free space
- - the rest are in the table under Configuration below
+ comma-separated; default none. Every image on a listed host is served to
+ anyone, so list only hosts you control, where nobody else can put images.
+ - `PIXA_CACHE_MAX_BYTES`: the disk cache limit in bytes; `0` turns the disk
+ cache off. Set it. The volume is a directory on the host's filesystem. The
+ host and every other app use that filesystem too. The default is 75% of
+ the sum of that filesystem's free space and what the cache already holds.
+ So by default the cache may grow to three quarters of the space the host
+ and the other apps have left. The default is also at least 500 MiB. So
+ when that space is 500 MiB or less, the cache may take all of it.
+ - the rest are in the list under Configuration.
+- **Memory:** pixa can use up to `max_concurrent_processing` times 4.2 GiB for
+ the images it processes at once. On top of that, the fetched source images
+ waiting for processing can take `upstream_connections` times
+ `upstream_max_response_size`. 4.2 GiB is a measured value. It is the largest
+ rise in resident memory measured while pixa saved one 8192x8192 source, the
+ largest it accepts, to each output format at full size. The rise was largest
+ for a 16-bit JPEG XL of random pixels saved as WebP. At the defaults the total
+ is half the CPUs pixa sees times 4.2 GiB, plus 64 times 50 MiB. On a host
+ where pixa sees 48 CPUs, that is 24 times 4.2 GiB plus 3.1 GiB, about 104 GiB.
+ Set `PIXA_MAX_CONCURRENT_PROCESSING` and `PIXA_UPSTREAM_CONNECTIONS` so that
+ this fits the memory the container may use. An SVG source is rendered in a
+ process of its own, which can use up to 1 GiB more.
+- **Temporary files:** while pixa processes a large source, libvips writes the
+ decoded pixels to a file in `/tmp`. The file is removed from the directory
+ once it is written, so a listing of `/tmp` does not show it. It keeps its
+ space until the image is done: on disk, or in memory where `/tmp` is a tmpfs.
+ For a GIF, WebP, AVIF or JPEG XL source the decoded pixels are held in memory
+ as well, and the 4.2 GiB above includes that memory. For an 8192x8192 source
+ the file was measured at:
+ - 192 MiB for an 8-bit RGB JPEG;
+ - 512 MiB for a 16-bit RGBA PNG or JPEG XL;
+ - 1 GiB for a 32-bit floating-point RGBA JPEG XL.
+
+ So count up to 1 GiB per image processed at once: 24 GiB at the defaults on
+ a host where pixa sees 48 CPUs. This `/tmp` is in the container's own
+ filesystem, not on the volume at `/var/lib/pixa`. `cache_max_bytes` does not
+ count these files, and the 4.2 GiB above does not include them.
+
- **Health check:** the image's `HEALTHCHECK` requests
- `/.well-known/healthcheck.json`. upaas reads the container's health 60
- seconds after a deploy and marks the deploy failed unless it is
- `healthy`. The probe uses the port from `PORT` (default `8080`), so a
- port changed only in a mounted config file is not seen by it: change
- the port with `PORT`.
-- **First run:** create the host directory. It may be owned by root: the
- container gives it to its `pixad` user when it starts.
+ `/.well-known/healthcheck.json`. upaas reads the container's health 60 seconds
+ after a deploy and marks the deploy failed unless it is `healthy`. The probe
+ takes the port from `PORT` (default `8080`). It does not see a port changed
+ only in a mounted config file, so change the port with `PORT`.
## Rationale
-Image-heavy web applications need a fast, caching reverse proxy that
-can resize and transcode images on the fly. pixa fills that role as a
-single, self-contained binary with no external runtime dependencies
-beyond libvips. It supports HMAC-SHA256 signed URLs with expiration to
-prevent abuse, and allowlisted source hosts for open access.
+A web application that shows images needs each one at several sizes and in
+several formats. pixa makes them on request from one original, and keeps what it
+made. The application stores one image per picture and hands out URLs. pixa is
+one binary. At run time it needs only the image libraries and the CA
+certificates named under Deployment. Signed URLs with an expiry keep strangers
+from using it to resize anything they like. Allowlisted hosts give open access
+where that is wanted.
## Design
-### Storage
+### Concepts
-- **Source content**:
- `/cache/src-content///`
-- **Source metadata**:
- `/cache/src-metadata//.json`
- (fetch time, original headers, request, content hash)
-- **Database**: `/state.sqlite3` (SQLite)
-- **Output documents**:
- `/cache/dst-content///`
+These terms are the names the code uses:
-Multiple source paths may reference the same content blob; the
-database tracks references rather than using filesystem refcounting.
-In-process caching of request-to-output mappings targets 1-5k r/s.
+- **Source**: the image at one upstream URL, named by its host, path and query.
+- **Source key**: the SHA-256 hash of a source's host, path and query. SHA-256
+ is a hash function: it turns any text into a fixed-length fingerprint. The
+ source key names the source's file and its row in the database.
+- **Transform**: the output size, format, quality and fit that an image request
+ asks for. With the format `auto`, pixa chooses the output format from the
+ `Accept` header.
+- **Image request**: a source and a transform. Each image route reads one from
+ its URL.
+- **Variant**: the image pixa serves for an image request, made from its source
+ with its transform.
+- **Variant key**: the SHA-256 hash of a variant's source, size, format, quality
+ and fit. It names the variant's file and its row in the database. The
+ variant's `ETag` is made from it and the file's modification time.
+- **Signed URL**: a `/v1/image/` URL whose `sig` parameter signs its image
+ request and its expiry `exp` with `signing_key`.
+- **Encrypted URL**: a `/v1/e/` URL whose token holds its image request and any
+ expiry, encrypted with a key derived from `signing_key`.
+- **Login session**: an encrypted cookie that keeps a browser logged in to the
+ generator page. It lasts 30 days, or until a logout (see Encrypted URLs).
+- **Negative cache**: the source URLs pixa does not fetch again for 5 minutes,
+ after a fetch failed in one of these ways:
+ - the upstream answered with an error status;
+ - the body's first bytes were not those of the format its `Content-Type`
+ names;
+ - a check of a cached image failed in any way (see Cache TTL).
+- **Image cache**: the part of pixa that keeps the sources, the variants and the
+ negative cache (see Storage).
+- **Cache TTL**: how long pixa uses a cached source and serves its variants
+ before it checks the source with upstream again: 24 hours (see Cache TTL).
+- **Disk cache limit** (`cache_max_bytes`): the size in bytes that the cached
+ sources and variants may reach together.
+- **Image proxy**: the part of pixa that serves a variant from the image cache
+ when it can, and otherwise makes it from the source.
### Routes
+pixa answers these routes. Any other path answers 404. A listed path answers 405
+to any method the list does not give for it.
+
+- `GET /` — the login page, or the generator page with a login session (see
+ Encrypted URLs). Needs nothing. Answers 200.
+- `POST /` — log in with the signing key typed into the login page. Needs the
+ login page's form (below). For the right key it answers 303 to `/` and sets
+ the login session cookie. For a wrong key it answers 200 with the login page
+ and an error. Answers 429 over the login limit (see Limits).
+- `POST /generate` — make an encrypted URL from the generator page's form. Needs
+ a login session and the generator page's form. Without a login session it
+ answers 303 to `/`. Answers 200 with the page showing the URL. Answers 400
+ with the page naming the field it refused (see Encrypted URLs). Answers 500
+ when the URL cannot be made.
+- `POST /logout` — log out with the generator page's Logout button. That ends
+ every login session, in every browser (see Encrypted URLs), and removes the
+ cookie. Needs a login session and the generator page's form. Without a login
+ session it only removes the cookie. Answers 303 to `/`. Answers 500 when the
+ logout cannot be saved in the database.
+- `GET` or `HEAD` `/v1/image///.`, or
+ `/v1/image///` with no format — an image, fetched, resized
+ and converted (see Image URLs). Needs a signature, unless the host is
+ allowlisted (see Source Hosts). Answers:
+ - 200 with the image.
+ - 304 when `If-None-Match` matches the image's `ETag`.
+ - 400 for a URL or parameter that is not valid.
+ - 400 when the format is `auto` and the `Accept` header is not valid.
+ - 400 for `orig` of an SVG source at a size other than `orig`.
+ - 406 when the format is `auto` and `Accept` allows none of the formats it
+ chooses from.
+ - 401 for a missing or wrong signature.
+ - 401 for a missing `exp`.
+ - 401 for an `exp` in the past.
+ - 403 when the request's `Referer` names a host in `referer_blocklist`. This
+ is checked before the signature, the cache and the upstream fetch.
+ - 403 when the upstream host, or a host it redirects to, is `localhost` or
+ ends in `.localhost` or `.local`.
+ - 403 when the upstream host, or a host it redirects to, has an address in a
+ blocked network (see `blocked_networks`).
+ - 429 over `image_requests_per_minute` (see Limits).
+ - 499, with no body, when the client goes away before its image is ready,
+ unless its request is the one making the image (see Limits).
+ - 415 when the upstream answers with a `Content-Type` that is not one of the
+ source formats (see Image URLs).
+ - 415 when the upstream sends a body whose first bytes are not those of the
+ format its `Content-Type` names.
+ - 415 when the upstream sends an image whose first bytes libvips reads as
+ another format.
+ - 415 for an SVG source with one side more than 8192 times the other, in any
+ format but `orig`.
+ - 415 for an SVG source that is not valid, or that takes more than 20
+ seconds or 1 GiB of memory to render.
+ - 502 when the upstream answers with an error status.
+ - 502 for 5 minutes after that for the same source URL, when pixa has
+ neither the image nor its source in its cache.
+ - 502 likewise for 5 minutes after a body whose first bytes are not those of
+ its `Content-Type`.
+ - 502 when the upstream host, or a host it redirects to, has no address.
+ - 502 when the upstream redirects to a URL that is not `https`, or redirects
+ 10 times.
+ - 502 when the fetch fails before the upstream answers, such as on a refused
+ connection or a failed TLS handshake.
+ - 503 when pixa is busy (see Limits).
+ - 503 in maintenance mode.
+ - 504 when the address lookup of the upstream host, or of a host it
+ redirects to, times out.
+ - 504 when the connection to the upstream host, or to a host it redirects
+ to, times out.
+ - 504 when the upstream has not finished the TLS handshake within 10
+ seconds.
+ - 504 when the upstream has not sent its response headers within
+ `upstream_fetch_timeout`.
+ - 500 when that time runs out while the image itself is still arriving.
+ - 500 for any other failure.
+- `GET` or `HEAD` `/v1/e//` — an image through an encrypted URL
+ (see Encrypted URLs). Needs nothing but the URL. Answers:
+ - 200 with the image.
+ - 304 when `If-None-Match` matches the image's `ETag`.
+ - 400 for a token that does not decrypt.
+ - 400 for a token that asks for a size or fit that is not valid.
+ - 400 for a token that asks for `orig` of an SVG source at a size other than
+ `orig`.
+ - 410 once the token has expired.
+ - 400 when the token's format is `auto` and the `Accept` header is not
+ valid.
+ - These answers, for the same causes as on `/v1/image/`:
+ - 406
+ - 403
+ - 415
+ - 429
+ - 499, with no body
+ - 502
+ - 503
+ - 504
+ - 500
+- `GET /robots.txt` — asks every crawler to stay away (`Disallow: /`). Needs
+ nothing. Answers 200.
+- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`,
+ `uptime_seconds`, `uptime_human`, `version`, `appname` and `maintenance_mode`.
+ Needs nothing. Answers 200, always.
+- `GET /static/` — the stylesheet and script the login and generator pages
+ load. Needs nothing. Answers 200, or 404 for a file that does not exist. It
+ answers any other method as it answers `GET`.
+- `GET /metrics` — Prometheus metrics (see Architecture). Needs HTTP basic
+ authentication with `metrics.username` and `metrics.password`. Answers 200.
+ Answers 401 without them. Answers 404 when they are not set, as the route then
+ does not exist.
+
+CORS is the browser rule that lets a page on one site read responses from
+another. The preflight is the `OPTIONS` request the browser sends first. It
+carries an `Origin` header and an `Access-Control-Request-Method` header. A
+preflight to any path under `/v1/` answers 200, in maintenance mode too.
+
+Every response carries an `X-Request-Id` header with the request's ID. A client
+can quote it when reporting a problem. A reverse proxy may send an
+`X-Request-Id` header with the request. pixa keeps that ID when it is 1 to 64
+letters, digits, `-`, `_` or `.`. Otherwise pixa makes a random ID, which tells
+nothing about the machine or the other requests. pixa's log line for the request
+carries the same ID as `request_id`. So do the lines it logs when it fetches,
+converts and serves an image. The fetch sends it to the upstream host as
+`X-Request-Id`.
+
+The `POST` routes accept only a form that pixa's own page served. The page puts
+a token in the form and sets a cookie to match. A request without both is
+refused with 403. So another site cannot submit the form from a visitor's
+browser. A form is also refused with 403 when the page's host is not the `Host`
+header pixa receives. So a reverse proxy in front of pixa must pass that header
+on unchanged. A form body over 1 MiB is refused with 413.
+
+The login and generator pages are meant to be opened over HTTPS. While `debug`
+is off, a form sent from a page opened over plain HTTP is refused with 403.
+While `debug` is on, a form sent from a page opened over HTTPS is refused
+instead. Plain HTTP is for development on the browser's own machine. The login
+session cookie is always marked `Secure`. Over plain HTTP a browser keeps such a
+cookie only for its own machine (`localhost`), if at all.
+
+A form refused with 403 gets a page that names the cause and what to do about
+it. pixa logs a `form refused` warning with the request ID and the form-token
+library's reason, such as `origin invalid`.
+
+The image routes answer the errors listed for them with a JSON body holding
+`error`, `status` and `timestamp`.
+
+### Image URLs
+
+An image URL has one of these forms. The second has no format:
+
```
-/v1/image///.?sig=&exp=
+/v1/image///.?sig=&exp=&q=&fit=
+/v1/image///?sig=&exp=&q=&fit=
```
-Images are only fetched from origins using TLS with valid certificates.
+- ``: the upstream host, with a port when the source URL has one (see
+ Source Hosts).
+- ``: the source URL's path, and its query after `%3F` (below).
+- ``: `orig` or `x`, for example `800x600`. A side of `0`
+ is worked out from the other side and the source's proportions (below). `orig`
+ is `0x0`: the source's own size.
+- ``: `orig` (or `original`), `jpeg` (or `jpg`), `png`, `webp`, `avif`,
+ `jxl` (JPEG XL), `gif`, or `auto` (below). `orig` is the source's own format.
+ A URL with no format is served as JPEG XL, the default, as with `jxl`.
+- `sig` and `exp`: the signature and its expiry, needed unless the host is
+ allowlisted (see Signed URLs).
+- `q`: the output quality, a whole number from 1 to 100. It is `85` when the URL
+ has no `q`.
+- `fit`: how the image is fitted to ``: `cover`, `contain`, `fill`,
+ `inside` or `outside` (below). It is `cover` when the URL has no `fit`.
-A request whose query string cannot be decoded, or gives any parameter more
-than once, is refused with 400.
+`q` and `fit` are part of what is cached, so each value of either is a separate
+cached image. A request is refused with 400 when its query string cannot be
+decoded, or when it gives any parameter more than once. A request with a `q`,
+`fit` or `exp` that is empty or not valid is refused with 400 too.
-- ``: one of `orig`, `png`, `jpeg`, `webp`
-- ``: `orig` or `x` (e.g. `800x600`)
+pixa fetches sources over HTTPS only, and only from hosts with valid
+certificates. With `allow_http` set, it fetches every source over plain HTTP
+instead. That is for testing only.
-The login form (`POST /`) is limited to 5 attempts per minute per client
-address, counting an IPv6 client by its /64; an attempt over the limit is
-refused with 429 and a `Retry-After` header. Behind a reverse proxy the client
-address comes from `X-Forwarded-For` only when the proxy's address is in
-`trusted_proxies`; otherwise all users behind the proxy are counted as one
-client. With the default `trusted_proxies` (the RFC 1918 ranges), a client
-with a private address can choose the address it is counted by through its own
-`X-Forwarded-For`, whether it connects directly or through the proxy, because
-its own address is trusted too. Setting `trusted_proxies` to the proxy's own
-address closes this.
+**The source path and query.** A source URL's query follows its path after
+`%3F`, the encoded `?`:
-### Image Metadata
+```
+/v1/image//%3F/.
+```
-pixa decodes and re-encodes every image it serves, and removes all metadata from
-the output: EXIF (GPS position, camera make, model and serial number, capture
-time, embedded thumbnail), XMP, IPTC and the ICC colour profile. This cannot be
-turned off.
+pixa decodes the percent-escapes in the image URL's path once. It then reads the
+host up to the first `/`, and the size and format after the last `/`. Between
+them it reads the source path up to the first `?`, and the source query after
+it. So these must be percent-encoded, as `%3F` and `%25`:
-- The `orig` format means the source's own format, not the source's bytes: an
- `orig` image is re-encoded and stripped like any other.
-- An image with an EXIF orientation is turned upright first, so it displays the
- same without the tag; a requested size applies to the upright image.
-- An image with an ICC profile is converted to sRGB first, since clients show an
- image with no profile as sRGB. Colours outside sRGB, such as the most
- saturated ones in a Display P3 photo, are clipped.
+- the `?` before the query;
+- any `?` in the query;
+- every `%` in the source path or query.
-### Source Hosts
+Any other character may be written as itself or percent-encoded. `/`, `;`, `,`,
+`=` and `&` work either way. Neither the source path nor the source query may
+hold a `#`. An image URL that writes one as `%23` is refused with 400. For
+example, the image
+`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg` at
+800x600 as WebP is:
-Source hosts may be allowlisted in the configuration. Non-allowlisted
-hosts require an HMAC-SHA256 signature.
+```
+/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp
+```
-#### Signature Specification
+Signed URLs shows this URL with its signature.
-Signatures use HMAC-SHA256 and include an expiration timestamp to
-prevent replay attacks. Signatures are **exact match only**: every
-component (host, path, query, dimensions, format, expiration, quality,
-fit) must match exactly what was signed. No suffix matching, wildcard
-matching, or partial matching is supported.
+pixa keeps the source path decoded and fetches it percent-encoded again. Every
+character other than these is sent as a percent-escape: letters, digits, `-`,
+`.`, `_`, `~`, `/`, `$`, `&`, `+`, `,`, `:`, `;`, `=` and `@`. So:
-**Signed data format** (colon-separated):
+- `my%20cat.jpg` in the image URL is the source path `/my cat.jpg`, fetched as
+ `/my%20cat.jpg`;
+- `my%2520cat.jpg` is the source path `/my%20cat.jpg`, fetched as
+ `/my%2520cat.jpg`;
+- `100%25.png` is fetched as `/100%25.png`.
+
+The source query is sent as the source URL writes it, its percent-escapes
+included. Two kinds of character are percent-encoded first: those a browser
+percent-encodes in a query, and a `%` that does not begin a percent-escape. The
+first kind is a space, `"`, `'`, `<`, `>`, control characters and characters
+outside ASCII. pixa signs and caches the query in that form. So
+`img%3Fname=my%20cat` and `img%3Fname=my%2520cat` in the image URL are the same
+source query, `name=my%20cat`.
+
+**Sizes.** The largest width or height is 8192, for the output and for a source
+in any format but SVG. A size with one side `0` has that side worked out from
+the other side and the source's proportions. The side is rounded down, but is at
+least 1. When it would come out above 8192, the request is refused with 400. The
+fit modes:
+
+- `cover`: scale the image to cover the size, keeping its proportions, and cut
+ out the centre. The output is exactly the size.
+- `contain`: scale the image to fit inside the size, keeping its proportions.
+ The output can be smaller than the size on one side.
+- `fill`: stretch the image to the size, without keeping its proportions.
+- `inside`: as `contain`, but never enlarge the image.
+- `outside`: scale the image to cover the size, keeping its proportions, without
+ cutting. The output can be larger than the size on one side. An 8192x1 source
+ at `100x0` is 100x1 by its proportions, but comes out at 8192x1. A side that
+ would come out above 8192 is refused with 400.
+
+An AVIF output of more than 16,777,216 pixels is refused with 400 before
+anything is decoded. That is the number of pixels in 4096x4096 or 8192x2048. A
+larger AVIF encode could take more than 40 seconds (see Limits). The limit also
+applies to the format `orig` when the source is AVIF, and to `auto` when it
+chooses AVIF. So a client whose `Accept` names `image/avif` but not `image/jxl`
+is refused such a size.
+
+**Source formats.** The source may be JPEG, PNG, GIF, WebP, AVIF, JPEG XL or
+SVG. The upstream's `Content-Type` must name its format, and the image's first
+bytes must match it. An upstream answering with any other `Content-Type` is
+refused with 415. So is a body whose first bytes are not those of the format its
+`Content-Type` names: a JPEG sent as `image/png`, text sent as an image, or an
+empty body. pixa then does not fetch that source URL again for 5 minutes, as
+after an upstream error status. Meanwhile it answers 502 for an image that would
+need that fetch. An image whose first bytes libvips reads as another format is
+refused with 415 too. An SVG whose first bytes pass the check for AVIF is one
+example.
+
+**SVG sources.** `orig` of an SVG source is the SVG itself, unchanged, and is
+cached like any other image. It is given only at the `orig` size, since pixa
+does not resize an SVG as SVG. `orig` of an SVG at any other size is refused
+with 400. A browser that opens an SVG from pixa applies its styles and draws its
+embedded `data:` images, but runs none of its scripts and loads nothing else.
+
+In any format but `orig`, pixa renders an SVG source at the requested size, so
+an enlarged SVG stays sharp. Rendering fetches no URL and reads no file that the
+SVG refers to. In any format but `orig`, an SVG at the `orig` size is rendered
+at its own size. In any format but `orig`, an SVG above 8192 on a side is
+refused with 400 at the `orig` size. In any format but `orig`, an SVG with one
+side more than 8192 times the other is refused with 415. Rendered with no side
+above 8192, its shorter side would be under 1 pixel.
+
+**The format `auto`.** With `auto`, pixa chooses the format for each request
+from its `Accept` header, in this order:
+
+1. JPEG XL, when the header names `image/jxl`;
+2. AVIF, when it names `image/avif`;
+3. WebP, when it names `image/webp`;
+4. JPEG, when the most specific of `image/jpeg`, `image/*` and `*/*` that the
+ header names has a `q` above 0, or when there is no `Accept` header or it is
+ empty.
+
+An entry with `q=0` refuses its format. Other `q` values do not change the
+order. JPEG XL, AVIF and WebP must be named, as clients that cannot show them
+also send `image/*` and `*/*`. pixa never sends a format the client refused.
+When the header allows none of the four, the answer is 406. A header that does
+not parse, or has a `q` that is not a number from 0 to 1, is refused with 400.
+The signature, or the token of an encrypted URL, covers `auto` itself, so one
+URL serves every client. Each format chosen is cached as a separate image. Every
+answer that depends on `Accept` carries `Vary: Accept`: the image, a 304, and
+the 400 and 406 above. So a shared cache keeps the formats apart too.
+
+**Response headers.** An image is served with
+`Cache-Control: public, max-age=, immutable`. When the URL has an
+expiry (an `exp`, or the expiry of an encrypted URL), `max-age` is the whole
+seconds left until then, at most one year. So no browser or proxy cache keeps
+the image after pixa would refuse the URL. A URL with no expiry gets one year.
+`immutable` only stops a client revalidating while its copy is fresh. The
+response also carries `X-Pixa-Cache`: `HIT` for an image served from the cache,
+and `MISS` for one made for this request or older than 24 hours (see Cache TTL).
+The `ETag` is made from the first 16 characters of the variant key and the
+variant file's modification time (see Cache TTL). So a variant made again from a
+changed source gets a new `ETag`.
+
+### Signed URLs
+
+A signed URL is a `/v1/image/` URL with two query parameters: `exp`, the Unix
+time (seconds since 1 January 1970) when it expires, and `sig`, its signature.
+The signature is an HMAC-SHA256: a hash of the request's details, keyed with
+`signing_key`, that only someone who knows the key can make. Signatures match
+exactly: every part of the signed data must be exactly what was signed. There is
+no suffix, wildcard or partial matching.
+
+The signed data is one line of nine parts, joined with `:`:
```
HMAC-SHA256(secret, "host:path:query:width:height:format:expiration:quality:fit")
```
-Where:
+- `host`: the source host as the URL writes it up to the first `/`, in lower
+ case. An IPv6 address keeps its brackets, and a port is kept after its `:`.
+ Examples: `cdn.example.com`, `cdn.example.com:8443`, `[2001:db8::1]:8443`.
+- `path`: the source path with its percent-escapes decoded, as under Image URLs,
+ then with each `%` written as `%25` and each `:` as `%3A`. So
+ `/photos/my%20cat.jpg` in the URL is signed as `/photos/my cat.jpg`, and
+ `/photos/a:b.jpg` as `/photos/a%3Ab.jpg`.
+- `query`: the source query as under Image URLs, with `%` and `:` written as in
+ `path`. It is empty when there is no query. Neither part then holds a `:`, so
+ path `/a:b` with no query and path `/a` with query `b:` are signed
+ differently.
+- `width` and `height`: the requested size in pixels. A side of `0` is signed as
+ `0`, and `orig` as `0` and `0`.
+- `format`: the output format as listed under Image URLs, with `original` signed
+ as `orig`, `jpg` as `jpeg`, and no format as `jxl`. So a URL with no format
+ has the signature of the same URL ending in `.jxl`. `auto` is signed as
+ `auto`, not as the format chosen for the request.
+- `expiration`: the URL's `exp`, the Unix time in seconds when the signature
+ expires. A request whose `exp` is not a whole number is refused with 400. An
+ empty `exp=` is one example.
+- `quality`: the URL's `q`, or `85` when the URL has no `q`.
+- `fit`: the URL's `fit`, or `cover` when the URL has no `fit`.
-- `host` — source origin hostname (e.g. `cdn.example.com`)
-- `path` — source path (e.g. `/photos/cat.jpg`)
-- `query` — source query string, empty string if none
-- `width` — requested width in pixels, `0` for original
-- `height` — requested height in pixels, `0` for original
-- `format` — output format (jpeg, png, webp, avif, gif, orig)
-- `expiration` — the URL's `exp` query parameter, the Unix timestamp when
- the signature expires; a request whose `exp` is not a whole number, an
- empty `exp=` included, is refused with 400
-- `quality` — the URL's `q` query parameter, a whole number from 1 to 100,
- or `85` when the URL has no `q`; a request whose `q` is anything else is
- refused with 400
-- `fit` — the URL's `fit` query parameter (cover, contain, fill, inside,
- outside), or `cover` when the URL has no `fit`; a request whose `fit` is
- anything else, an empty `fit=` included, is refused with 400
+The URL's `sig` is the HMAC-SHA256 result in base64url, the URL-safe alphabet of
+RFC 4648, with the trailing `=` padding kept: 44 characters in all. pixa
+compares it exactly. A signature encoded without padding is refused with 401.
+Node's `base64url` and Go's `base64.RawURLEncoding` encode without padding.
-**Example:** resize `https://cdn.example.com/photos/cat.jpg` to 800x600
-WebP with expiration 1704067200, default quality and fit:
+**Example.** With the signing key `example-signing-key-for-documentation`,
+resize `https://cdn.example.com/photos/cat.jpg` to 800x600 WebP, expiring at
+1704067200, with the default quality and fit:
-1. Build input:
+1. Build the signed data:
`cdn.example.com:/photos/cat.jpg::800:600:webp:1704067200:85:cover`
-2. Compute HMAC-SHA256 with your secret key
-3. Base64URL-encode the result
-4. URL:
- `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=&exp=1704067200`
+2. Compute its HMAC-SHA256 with the signing key.
+3. Encode the result as base64url, keeping the `=` padding:
+ `-ay7KHpfqmtIGbibDGbUuBDkymi-Ymdn0NkC6j5EJag=`
+4. The URL:
+ `/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=-ay7KHpfqmtIGbibDGbUuBDkymi-Ymdn0NkC6j5EJag=&exp=1704067200`
-For the same image at quality 40 with fit `contain`, the input ends in
-`:40:contain` and the URL is
-`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=&exp=1704067200&q=40&fit=contain`.
+For the same image at quality 40 with fit `contain`, the signed data ends in
+`:40:contain`. The signature is `5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=`,
+and the URL is
+`/v1/image/cdn.example.com/photos/cat.jpg/800x600.webp?sig=5IwXUx6vf7yefhaUvFzgXZvG2o0Df4RJxPTK3pKq5VU=&exp=1704067200&q=40&fit=contain`.
-**Allowlist patterns:**
+For a source URL with a query,
+`https://cdn.example.com/img?url=https://origin.example.com/my%20cat.jpg`, at
+800x600 WebP with the same key, expiry, quality and fit:
-- **Exact match**: `cdn.example.com` — matches only that host
-- **Suffix match**: `.example.com` — matches `cdn.example.com`,
- `images.example.com`, and `example.com`
+1. The signed data, with the query's `:` written as `%3A` and its `%` as `%25`:
+ `cdn.example.com:/img:url=https%3A//origin.example.com/my%2520cat.jpg:800:600:webp:1704067200:85:cover`
+2. The signature: `37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=`
+3. The URL, with the `?` before the query written as `%3F` and the query's `%`
+ as `%25`:
+ `/v1/image/cdn.example.com/img%3Furl=https://origin.example.com/my%2520cat.jpg/800x600.webp?sig=37aFa9GSVCZM1kRY9SzMOk2fOg_M7Wt5gKNR4xM-FYA=&exp=1704067200`
+
+### Encrypted URLs
+
+An encrypted URL is an image URL made on pixa's own web page by someone who
+knows the signing key. It works for any upstream host, allowlisted or not,
+without a signature. Whoever gets it can neither read the source URL from it nor
+change what it asks for.
+
+1. Open `/` in a browser over HTTPS and log in with the signing key. The login
+ session lasts 30 days, or until someone logs out with the generator page's
+ Logout button.
+2. On the generator page, fill in the form. Its fields are listed below. Then
+ submit it (`POST /generate`).
+3. The page shows the URL, `https:///v1/e//img.`, and when
+ it expires. `` is the host the page was opened on. While `debug` is on,
+ the URL starts with `http` instead. The name after the token is ignored. It
+ only gives the URL a file extension: `jpg` for `orig` and `auto`, and `jxl`
+ for a form with no format.
+
+The form's fields:
+
+- The source image's URL. Its scheme is not kept (below).
+- The width and height. Both empty or `0` keep the source's size. When only one
+ of them is empty or `0`, that side is worked out from the source's proportions
+ when the image is made (see Image URLs).
+- The format: `orig`, `auto`, `jpeg`, `png`, `webp`, `avif`, `jxl` or `gif`.
+ `jxl` is selected unless another is chosen. A form sent with an empty format,
+ or none, also makes a JPEG XL URL.
+- The quality, a whole number from 1 to 100. The page offers five values: `25`,
+ `50`, `70`, `85` and `100`. `85` is selected unless another is chosen.
+- The fit, one of the five fit modes (see Image URLs). `cover` is selected
+ unless another is chosen. A form sent with an empty fit, or none, makes a
+ `cover` URL.
+- How long the URL lasts, in seconds, or `0` for never. The page offers seven
+ choices: never, 1 minute, 1 hour, 1 day, 1 week, 30 days and 1 year. Never is
+ selected unless another is chosen.
+
+The page refuses a form with 400 when:
+
+- the source URL does not parse, or has no host;
+- the source URL's path holds a `..` segment, as `/v1/image/` refuses one;
+- a width, height, quality or expiry is not a number, or is out of range;
+- the fit is not one of the five fit modes;
+- pixa would fetch the source URL in another form (below).
+
+The page refuses a source URL that pixa would fetch in another form, and shows
+the URL pixa would fetch. A URL with a user name and password is one example:
+pixa does not send them. A URL ending in `?` with no query is another. A path
+holding `%2F` is another: pixa would fetch it as `/`. So is a path holding `(`:
+pixa would fetch it as `%28` (see Image URLs).
+
+The image is made only when the URL is requested. A side that would come out
+above 8192, or an AVIF output of more than 16,777,216 pixels, is refused with
+400 then (see Image URLs).
+
+Every login is made with the one signing key. So logging out ends every login
+session, in every browser, and a copied session cookie too. pixa keeps a random
+value in its database. It makes the value when the database has none, and
+replaces it at each logout. A login session carries the value current at its
+login. Once the value has changed, the session is refused, after a restart too.
+The check compares no time, so a login after a logout works whatever the host's
+clock says. The 30 days are still measured by the host's clock.
+
+The token holds the source's host, path and query, and the size, format,
+quality, fit and expiry. It is encrypted with a key derived from `signing_key`.
+The host is kept in lower case and an empty path as `/`, as for `/v1/image/`. So
+`https://CDN.Example.com` and `https://cdn.example.com/` are one source, fetched
+and cached once. The source URL's scheme is not kept. The image is fetched like
+any other (see Image URLs), and the blocked networks still apply.
+
+The expiry is fixed in the token when the URL is made. It cannot be changed or
+revoked afterwards. Until then the image is served with a `max-age` that ends at
+the expiry (see Image URLs). After it, the URL answers 410 `URL has expired`. A
+URL made to last forever holds no expiry. It stops working only when
+`signing_key` changes. Changing the key makes every encrypted URL already handed
+out answer 400, and ends every login session.
+
+### Source Hosts
+
+Hosts in `allowlist_hosts` are served without a signature. Any other host needs
+one. An entry is written in one of two ways:
+
+- `cdn.example.com`: an exact match, only that host;
+- `.example.com`: a suffix match, which matches `cdn.example.com`,
+ `images.example.com` and `example.com` itself.
+
+An IP address is matched exactly. Write an IPv6 address without brackets. An
+entry that is neither a host name nor an IP address aborts startup. A host name
+here is letters, digits, hyphens, underscores and dots, with at most one leading
+dot. So an entry with a port, or a `*.` wildcard, aborts startup.
+
+pixa lower-cases the host of a `/v1/image/` URL before anything else reads it.
+So `CDN.Example.com` is matched, fetched, cached and counted against
+`upstream_connections_per_host` as `cdn.example.com`. A URL whose host has a
+port needs a signature even when the host is allowlisted. So does one whose host
+ends in `:` with no port. `cdn.example.com:8443` is a host with a port. The host
+must be one of:
+
+- a host name or an IPv4 address: letters, digits, hyphens, underscores and
+ dots;
+- an IPv6 address in brackets.
+
+Either may be followed by `:` and a port. Any other host is refused with 400.
+One holding `@` is an example. So is one with a `?`, `#`, `%`, `\` or space
+written percent-encoded.
+
+### Image Metadata
+
+pixa decodes and re-encodes every image it serves, except `orig` of an SVG
+source, which is the SVG unchanged. It removes all metadata from every image it
+re-encodes. This cannot be turned off. The metadata removed is:
+
+- EXIF, which can hold the GPS position, the camera make, model and serial
+ number, the capture time and an embedded thumbnail;
+- XMP and IPTC, two other metadata formats;
+- the ICC colour profile.
+
+The rest of the rules:
+
+- For a source in any format but SVG, the `orig` format means the source's own
+ format, not the source's bytes. An `orig` image is re-encoded and stripped
+ like any other. For an SVG source, `orig` is the SVG's own bytes, its metadata
+ included.
+- An image with an EXIF orientation is turned upright first, so it displays the
+ same without the tag. A requested size applies to the upright image.
+- An image with an ICC profile is first converted to sRGB, the standard colour
+ space, since clients show an image with no profile as sRGB. Colours outside
+ sRGB are clipped: the most saturated ones in a Display P3 photo, for example.
+- JPEG XL output is an exception, with libvips 8.16 and later. libvips then
+ writes an EXIF block of its own, which pixa cannot stop. The block holds the
+ image's size and fixed values: an orientation of 1 and a resolution of 72 dpi.
+ Nothing in it comes from the source. The Docker image has libvips 8.16, so its
+ JPEG XL output carries this block.
+
+### Limits
+
+**Shared work.** Several requests for one variant can miss the cache at the same
+time. They then share one upstream fetch, or one read of the cached source, and
+one conversion. The first request does the work. The others wait for its image
+or its error, and hold no upstream connection or processing slot of their own. A
+waiting request stops waiting when its own client goes away. It is then
+answered 499. The work goes on for the others even when the first request's
+client goes away. The shared fetch sends the first request's ID upstream. The
+lines logged for the fetch and the conversion carry that ID.
+
+**Timeouts.** `downstream_timeout` (default `60s`) bounds the whole answer to
+one request. The upstream fetch counts toward it. So do the waits for a
+connection slot and for a processing slot. The wait for a shared
+`upstream_connections` slot ends after 10 seconds, and so does the wait for a
+processing slot. The wait for an `upstream_connections_per_host` slot ends only
+with `downstream_timeout`. Keep `downstream_timeout` longer than
+`upstream_fetch_timeout` plus 20 seconds.
+
+A conversion whose request's `downstream_timeout` has ended stops waiting for a
+processing slot. If its `downstream_timeout` ends after it has a slot but before
+it begins to encode, it gives the slot back. For a shared conversion, the first
+request's deadline counts. When a shared conversion stops this way, each request
+waiting for it is answered 503 `server busy, try again later`, even if its own
+`downstream_timeout` has not ended. `govips`, the library pixa drives libvips
+through, gives no way to stop an encode once it has begun. So an encode keeps
+its slot until it finishes, which can be after `downstream_timeout`.
+
+**Encode times.** The limits on image size bound how long a conversion can take.
+They are 8192 on each side of the output and of a source in any format but SVG,
+and 16,777,216 pixels for an AVIF output. Decoding an 8192x8192 source takes up
+to about 11 seconds. The WebP and JPEG XL encoders run below libvips' default
+effort. AVIF runs at the lowest effort `govips` can set. With one libvips
+thread, as pixa runs it, encoding 8192x8192 random pixels takes up to about:
+
+- 1 second as JPEG;
+- 15 seconds as PNG;
+- 16 seconds as GIF;
+- 23 seconds as WebP with an alpha channel, and 16 without;
+- 27 seconds as JPEG XL with an alpha channel, and 13 without.
+
+An AVIF output of 4096x4096, 8192x2048 or 2048x8192 random pixels takes up to
+about 34 seconds with an alpha channel, and 22 without. At 8192x8192 and quality
+85 it would take about 96 seconds with an alpha channel, and 52 without. The 34
+seconds hold at any quality and at 8 or 16 bits per sample. No encode within the
+limits takes more than about 40 seconds. The default `downstream_timeout` is 60
+seconds. An SVG source is rendered in a process of its own, which pixa stops
+after 20 seconds and limits to 1 GiB of memory.
+
+**Processing and connections.** `max_concurrent_processing` images are decoded
+and encoded at once. A request that finds every slot in use waits up to 10
+seconds for one. If none frees up, it is answered 503
+`server busy, try again later`. `upstream_connections` bounds the fetches in
+progress to all upstream hosts together, and `upstream_connections_per_host`
+those to one host. A fetch holds its connection until its image has been
+processed. A fetch that finds every shared connection in use waits up to 10
+seconds for one. If none frees up, the request is answered 503
+`server busy, try again later`. A fetch that finds every connection to its host
+in use waits for one until `downstream_timeout` ends.
+
+**Client limits.** The login form (`POST /`) is limited to 5 attempts per minute
+per client address. An attempt over the limit is refused with 429 and a
+`Retry-After` header. One client may send `image_requests_per_minute` requests
+per minute (default 600) to `/v1/image/` and `/v1/e/` together, `GET` and
+`HEAD`, cache hits included. A request over the limit is refused with 429 and a
+`Retry-After` header. A browser's CORS preflight is not counted. `0` turns the
+limit off. Both limits count an IPv6 client by its `/64`.
+
+**The client address.** Behind a reverse proxy, pixa takes the client address
+from `X-Forwarded-For` only when the request arrives from an address in
+`trusted_proxies`. Otherwise all users behind the proxy count as one client. The
+address the request arrives from is not always the proxy's own. A proxy on the
+Docker host may connect to pixa over `127.0.0.1`. pixa then sees the gateway of
+the container's Docker network, such as `172.17.0.1` on the default bridge. A
+proxy that connects through another of the host's addresses is seen with that
+address. To be sure, find the address as described under `trusted_proxies` (see
+Configuration). The default `trusted_proxies` is the RFC 1918 ranges, the
+private address ranges. With that default, a client with a private address is
+itself trusted. It can then choose the address it is counted by, through its own
+`X-Forwarded-For`. Setting `trusted_proxies` to only the proxy's address closes
+this. Behind a content delivery network (CDN) or proxy whose addresses are not
+in `trusted_proxies`, all requests arriving from one address count as one
+client. Such a deployment needs a higher `image_requests_per_minute`, or `0`.
+
+**Request limits.** pixa reads a request's headers within 10 seconds and the
+whole request within 30 seconds. Headers are limited to 8 KiB. An idle
+connection is closed after 120 seconds. A form body over 1 MiB is refused
+with 413.
+
+### Storage
+
+- **Sources**: `/cache/sources///`, one file for
+ each source URL
+- **Database**: `/state.sqlite3`
+- **Variants**: `/cache/variants///`
+
+`` and `` are the first and second pairs of characters of the file's
+name. The database is one SQLite file. SQLite is the database library pixa
+embeds; nothing else needs to run. By default the database is in WAL mode,
+SQLite's write-ahead log (see `db_url`). SQLite then keeps `state.sqlite3-wal`
+and `state.sqlite3-shm` beside the file while pixa runs.
+
+The image cache keeps these tables in the database:
+
+- `source_content`: a row for each source, by source key, with:
+ - its size and when it was last used
+ - when it was fetched, and when upstream last sent it or confirmed it
+ unchanged
+ - the `ETag` and `Last-Modified` headers upstream sent with it
+- `variant_content`: a row for each variant, by variant key (column
+ `cache_key`), with:
+ - its size and when it was last used
+ - its source's check time when the variant was made or last renewed
+ - the `ETag` and `Last-Modified` headers upstream sent with the source it
+ was made from
+- `cache_usage`: the total size of the cached sources and variants, which
+ eviction reads
+- `negative_cache`: a row for each source URL in the negative cache, with when
+ it expires
+
+The `logout` table holds the random value that a logout replaces (see Encrypted
+URLs).
+
+pixa reads a cached source's format from its file's first bytes. When they match
+none of the source formats, pixa fetches the source again. Two source URLs that
+serve the same bytes are stored twice, once under each URL. A source is stored
+by writing its file and then its row. When the row cannot be written, the file
+is removed again.
+
+Each source and variant file is written under a temporary name and synced to
+disk before it is renamed into place. So a crash never leaves part of a file
+under its own name.
+
+When pixa starts, and then at the spacing below, it checks the cached files
+against the database. It removes each source or variant whose file is empty, or
+not the size its row records. It logs a warning naming the file. A later request
+for that image fetches the source or makes the variant again. The check also
+drops the rows of variants whose files are missing. When pixa cannot read a
+variant file's size for another reason, it keeps the variant's row and logs a
+warning naming the variant key and the error. During a check, pixa skips a
+directory it cannot list and a file whose size it cannot read. It logs a warning
+naming the path and the error, and goes on with the rest of the check. A file
+renamed or removed after pixa listed its directory gets no warning.
+
+A check reads the database once for every cached file, so the checks are spaced
+out. After a check ends, pixa waits at least ten times as long as that check
+took. Eviction runs every five minutes (below). The next check starts at the
+first of those runs after that wait. After a check that took one minute, the
+next one starts 10 to 15 minutes after that check ended.
+
+A cache hit opens the variant's file and nothing else. It makes no database
+call. The content type comes from the file's first bytes, which are those of the
+format pixa wrote it in. A file whose first bytes are not those of a format pixa
+writes is not served, and the image is converted again. A hit hands the time it
+was served to a background writer and does not wait for the writer. Eviction
+orders by that time. The hit is counted in memory, for the
+`pixa_cache_hits_total` metric. A miss that uses a cached source also hands that
+writer the time it used the source. Once a second the writer writes the waiting
+times in one transaction. Up to 10,000 access times of variants, and as many of
+sources, can wait to be written. A time handed over beyond that is dropped. Its
+image then only looks less recently used to eviction. `script/bench` measures
+how many hits pixa serves per second (see Load Test).
+
+Eviction removes the least recently used sources and variants until the cache is
+within `cache_max_bytes`. It runs after each store, and every five minutes. It
+also removes the expired rows of the negative cache every five minutes, at most
+1000 per statement, so a request waits behind 1000 deletions at most.
+
+### Cache TTL
+
+pixa uses a cached source for 24 hours after upstream last sent it or confirmed
+it unchanged. During that time it serves the source's variants from the cache.
+After that, the next request for an image of that source checks the source with
+upstream first:
+
+- pixa asks upstream to send the source only if it changed. It sends the
+ source's `ETag` as `If-None-Match` and its `Last-Modified` as
+ `If-Modified-Since`, when upstream gave them. With neither, upstream sends the
+ whole image again.
+- When upstream answers 304 Not Modified, pixa uses the cached source and serves
+ its variants for another 24 hours. Nothing is downloaded or converted again.
+- When upstream sends the image, it replaces the cached source. Each variant
+ made from the old image is converted again from the new one when it is next
+ requested.
+- When the check fails, pixa serves the cached image. A failure can be an error
+ status, a timeout or no connection. The failure puts the source URL in the
+ negative cache, so pixa does not ask upstream again for 5 minutes, however
+ often it serves the image.
+
+pixa evicts a source and its variants 30 days after upstream last sent the
+source or confirmed it unchanged. So pixa uses a source whose checks keep
+failing for at most 30 days. Eviction to stay under the disk cache limit can
+remove images sooner.
+
+That eviction can remove a source and keep its variants. Each variant's row
+keeps the `ETag` and `Last-Modified` of the source it was made from. So pixa
+checks such a variant with upstream in the same way after 24 hours:
+
+- When upstream answers 304 Not Modified, pixa serves the cached variant for
+ another 24 hours. Nothing is downloaded or converted again.
+- When upstream sends the image, pixa caches it as the source and converts the
+ variant again.
+- When the check fails, pixa serves the cached variant and puts the source URL
+ in the negative cache.
+
+A variant file's modification time is its source's check time when the variant
+was made or last renewed. So a cache hit reads no database row to know whether
+the variant is older than 24 hours. A request that checks its source with
+upstream counts as a miss.
### Configuration
-Every setting can be given as an environment variable, in a YAML config
-file (`--config`), or both. A variable present in the environment wins over
-the file, even when it is empty, and the file wins over the built-in
-default. The one exception is a variable named in the file's `env:` section:
-it is set while the file loads, so it overrides both the environment the
-process was started with and the file's own key. A variable's value is
-parsed as the same text in the file would be. The three lists take
-comma-separated entries, with the spaces around each trimmed; an empty
-variable is an empty list. A value that does not parse or is invalid aborts
-startup, naming the variable. A variable whose name starts with `PIXA_` but
-is not in the table below, such as a misspelled one or `PIXA_PORT`, aborts
-startup naming it, as an unknown config key does. The one other accepted
-name is `PIXA_CONFIG_PATH`, the config file's path (like `--config`). The
-variables set by the file's `env:` section are checked the same way.
+Every setting can be given as an environment variable, in a config file
+(`--config`), or both. The file is YAML, a plain-text format. A variable present
+in the environment wins over the file, even when it is empty. The file wins over
+the built-in default. pixa reads a variable's value as a quoted string in the
+file. So `PIXA_DEBUG=1` turns debug on, but `debug: 1` in the file aborts
+startup. A setting that holds a list takes comma-separated entries, with the
+spaces around each trimmed. An empty variable is an empty list.
-| Variable | Config key | Meaning |
-| ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- |
-| `PIXA_SIGNING_KEY` | `signing_key` | Required: secret for signed and encrypted URLs and login, 32+ characters |
-| `PORT` | `port` | Port to listen on; default `8080` |
-| `PIXA_STATE_DIR` | `state_dir` | Directory for the database and the disk cache; default `/var/lib/pixa` |
-| `PIXA_DB_URL` | `db_url` | SQLite database URL; default `state.sqlite3` in the state directory |
-| `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of free space |
-| `PIXA_ALLOWLIST_HOSTS` | `allowlist_hosts` | Upstream hosts served without a signature |
-| `PIXA_BLOCKED_NETWORKS` | `blocked_networks` | CIDR ranges never fetched from, on top of the built-in ones |
-| `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 |
-| `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` |
-| `PIXA_UPSTREAM_CONNECTIONS_PER_HOST` | `upstream_connections_per_host` | Concurrent connections per upstream host; default `20` |
-| `PIXA_METRICS_USERNAME` | `metrics.username` | Username for `/metrics`, which is served only when both are set |
-| `PIXA_METRICS_PASSWORD` | `metrics.password` | Password for `/metrics`; set together with the username |
-| `PIXA_SENTRY_DSN` | `sentry_dsn` | Sentry DSN for error reporting; empty disables it |
-| `PIXA_DEBUG` | `debug` | Debug logging and plain-HTTP local development; default `false` |
-| `PIXA_MAINTENANCE_MODE` | `maintenance_mode` | Maintenance flag reported by the health check; default `false` |
+The file's `env:` section is the one exception to that order. pixa sets each
+variable named there while it loads the file. So that variable wins over the
+environment pixa started with, and over the file's key for the same setting.
-Key settings in more detail:
+pixa refuses to start when a value does not parse or is invalid. The error names
+the setting. It also refuses to start when a key in the file is set to null. A
+key written as `key:` with nothing after it is null. Omit the key to use the
+default. It refuses to start when the file has a key that is not in the list
+below, and the error names the key. It refuses to start when a variable starts
+with `PIXA_` but is not in the list below, such as a misspelled one or
+`PIXA_PORT`. The error names the variable. The variables set by the file's
+`env:` section are checked the same way.
-- `access_control_allow_origin` — CORS origin
-- `allowlist_hosts` — list of allowed upstream hosts
-- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection,
- added to the always-enforced built-in ranges (loopback, private,
- link-local, CGNAT, benchmark, NAT64, and the like); an invalid CIDR
- aborts startup
-- `trusted_proxies` — list of CIDR ranges of the reverse proxies in front
- of pixa. `X-Forwarded-For` is believed only when the direct peer falls
- inside one of these ranges; the logged and login-recorded client
- address is then the rightmost forwarded entry that is not itself a
- trusted proxy. Otherwise the direct peer address is used and the header
- is ignored, so a client connecting directly from an address outside
- these ranges cannot spoof its address.
- An omitted key defaults to the RFC 1918 private ranges (`10.0.0.0/8`,
- `172.16.0.0/12`, `192.168.0.0/16`), since pixa is deployed behind a
- proxy on a private network; an explicitly empty list (`[]`) trusts no
- one, and an explicit list replaces the default. An invalid CIDR aborts
- startup. Set this to your proxy's address range if it is not already
- covered by the defaults
-- `upstream_fetch_timeout` — timeout for origin requests
-- `upstream_max_response_size` — max origin response size
-- `downstream_timeout` — client response timeout
-- `signing_key` — HMAC secret for URL signatures
-- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the
- disk cache entirely; omitted defaults to 75% of the free space on
- the filesystem containing `/cache/` (minimum 500 MiB)
+pixa also accepts `PIXA_CONFIG_PATH`, which names the config file, like
+`--config`.
-See `config.example.yml` for all options with defaults.
+pixa reads at most one config file. It reads the file given with `--config` (or
+`-c`), or else the one `PIXA_CONFIG_PATH` names. pixa refuses to start when that
+file does not exist, cannot be read or does not parse. Without `--config` or
+`PIXA_CONFIG_PATH`, pixa looks in `/etc/pixa/`, then in `~/.config/pixa/`, then
+in the working directory. In each directory it looks for `config.yml`, then
+`config.yaml`. It reads the first file it finds. It skips a missing file without
+a message. pixa refuses to start when a file it finds cannot be read or does not
+parse, and the error names the file. It also refuses to start when it may not
+enter one of these directories, whether or not the file is there, since it
+cannot tell. With no file, pixa uses the environment and the defaults.
+
+Durations are written as Go durations, such as `30s` or `2m`, and must be
+positive. A bare number has no unit and aborts startup. Sizes are a whole number
+of bytes. A CIDR range is an address range written as an address and a prefix
+length, such as `10.0.0.0/8`. An address matches a range that contains it.
+
+`blocked_networks` and `trusted_proxies` hold CIDR ranges. For both, an IPv6
+address in the IPv4-mapped form (`::ffff:a.b.c.d`) is matched only as the IPv4
+address it holds. So an IPv6 range such as `::/0` matches neither it nor a plain
+IPv4 address. A range written in that form is read as the IPv4 range it covers:
+`::ffff:10.0.0.0/104` is `10.0.0.0/8`. Such a range shorter than `/96` covers
+more than IPv4 addresses and aborts startup.
+
+Each item below is one setting, named by its config key:
+
+- `signing_key`: variable `PIXA_SIGNING_KEY`, required. The secret for signed
+ URLs, encrypted URLs and login. It must be 32 characters or longer. The
+ placeholder in `configs/config.example.yml` is refused. pixa uses the key as
+ it is for two things: as the HMAC key of signed URLs (see Signed URLs), and as
+ the password of the login page. From it, pixa derives four more keys at
+ startup, with HKDF-SHA256, a key derivation function. One signs the session
+ cookie and one encrypts it. One encrypts the token of an encrypted URL. One
+ signs the token cookie of the login and generator pages' forms.
+- `port`: variable `PORT`, default `8080`. The port to listen on, from 1
+ to 65535.
+- `state_dir`: variable `PIXA_STATE_DIR`, default `/var/lib/pixa`. The directory
+ for the database and the disk cache. pixa creates it at startup when it is
+ missing. It refuses to start when it cannot create or write the directory.
+- `db_url`: variable `PIXA_DB_URL`, default
+ `file:/state.sqlite3?_pragma=journal_mode(WAL)`. The SQLite
+ database to open. The default keeps the database in WAL mode, SQLite's
+ write-ahead log, which lets reads go on during a write. pixa adds
+ `_pragma=busy_timeout(5000)` to any `db_url`. A write that finds another
+ program writing to the file then waits up to five seconds for it instead of
+ failing. pixa opens one connection, so its own reads and writes run one at a
+ time. WAL mode comes only from the URL: keep `_pragma=journal_mode(WAL)` in
+ one you set. An empty `db_url` aborts startup: omit the key to use the
+ default. In the default, `state_dir` is percent-encoded: a `%`, `?` or `#` in
+ the name is written as `%25`, `%3F` or `%23`. For a `state_dir` of `/`, or one
+ starting with `//`, the default starts with `file:////`, so SQLite does not
+ read the first name in the path as a host.
+- `cache_max_bytes`: variable `PIXA_CACHE_MAX_BYTES`, default below. The disk
+ cache limit in bytes. When the key is omitted, pixa works the limit out at
+ startup. It takes the free space on the filesystem holding
+ `/cache/`, adds the bytes the cache already holds, and takes 75% of
+ the sum. The default is at least 500 MiB. A value you set is used exactly as
+ given. `0` turns the disk cache off: every request then fetches and converts
+ its image, and only the negative cache is used. A negative value aborts
+ startup.
+- `allowlist_hosts`: variable `PIXA_ALLOWLIST_HOSTS`, default none. The upstream
+ hosts whose images pixa serves without a signature, written as under Source
+ Hosts. Every image on a listed host is served to anyone, so list only hosts
+ you control. An entry that is neither a host name nor an IP address aborts
+ startup.
+- `referer_blocklist`: variable `PIXA_REFERER_BLOCKLIST`, default none. Hosts
+ whose pages may not show pixa's images, to stop other sites embedding them.
+ Entries are written and matched as for `allowlist_hosts`. A request whose
+ `Referer` header names a listed host is refused with 403. That applies to
+ `/v1/image/` and `/v1/e/`. It happens before the signature or token is
+ checked, and before the cache or the upstream host is used. The request is
+ refused even when the image is cached. A request with no `Referer` is served,
+ as many clients send none. So is one whose `Referer` does not parse as a URL
+ with a host. So a site whose pages send no `Referer`, for example with
+ `Referrer-Policy: no-referrer`, is not stopped. The login and generator pages
+ are not covered.
+- `blocked_networks`: variable `PIXA_BLOCKED_NETWORKS`, default none. CIDR
+ ranges pixa never fetches from, on top of the built-in ones. An invalid CIDR
+ aborts startup. The built-in ranges always apply. They are:
+ - `0.0.0.0/8`, this network, `0.0.0.0` included;
+ - `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`, the private ranges;
+ - `100.64.0.0/10`, carrier-grade NAT;
+ - `127.0.0.0/8`, loopback;
+ - `169.254.0.0/16`, link-local;
+ - `192.0.0.0/24`, IETF protocol assignments;
+ - `198.18.0.0/15`, benchmarking;
+ - `224.0.0.0/4`, multicast;
+ - `::`, the unspecified address;
+ - `::1`, loopback;
+ - `64:ff9b::/96` and `64:ff9b:1::/48`, NAT64;
+ - `fc00::/7`, the unique local (private) range;
+ - `fe80::/10`, link-local;
+ - `ff00::/8`, multicast.
+- `trusted_proxies`: variable `PIXA_TRUSTED_PROXIES`, default the RFC 1918
+ private ranges: `10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`. CIDR
+ ranges of the reverse proxies in front of pixa. `X-Forwarded-For` is believed
+ only when the request reaches pixa from an address in one of these ranges. The
+ client address is then the rightmost entry of the header that is not itself a
+ trusted proxy. Otherwise the header is ignored and the address the request
+ came from is used. So a client connecting from outside these ranges cannot
+ choose its address. An explicitly empty list (`[]`, or an empty variable)
+ trusts no one. A list you set replaces the default. An invalid CIDR aborts
+ startup. Set this to the address pixa sees for requests that come through your
+ proxy, such as `172.17.0.1/32`. Do so when the default does not cover it, or
+ to trust nothing else (see Limits). To find that address, set this to `[]`,
+ send a request through the proxy, and read `remoteIP` in pixa's log line for
+ it.
+- `allow_http`: variable `PIXA_ALLOW_HTTP`, default `false`. While `true`, pixa
+ fetches every source over plain HTTP, and follows redirects to plain HTTP. It
+ is for testing only.
+- `upstream_connections_per_host`: variable
+ `PIXA_UPSTREAM_CONNECTIONS_PER_HOST`, default `20`. The most connections to
+ one upstream host at once. It must be at least 1. A fetch that finds every
+ connection to its host in use waits for one until `downstream_timeout` ends
+ (see Limits).
+- `upstream_connections`: variable `PIXA_UPSTREAM_CONNECTIONS`, default `64`.
+ The most connections to all upstream hosts together at once, on top of the
+ limit per host. It must be at least 1. A fetch holds its connection until its
+ image has been processed (see Limits).
+- `max_concurrent_processing`: variable `PIXA_MAX_CONCURRENT_PROCESSING`,
+ default half the CPUs Go uses (`GOMAXPROCS`), rounded down, and at least 1.
+ The most images decoded and encoded at once. `GOMAXPROCS` follows a
+ container's CPU limit. A value that is not a whole number of at least 1 aborts
+ startup. Each image can take a lot of memory (see Running under upaas).
+- `image_requests_per_minute`: variable `PIXA_IMAGE_REQUESTS_PER_MINUTE`,
+ default `600`. How many requests one client may send per minute to
+ `/v1/image/` and `/v1/e/` together (see Limits). `0` turns the limit off. A
+ negative or non-integer value aborts startup.
+- `upstream_fetch_timeout`: variable `PIXA_UPSTREAM_FETCH_TIMEOUT`, default
+ `30s`. The time allowed for one fetch from an upstream host. A fetch whose
+ response headers have not arrived by then answers 504.
+- `upstream_max_response_size`: variable `PIXA_UPSTREAM_MAX_RESPONSE_SIZE`,
+ default `52428800` (50 MiB). The largest upstream response accepted, in bytes,
+ from 1 to `1073741824` (1 GiB). It also limits the image data pixa decodes. A
+ cached source larger than it is fetched again.
+- `downstream_timeout`: variable `PIXA_DOWNSTREAM_TIMEOUT`, default `60s`. The
+ time allowed for answering one client request. Keep it longer than
+ `upstream_fetch_timeout` plus 20 seconds (see Limits).
+- `access_control_allow_origin`: variable `PIXA_ACCESS_CONTROL_ALLOW_ORIGIN`,
+ default `*`. The origin whose pages may read the responses of `/v1/image/` and
+ `/v1/e/` in a browser. pixa sends it as the CORS `Access-Control-Allow-Origin`
+ header. No other route sends it. `*` is any site. Otherwise it is one `http`
+ or `https` origin such as `https://example.com`. Its host is a lowercase host
+ name (letters, digits, hyphens and dots, with a letter in its last part) or an
+ IP address (IPv6 in brackets, in its shortest form). It may have a port from 1
+ to 65535 that has no leading zero and is not the scheme's default. Any other
+ value aborts startup, including an origin with another scheme, such as a
+ browser extension's.
+- `metrics.username`: variable `PIXA_METRICS_USERNAME`, default empty. The
+ username for `/metrics`. pixa serves `/metrics`, and measures every HTTP
+ request, only when the username and the password are both set. Setting one
+ without the other aborts startup.
+- `metrics.password`: variable `PIXA_METRICS_PASSWORD`, default empty. The
+ password for `/metrics`. Set it together with the username.
+- `sentry_dsn`: variable `PIXA_SENTRY_DSN`, default empty. Sentry is an error
+ reporting service, and the DSN is the URL it gives you. A DSN turns error
+ reporting on. It must be a URL with a scheme and a host, or empty.
+- `debug`: variable `PIXA_DEBUG`, default `false`. Turns on debug logging and
+ plain-HTTP local development (see Routes).
+- `maintenance_mode`: variable `PIXA_MAINTENANCE_MODE`, default `false`. While
+ `true`, `/v1/image/` and `/v1/e/` answer every `GET` and `HEAD` request with
+ 503, a `Retry-After` header of 300 seconds and a JSON error body. The health
+ check still answers 200 and reports `"maintenance_mode": true`. It stays 200
+ because the image's Docker `HEALTHCHECK` requests it, and upaas marks a deploy
+ failed when its container is unhealthy. The login and generator pages and
+ `/metrics` keep working.
+
+`configs/config.example.yml` names every setting with its default.
### Architecture
-- **Dependency injection**: Uber fx
-- **HTTP router**: go-chi
-- **Image processing**: govips (CGO wrapper for libvips)
-- **Database**: SQLite via modernc.org/sqlite
-- **Static assets**: embedded via `//go:embed`
-- **Metrics**: Prometheus
-- **Logging**: stdlib slog
+- **Dependency injection**: `fx` (`go.uber.org/fx`), a library that builds the
+ program's parts in dependency order and starts and stops them. `internal/app`
+ puts every part of `pixad` in one `fx` graph. `internal/providers` builds the
+ image cache, the image proxy and the other parts that serve images and logins.
+ Each setting's default is declared once, in `internal/config`.
+- **Packages**: these packages under `internal/` work with the concepts above:
+ - `handlers`: the functions the routes call, which read the image request
+ from the URL, check the signed or encrypted URL, and make the login
+ session
+ - `imagerequest`: the image request, with the source key and the variant key
+ - `imageproxy`: the image proxy
+ - `imgcache`: the image cache, which keeps its sources in one `FileStore`
+ and its variants in another
+ - `httpfetcher`: fetching sources from upstream hosts
+ - `magic`: reading an image's format from its first bytes
+ - `imageprocessor`: making a variant from a source with libvips
+ - `signature`: checking the `sig` of a signed URL
+ - `encurl`: making and reading the tokens of encrypted URLs
+ - `seal`: the keys derived from `signing_key`, and the encryption of those
+ tokens
+ - `session`: the login session
+- **HTTP router**: `chi` (`github.com/go-chi/chi/v5`)
+- **Image processing**: `govips`, a Go wrapper for libvips
+- **Database**: SQLite via `modernc.org/sqlite`
+- **Static assets**: embedded in the binary via `//go:embed`
+- **Metrics**: Prometheus, at `/metrics`. Prometheus is a monitoring system that
+ reads metrics from that route. pixa serves `/metrics` and measures HTTP
+ requests only when `metrics.username` and `metrics.password` are set.
+ `/metrics` holds:
+ - the duration of HTTP requests
+ - the size of HTTP responses
+ - the HTTP requests in flight
+ - the Go runtime and process metrics
+ - `pixa_cache_hits_total`: image requests served from the cache
+ - `pixa_cache_misses_total`: image requests not served from the cache
+ - `pixa_source_cache_hits_total`: variants made or checked whose source was
+ in the cache
+ - `pixa_source_cache_misses_total`: variants made or checked whose source
+ was not in the cache
+ - `pixa_cache_usage_bytes`: the bytes of sources and variants in the cache,
+ as eviction last read them; eviction runs after each store and every five
+ minutes
+ - `pixa_cache_max_bytes`: the disk cache limit, `cache_max_bytes`
+ - `pixa_cache_evictions_total`: sources and variants removed by eviction
+ - `pixa_cache_evicted_bytes_total`: the bytes those evictions freed
+ - `pixa_upstream_fetches_total`: fetches of a source image that read any
+ bytes
+ - `pixa_upstream_fetch_bytes_total`: the bytes those fetches read
+ - `pixa_upstream_fetch_duration_seconds`: a histogram of how long those
+ fetches took, up to the last byte of the response
+ - `pixa_upstream_fetch_errors_total`: fetches of a source image that failed,
+ before or while reading the response
+ - `pixa_transforms_total`: images converted
+ - `pixa_transform_duration_seconds`: a histogram of how long those
+ conversions took, including the wait for a processing slot
+ - `pixa_transforms_in_progress`: images being converted or waiting for a
+ processing slot
+
+ pixa measures the `pixa_` metrics whether or not `/metrics` is served. They
+ have no labels, so their number never grows with the hosts and paths pixa
+ serves. Requests that share work (see Limits) count one source cache hit or
+ miss between them.
+
+ A request's `handler` label is the pattern of the route it matches, such as
+ `/v1/image/*`, never its path. Requests that match no route share the label
+ `unmatched`: those the router answers 404 or 405, and CORS preflights. A
+ request's `method` label is its method, for these nine: `GET`, `HEAD`,
+ `POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`, `CONNECT` and `TRACE`. Any
+ other method is labelled `other`.
+
+- **Logging**: the standard library's `slog`, as JSON on standard output, or as
+ text when standard output is a terminal.
## Entrypoints
This repository adheres to the
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
-standard: normalized scripts in `script/` are the entrypoints for the
-development workflow, and the Makefile targets are thin shims that call
-them. We provide:
+standard. The scripts in `script/` are the entrypoints for the development
+workflow, and most `Makefile` targets are thin shims that call them. We provide:
-- `script/bootstrap` — install all dependencies (idempotent)
-- `script/setup` — make a fresh clone ready for development
- (bootstrap, then install-precommit)
-- `script/projectname` — output the project name ("pixa")
-- `script/test` — run the test suite
-- `script/lint` — run golangci-lint, always in a container (builds
- `Dockerfile.lint` when run outside one)
-- `script/fmt` — format all code (writes)
-- `script/fmt-check` — check formatting (read-only)
-- `script/check` — run test, lint, and fmt-check
-- `script/docker` — build the Docker image tagged via `script/projectname`
-- `script/docker-smoke` — build the image, start it, wait for it to be healthy
-- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile
- runs the checks, so a green build implies a green repo)
-- `script/precommit` — pre-commit checks (`go mod tidy` guard, then
- `script/check`)
-- `script/install-precommit` — install the git pre-commit hook that
- runs `script/precommit`
+- `script/bootstrap` — install `git`, `make`, Go, Node, Yarn and `prettier`, and
+ download the Go modules. It can be run again at any time. With `--cgo`, it
+ installs a C compiler, `pkg-config`, libvips (with its JPEG XL support) and
+ libheif instead of Node, Yarn and `prettier`. Compiling pixa outside Docker,
+ as `make build` and `script/bench` do, needs those. Docker itself must already
+ be installed.
+- `script/setup` — make a fresh clone ready for development: `script/bootstrap`,
+ then `script/install-precommit`.
+- `script/projectname` — output the project name (`pixa`).
+- `script/test` — run the test suite: build the `test` phase of the
+ `Dockerfile`, writing no image.
+- `script/lint` — run `golangci-lint`: build the `lint` phase of the
+ `Dockerfile`, writing no image. The linter never runs on the host.
+- `script/fmt` — format the Go code with `gofmt` and the markdown with
+ `prettier` (writes).
+- `script/fmt-check` — check the same formatting (read-only), on the host.
+- `script/check` — run `script/test`, `script/lint` and `script/fmt-check`.
+- `script/docker` — build the Docker image tagged via `script/projectname`, with
+ the version from `git describe`. The image's build stage depends on the `lint`
+ and `test` phases, so this runs them too.
+- `script/docker-smoke` — build the image, start it, and wait for it to be
+ healthy.
+- `script/loadtest` — measure `pixad`'s throughput, latency and peak memory. It
+ is a benchmark, not part of `script/check` (see Load Test).
+- `script/bench` — run the Go benchmark that serves cache hits in parallel
+ through the image handler, with Go on the host after `script/bootstrap --cgo`.
+ It is a benchmark, not part of `script/check` (see Load Test).
+- `script/cibuild` — the CI entrypoint: run `script/bootstrap` (without
+ `--cgo`), then `script/check`, then build the image as `script/docker` does.
+- `script/precommit` — run `go mod tidy` and fail when it changes `go.mod` or
+ `go.sum`, then run `script/check`.
+- `script/install-precommit` — install the git pre-commit hook that runs
+ `script/precommit`.
+
+Every `docker build` in these scripts passes a new value of the `CACHEBUST`
+build argument. Every stage of the `Dockerfile` declares it with an
+`ARG CACHEBUST` line. So every `RUN` step after that line, the lint and test
+steps included, runs on every build instead of coming from the build cache. The
+steps above that line install system packages and download the Go modules.
+Docker keeps their layers until the base image, the step itself, `go.mod` or
+`go.sum` changes.
+
+Go's build cache is kept from build to build: the steps that compile mount it,
+so a package whose inputs are unchanged is not compiled again. The test step
+runs `go test`, and reruns it with `-v` after a failure. Both runs pass
+`-count=1`, so no stored test result is replayed. The lint step also mounts
+`golangci-lint`'s cache. That cache reuses a package's findings only while its
+files, its imports, `go.mod`, `.golangci.yml` and the `golangci-lint` version
+are unchanged.
+
+The `test` phase runs on the Debian Go image, with Debian's libvips and libheif.
+The image pixa ships is built and runs on Alpine.
+
+These scripts are the standard copies from `sneak/prompts`:
+
+- `script/check`
+- `script/cibuild`
+- `script/docker`
+- `script/lint`
+- `script/test`
+- `script/setup`
+- `script/install-precommit`
+
+`script/fmt` and `script/fmt-check` are the standard copies with pixa's `gofmt`
+step kept before `prettier`. They also run `yarn` under the Node version that
+`script/bootstrap` pins. `prettier` formats the markdown only, not the HTML
+templates, as it cannot parse a Go template action inside a tag.
+
+## Load Test
+
+`script/loadtest` (or `make loadtest`) measures how fast `pixad` answers and how
+much memory it uses. It is a benchmark, not a check: `script/check` does not run
+it. It needs Docker and Go.
+
+```bash
+script/loadtest # 10 seconds per scenario, 4 clients
+script/loadtest 30s 32 # 30 seconds per scenario, 32 clients
+```
+
+It builds the image with `script/docker`. It also builds the load tool,
+[`vegeta`](https://github.com/tsenart/vegeta), from a pinned commit. Each
+scenario starts a new `pixad` container and a new origin container,
+`cmd/loadtest-origin`. The origin is an upstream host that answers every path
+with the same generated 1600x1200 JPEG. `vegeta` then sends requests from the
+given number of clients. Each client sends its next request as soon as its last
+one is answered. Every request asks for an image resized to 400x300 WebP:
+
+- `hit`: the same image every time, put in the cache first;
+- `miss`: a new source image every time, so `pixad` fetches and converts each
+ one;
+- `herd`: every client asks for each new source image at the same moment. The
+ clients then share one fetch and one conversion (see Limits).
+
+`pixad` refuses upstream hosts with private or local addresses, so the
+containers share a Docker network in `203.0.113.0/24`, a range set aside for
+documentation. A second run on the same Docker host while one is going fails, as
+it cannot create that network.
+
+For each scenario the script prints `vegeta`'s report and two lines of its own:
+
+- `Requests [total, rate, throughput]`: the requests sent, how many were sent
+ per second, and how many were answered successfully per second;
+- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request
+ to the end of its answer. `50`, `90`, `95` and `99` are the 50th, 90th, 95th
+ and 99th percentiles;
+- `Status Codes` and `Error Set`: anything other than `200` means the other
+ numbers are not for the scenario described, such as `503` when `pixad` was
+ busy;
+- `Bytes In`: `0`, as `vegeta` is told not to keep the images it receives;
+- `pixad peak memory (VmHWM)`: the peak resident memory of the `pixad` process
+ since its container started, in kB; for `hit` it includes the request that put
+ the image in the cache;
+- `requests to the origin`: the fetches `pixad` made. That is one for `hit`, one
+ per request for `miss`, and one per image for `herd`: the requests sent
+ divided by the number of clients.
+
+The numbers depend on the machine and on whatever else runs on it.
+
+`script/bench` (or `make bench`) measures cache hits alone, without the network
+and with no `pixad` process. A Go benchmark in `internal/handlers` serves one
+cached image through the `/v1/image/` handler from several goroutines at once,
+one per CPU. A goroutine is a Go thread. The benchmark reports the hits served
+per second as `hits/s`. Each run starts with an empty cache and an empty
+database, a SQLite file of its own, and puts the image in the cache before it
+measures. The benchmark fails if serving the hits ran any statement on the
+database. Like the load test it is not part of `script/check`. It runs Go on the
+host and compiles pixa there. So it needs the C compiler and libraries that
+`script/bootstrap --cgo` installs, but not Docker.
## TODO
-See [TODO.md](TODO.md) for the full prioritized task list.
+See [`TODO.md`](TODO.md) for the full prioritized task list.
## License
-GPL-3.0. See [LICENSE](LICENSE).
+GPL-3.0. See [`LICENSE`](LICENSE).
## Author
diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md
index bc2f161..1067698 100644
--- a/REPO_POLICIES.md
+++ b/REPO_POLICIES.md
@@ -1,6 +1,6 @@
---
title: Repository Policies
-last_modified: 2026-07-06
+last_modified: 2026-10-18
---
This document covers repository structure, tooling, and workflow standards. Code
@@ -54,21 +54,37 @@ style conventions are in separate documents:
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
assumes nothing is present: base tools come from nix, apt, brew, or apk
(detected in that order; apt runs noninteractive). For node it uses the
- installed node if present; otherwise it installs a PINNED node version via
- nvm, first installing nvm itself if missing — from a hash-verified GitHub
- release archive (never `curl | sh`), with bash installed as an explicit
- prerequisite since nvm requires bash. yarn is then pinned via
- `corepack prepare yarn@ --activate`. Never install "latest" or "lts";
- always exact versions. `script/cibuild` runs the CI build: it changes to the
- repo root and runs `docker build .`; the Gitea workflow calls it. Four further
- scripts are our own extensions to the standard: `script/check` runs
- `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
- what the git pre-commit hook runs, and it calls `script/check`;
- `script/install-precommit` installs the git pre-commit hook (the `make hooks`
- target shims to it); and `script/projectname` (literally that filename) simply
- outputs the project's name. Scripts that need the name call
- `script/projectname` — e.g. `script/docker` assembles its image tag from it —
- so those scripts stay byte-identical across all repos. Repo-type-specific
+ installed node only when its major version is the pinned one; otherwise (node
+ missing, or another major, such as the node 24 the CI runner image ships) it
+ installs the PINNED node version via nvm and installs yarn and the packages
+ under it. nvm itself is installed first if missing, from a hash-verified
+ GitHub release archive (never `curl | sh`), with bash installed as an explicit
+ prerequisite since nvm requires bash. Only the major version is compared
+ because the `Dockerfile` stages start from a node image whose exact version is
+ not the pin, and nvm cannot install a prebuilt node on alpine. yarn is pinned
+ via `corepack prepare yarn@ --activate`. Never install "latest" or
+ "lts"; always exact versions. `script/cibuild` runs the CI build: it changes
+ to the repo root, runs `script/bootstrap`, runs `script/check`, and builds the
+ image with the version; the Gitea workflow calls it. **`script/cibuild` runs
+ `script/bootstrap` first**, because the workflow checks out the repo and runs
+ nothing else, while `script/fmt-check` runs the formatter on the host: on a
+ pristine checkout with nothing installed the run dies there, after the
+ containerised gates have passed. **The bootstrap alone is not enough**: when
+ `script/bootstrap` installs node and yarn under nvm it leaves neither on the
+ `PATH` of the shell that called it, so a bare `yarn` either exits 127 or runs
+ under another node. The host entrypoints that need yarn — `script/fmt` and
+ `script/fmt-check` — therefore use the same test as `script/bootstrap`: when
+ the node on `PATH` has the pinned major version they run the `yarn` on `PATH`,
+ and otherwise they load nvm through `$HOME/.nvm/nvm.sh` and run yarn under the
+ pinned node. A runner carrying nothing but docker and git then gets through
+ `script/check`. Four further scripts are our own extensions to the standard:
+ `script/check` runs `script/test`, `script/lint` and `script/fmt-check`;
+ `script/precommit` is what the git pre-commit hook runs, and it calls
+ `script/check`; `script/install-precommit` installs the git pre-commit hook
+ (the `make hooks` target shims to it); and `script/projectname` (literally
+ that filename) simply outputs the project's name. Scripts that need the name
+ call `script/projectname` — e.g. `script/docker` assembles its image tag from
+ it — so those scripts stay byte-identical across all repos. Repo-type-specific
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/precommit`, not in the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README
@@ -89,87 +105,290 @@ style conventions are in separate documents:
contributor should be able to understand the entire development workflow by
reading the Makefile.
-- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
- as a build step so the build fails if the branch is not green. For non-server
- repos, the Dockerfile should bring up a development environment and run
- `make check`. For server repos, `make check` should run as an early build
- stage before the final image is assembled. Dockerfiles install development
- prerequisites by running `script/bootstrap` rather than duplicating installs
- inline; COPY `script/` and the dependency manifests (`package.json` +
- `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
- layer stays cached until dependencies change.
+- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
+ `lint` phase and a `test` phase, with the final stage depending on both so the
+ image cannot be built unless they pass. For non-server repos the final stage
+ brings up a development environment; for server repos it is the runtime image.
+ The gate phases and the build stage start from their pinned base images and
+ install what those images lack either inline, as the canonical Go `Dockerfile`
+ below does for `git`, or by running `script/bootstrap`, as the `prompts`
+ repo's own `Dockerfile` does for its yarn packages. The development
+ environment stage installs development prerequisites by running
+ `script/bootstrap` rather than duplicating its installs inline. A stage that
+ runs `script/bootstrap` COPYs `script/` and the dependency manifests
+ (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
-- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
- repos use a multistage build where linting runs in an independent stage based
- on the `golangci/golangci-lint` image (pinned by hash). This stage runs
- `make fmt-check` and `make lint` before the full build begins. The build stage
- then declares an explicit dependency on the lint stage via
- `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
- linting before proceeding to compilation and tests. This ensures lint failures
- surface in seconds rather than minutes, without blocking on dependency
- download or compilation in the build stage.
+- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
+ no separate lint file. `script/lint` and `script/test` each build one phase
+ and nothing else:
- The standard pattern for a Go repo Dockerfile is:
+ ```sh
+ docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+ --target lint --output type=cacheonly .
+ docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+ --target test --output type=cacheonly .
+ ```
+
+ **A stage that is not the last one in the file is built only when the final
+ stage's chain depends on it, or when `--target` names it.** That is why the
+ two gates are always invoked by name here, and why the final stage carries a
+ `COPY --from=` of a harmless file from each of them: without that edge a
+ plain `docker build .` builds the last stage alone and exits 0 having linted
+ and tested nothing.
+
+ **The gate builds write no image.** With `--output type=cacheonly` the phase
+ runs and a failing step fails the build, but the result is not exported.
+ Nothing uses those images, and writing one out is slow: a Go test phase's
+ image holds the toolchain. A build given neither `--output` nor `-t` writes
+ an untagged image and leaves it dangling, on every developer host and every
+ CI runner. `script/cibuild` and `script/docker` build the image that ships
+ and tag it, so each build replaces the previous image; each assigns the tag
+ on its own line before the build, so `set -e` stops it where
+ `script/projectname` fails.
+
+ Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
+ `eslint`, `prettier` — never through `make lint` or `script/test`, which are
+ themselves a `docker build` and would recurse into a daemon that does not
+ exist in a build step. Formatting is the exception and stays on the host:
+ `script/fmt` writes the working tree, and `script/fmt-check` is its
+ read-only twin.
+
+ **No lint verdict may come from a host invocation of the linter.** On a
+ shared host golangci-lint reads a result cache keyed on file content rather
+ than location, so a second checkout of the same content is served the first
+ one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
+ exit non-zero with `parallel golangci-lint is running` — a status a caller
+ cannot tell from real findings. Both have produced wrong verdicts in this
+ org, in both directions. A container has its own `TMPDIR`, so the lock is
+ out of reach. The lint phase does keep golangci-lint's cache between check
+ builds (below), but it lints every checkout at `/src`, and a finding is
+ taken from that cache only for the same files, settings and golangci-lint
+ version.
+
+- **Any build that runs checks is given a new `CACHEBUST` value.** Docker
+ invalidates a `COPY` layer only when the copied content changes, so on an
+ unchanged tree the check `RUN` is served from cache, nothing executes, and the
+ build still exits 0. Every `docker build` in `script/` therefore passes
+ `--build-arg CACHEBUST="$(date +%s)-$$"`, the time and the script's process
+ id, so no two builds share a value, not even the three in one
+ `script/cibuild`. `script/lint`, `script/test`, `script/cibuild` and
+ `script/docker` are the four, and there is no fifth — `script/check` runs the
+ two gate phases and `script/fmt-check`, and builds no image of its own. In
+ every stage of the `Dockerfile`, `ARG CACHEBUST` comes right after the steps
+ that install system packages and download Go's modules, or right after `FROM`
+ in a stage with neither, and every step that lints, tests or compiles comes
+ after it. A step that installs anything else as well, such as a
+ `script/bootstrap` that also installs node packages, comes after it too. A new
+ value reruns every `RUN` step after that line; `COPY` steps are compared by
+ content and need nothing. Docker keeps the layers above the line until the
+ base image, their own lines or the files they copy change. A stage without the
+ `ARG` line is served from cache on an unchanged tree, so a repository
+ re-vendors the scripts and adds the `ARG` lines in one commit. The scripts do
+ not pass `--no-cache`: it reruns every step too, but it also builds the layers
+ above the line again and empties every cache mount, so Go's build cache and
+ golangci-lint's, which the canonical Go `Dockerfile` below mounts so that they
+ survive between check builds, would start empty every time. A bare
+ `docker build .` is not evidence that anything ran: a sub-second build
+ reporting success is a cache hit, not a result. Never invalidate by pruning —
+ `docker builder prune` and friends destroy a build cache shared with every
+ other build on the host. When a check is added or changed, prove it works by
+ planting a defect it must catch and watching the run fail on it, then revert
+ the defect. A green run alone shows neither that the check ran nor that it
+ covers what it should.
+
+- **The gate phases are separate stages, and the build stage depends on both.**
+ The lint phase is based on the `golangci/golangci-lint` image (pinned by
+ hash), so lint failures surface in seconds rather than after a full compile,
+ and the test phase is based on the Debian Go image. The canonical Go repo
+ `Dockerfile`:
```dockerfile
- # Lint stage — fast feedback on formatting and lint issues
+ # In every stage ARG CACHEBUST comes right after the steps that install
+ # system packages and download Go's modules, or right after FROM in a
+ # stage with neither, and every step that lints, tests or compiles
+ # comes after it. script/ passes a new value on every build, which
+ # reruns every RUN step after it. Docker keeps the layers above it
+ # until the base image, their own lines or go.mod and go.sum change.
+ # The steps that compile mount Go's build cache, and golangci-lint run
+ # also mounts golangci-lint's; both survive from build to build.
+
+ # Lint phase
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
+ ARG CACHEBUST
COPY . .
- RUN make fmt-check
- RUN make lint
+ RUN --mount=type=cache,target=/root/.cache/go-build \
+ --mount=type=cache,target=/root/.cache/golangci-lint \
+ golangci-lint run --config .golangci.yml ./...
- # Build stage
- # golang:1.x-alpine, YYYY-MM-DD
- FROM golang@sha256:... AS builder
+ # Test phase. -race needs cgo and so a C compiler, which the Debian Go
+ # image ships and the alpine one does not. -count=1 because the mounted
+ # cache also holds test results, which go test would otherwise replay.
+ # golang:1.x, YYYY-MM-DD
+ FROM golang@sha256:... AS test
WORKDIR /src
-
- # Force BuildKit to run the lint stage before proceeding
- COPY --from=lint /src/go.sum /dev/null
-
COPY go.mod go.sum ./
RUN go mod download
+ ARG CACHEBUST
COPY . .
- RUN make test
+ RUN --mount=type=cache,target=/root/.cache/go-build \
+ go test -count=1 -timeout 90s -race -cover ./... || \
+ { echo "--- Rerunning with -v for details ---"; \
+ go test -count=1 -timeout 90s -race -v ./...; exit 1; }
- ARG VERSION=dev
- RUN CGO_ENABLED=0 go build -trimpath \
- -ldflags="-s -w -X main.Version=${VERSION}" \
- -o /app ./cmd/app/
+ # Build stage. Nothing is wanted from either phase above; the copies
+ # are what make BuildKit build them first, so nothing after them can
+ # run unless lint and test passed.
+ # golang:1.x-alpine, YYYY-MM-DD
+ FROM golang@sha256:... AS builder
+ RUN apk add --no-cache git
+ WORKDIR /src
+ COPY go.mod go.sum ./
+ RUN go mod download
+ ARG CACHEBUST
+ COPY --from=lint /src/go.sum /dev/null
+ COPY --from=test /src/go.sum /dev/null
+ # A tar-stream context keeps the sender's file owners, which git refuses.
+ RUN git config --system --add safe.directory /src
+ COPY . .
- # Runtime stage
+ # The VERSION build arg when one is given, otherwise
+ # `git describe --tags --always` on the .git in the build context. With
+ # .git present, a version that is still empty, dev or unknown fails the
+ # build: git is missing or could not read the checkout.
+ ARG VERSION
+ RUN --mount=type=cache,target=/root/.cache/go-build \
+ VERSION="${VERSION:-$(git describe --tags --always)}"; \
+ if [ -e .git ]; then \
+ case "$VERSION" in ""|dev|unknown) \
+ echo "version is '$VERSION' although .git is present" >&2; \
+ exit 1 ;; \
+ esac; \
+ fi; \
+ CGO_ENABLED=0 go build -trimpath \
+ -ldflags="-s -w -X main.Version=${VERSION}" \
+ -o /app ./cmd/app/
+
+ # Runtime stage, and the last one
FROM alpine@sha256:...
+ ARG CACHEBUST
COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"]
```
Key points:
- - The lint stage uses the `golangci/golangci-lint` image directly (it
- includes both Go and the linter), so there is no need to install the
- linter separately.
- - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
- a stage dependency. BuildKit runs stages in parallel by default; without
- this line, the build stage would not wait for lint to finish and a lint
- failure might not fail the overall build.
+ - The lint phase uses the `golangci/golangci-lint` image directly (it has
+ both Go and the linter), so nothing needs installing.
+ - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only
+ purpose is the ordering edge. BuildKit runs stages in parallel by default,
+ and a stage nothing depends on is not built at all, so without these two
+ lines a red gate would not fail the build.
+ - Keep the runtime stage last, and if you add a stage after it, give it the
+ same two copies. A plain `docker build .` builds the last stage's chain
+ and nothing else.
+ - Check builds keep these, and none of them can hand back a stale result:
+ - The layers above `ARG CACHEBUST`: the system packages and Go's
+ modules. Docker installs the packages again when the base image or the
+ install line changes, and downloads the modules again when `go.mod` or
+ `go.sum` changes. Go checks every module against `go.sum` as it
+ downloads it, and no later build can change a kept layer. The modules
+ are kept in a layer rather than a cache mount because a cache mount is
+ shared by every build on the host, and Go builds from whatever files
+ it holds without checking them again (only `go mod verify` does), so a
+ file changed there would go unnoticed.
+ - Go's build cache (`/root/.cache/go-build`, where Go keeps it for
+ root), mounted on the steps that compile: `golangci-lint run`, both
+ `go test` runs and `go build`. Go reuses a compiled package only when
+ all its inputs are unchanged. The cache also holds test results, which
+ is why both `go test` runs carry `-count=1`.
+ - golangci-lint's cache (`/root/.cache/golangci-lint`, where the pinned
+ image keeps it), mounted on `golangci-lint run`. golangci-lint reuses
+ a package's findings only when the package's files and those of every
+ package it imports, `go.mod`, the linters and their settings in
+ `.golangci.yml`, and the golangci-lint version are all unchanged.
- If the project uses `//go:embed` directives that reference build artifacts
- (e.g. a web frontend compiled in a separate stage), the lint stage must
+ (e.g. a web frontend compiled in a separate stage), the lint phase must
create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
- The lint stage should not depend on the actual build output — it exists to
- fail fast.
- - If the project requires CGO or system libraries for linting (e.g.
- `vips-dev`), install them in the lint stage with `apk add`.
- - The build stage runs `make test` after compilation setup. Tests run in the
- build stage, not the lint stage, because they may require compiled
- artifacts or heavier dependencies.
+ - If the project requires CGO or system libraries for linting, install them
+ in the lint phase. The `golangci/golangci-lint` image is Debian-based and
+ has no `apk`, so install with `apt-get` under the Debian package name
+ (`libvips-dev`, where alpine says `vips-dev`), and delete the package
+ lists in the same `RUN`, so the layer does not keep them. The test phase,
+ on the Debian Go image, installs the libraries its tests need the same
+ way. Either install goes right after `FROM`, above `ARG CACHEBUST`, so
+ Docker keeps it until the base image or the install line changes:
+
+ ```dockerfile
+ FROM golangci/golangci-lint@sha256:... AS lint
+ RUN apt-get update \
+ && apt-get install -y --no-install-recommends libvips-dev \
+ && rm -rf /var/lib/apt/lists/*
+ WORKDIR /src
+ COPY go.mod go.sum ./
+ RUN go mod download
+ ARG CACHEBUST
+ ```
+
+ - `.dockerignore` lets `.git` into the build context. It keeps out every git
+ `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
+ repository's own, each submodule's under `.git/modules/`, and that of a
+ submodule keeping its own `.git` directory. `git describe` does not need
+ them, and each can hold a credential: a password in a remote URL, or the
+ token the CI checkout step stores there. A submodule whose name has a
+ `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
+ git directory to `**/.git/modules/**/config`, and Go's version stamping
+ then fails the build: give it a name without that segment
+ (`git submodule add --name`). The stage that compiles has `git` (the
+ Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
+ takes the version from the `VERSION` build argument when one is given,
+ otherwise from `git describe --tags --always`. That gives the tag on a
+ tagged commit; on a later commit, the tag, the number of commits since it
+ and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
+ tag is reachable. The stage that compiles also marks its working directory
+ safe for git (`git config --system --add safe.directory /src`): a context
+ sent as a tar stream keeps the sender's file owners, and git refuses a
+ checkout owned by another user, so the version would come out empty.
+ `ARG VERSION` has no default, and the build fails if the context carries
+ `.git` and the version still comes out empty, `dev` or `unknown`. A plain
+ `docker build .` with no build arguments must succeed; a Dockerfile that
+ refuses an empty build argument drops that refusal and keeps the argument.
+ A checkout whose `.git` is a file (a linked worktree, or a repository
+ checked out as a submodule) is the exception: that file points to a git
+ directory outside the build context, so the build cannot read the version
+ and a plain `docker build .` fails; pass the version with
+ `--build-arg VERSION=...`, as `script/docker` and `script/cibuild` already
+ do.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
- runs `script/cibuild` (which runs `docker build .`) on push. Since the
- Dockerfile already runs `make check`, a successful build implies all checks
- pass.
+ runs `script/cibuild` on push, and checks out the repo as its only other step,
+ with `persist-credentials: false`: `script/cibuild` needs no token, and
+ without it the checkout leaves the job's token in `.git/config` for every
+ later step. The checkout step also sets `fetch-depth: 0`, which fetches the
+ tags `git describe` needs: by default it clones shallow with no tags, and a
+ tagged repository's CI build would stamp a bare short commit id. The
+ workflow's `concurrency` block groups runs by workflow and branch
+ (`${{ github.workflow }}-${{ github.ref }}`) with `cancel-in-progress: true`,
+ so a new push cancels the older run on the same branch, queued or running, and
+ no other: runs for replaced commits do not hold up the shared runner.
+ `script/cibuild` bootstraps, runs the gate phases, and then builds the image,
+ so a successful run means every check passed; a bare `docker build .` does not
+ carry the same guarantee, because its gate phases may come from the cache. The
+ image build gets a `CACHEBUST` value of its own and so runs the gate phases a
+ second time. That is the price of the rule above, and it is worth paying: the
+ image that ships is built from a run of its own gates rather than from a cache
+ entry. The `check` job sets `timeout-minutes: 20`, so a hung build frees the
+ shared runner after 20 minutes. That allows for the three Docker builds
+ described above (the test phase, the lint phase, then the image), each held to
+ the 5-minute Docker build limit below, plus the bootstrap. A separate workflow
+ limited to `main` by a `branches` list under `on: push` cannot be checked by
+ review: to try a change to it, add the feature branch to that list and push,
+ then remove the branch from the list again before merging. Keep any job in it
+ that publishes behind `if: github.ref_name == 'main'`, so the run from the
+ feature branch publishes nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -189,14 +408,21 @@ style conventions are in separate documents:
module under test to verify it compiles/parses. There is no excuse for
`make test` to be a no-op.
-- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
- Makefile.
+- `make test` must complete in under 60 seconds. That is the hard cap, and a
+ suite that exceeds it fails. Under 20 seconds is the target. A suite between
+ 20 and 60 seconds is still green, but the overage must be filed as an
+ improvement bug against that repo. Add a 90-second timeout to the test
+ invocation (`go test -timeout 90s`). The backstop deliberately sits above the
+ hard cap so that it catches a genuinely hung test rather than a merely slow
+ one.
-- **`make test` should use the conditional verbose rerun pattern.** Run tests
- without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
- show full output. This keeps CI logs and `docker build` output clean on
- success (just package/suite summaries) while providing full diagnostic detail
- on failure (every test case, every assertion). The general shell pattern:
+- **The test command should use the conditional verbose rerun pattern.** Run
+ tests without `-v` (verbose) first. If tests fail, automatically rerun with
+ `-v` to show full output. This keeps CI logs and `docker build` output clean
+ on success (just package/suite summaries) while providing full diagnostic
+ detail on failure (every test case, every assertion). The command lives in the
+ `test` phase of the `Dockerfile`, since `script/test` builds that phase; the
+ Makefile form below is the same pattern for any repo-local invocation:
```makefile
test:
@@ -209,11 +435,25 @@ style conventions are in separate documents:
```makefile
test:
- @go test -timeout 30s -race -cover ./... || \
+ @go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
- go test -timeout 30s -race -v ./...; exit 1; }
+ go test -count=1 -timeout 90s -race -v ./...; exit 1; }
```
+ `-count=1` is required on both invocations: it defeats Go's test _result_
+ cache, so neither run can report a stored pass in place of running the
+ tests. It leaves the build cache alone, so it costs the runtime of the suite
+ and no recompilation.
+
+ That cache is Go's own, separate from Docker's layer cache. Go stores a
+ passing result in its cache directory (`GOCACHE`), and when the same tests
+ run again on unchanged code it prints that result, marked `(cached)`,
+ without running them. The directory lasts from one run to the next on a
+ developer's machine, where this target runs, and in the `test` phase of the
+ `Dockerfile`, which mounts it to keep compiled packages between check
+ builds, so the phase's `go test` runs carry `-count=1` too. A new
+ `CACHEBUST` value (above) is what makes that step run on an unchanged tree.
+
Python example:
```makefile
@@ -239,10 +479,89 @@ style conventions are in separate documents:
must be in `.gitignore`. No exceptions.
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
- editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
- Fetch the standard `.gitignore` from
+ editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
+ `node_modules/`, and the repo's own build outputs. Fetch the standard
+ `.gitignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
- a new repo.
+ a new repo. A repo's `.gitignore` is the standard file followed by the repo's
+ own entries, such as its binaries; a re-vendor replaces the standard part and
+ keeps those entries. These patterns are written to `.gitignore`'s own
+ semantics, in which an unanchored pattern already matches at every depth; they
+ are not a `.dockerignore` and must not be transplanted into one unmodified.
+
+- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
+ across unmodified leaves secrets in the build context.** Docker matches with
+ `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
+ `*` does not cross `/` and a pattern without a leading `**/` is anchored at
+ the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
+ therefore excludes only the copies at the repository root, while `config/.env`
+ and `certs/server.key` still reach the context and can land in an image layer
+ — which is more dangerous than a short file with no secret patterns at all,
+ because it reads as solved and stops anyone looking. Give every
+ depth-independent pattern the `**/` prefix and leave only genuinely
+ root-anchored entries unprefixed: `.claude`, and the repo's own host-built
+ binary, written `/myapp` and never `**/myapp`, which would also match
+ `cmd/myapp/` and delete the package directory from the context. Matching is
+ case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
+ secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
+ and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
+ also catches something the build needs, re-include it with a negation
+ (`!docs/example.env`); deleting the pattern reopens the exposure for every
+ other file it covers. Fetch the standard `.dockerignore` from
+ `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
+ it with the repo's own artifacts.
+
+- **In-repo agent scratch belongs in both files, written to each file's own
+ semantics.** `.claude/` holds one worktree per in-flight agent — an entire
+ additional checkout of the repo — so under `COPY . .` the build context
+ inflates by a multiple of the repo and another session's unreviewed work can
+ be copied into an image layer. In `.gitignore` the entry is `.claude/`,
+ unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
+ prefix, because the prefixed form would also delete any nested directory of
+ that name from the build. Anchoring carries a known gap that the canonical
+ `.dockerignore` states in its own comment, since consuming repos receive the
+ file and not the tracker: the directory is created in the agent's working
+ directory, so a repo running agents in subdirectories still ships
+ `services/api/.claude/` and must add its own anchored entry there.
+
+- **A plain `docker build .` of a clone stamps the version that
+ `git describe --tags --always` gives**, derived from the `.git` in the build
+ context as the canonical `Dockerfile` above shows. Without its failure check,
+ a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
+ and the build would still exit 0. `script/docker` and `script/cibuild` pass
+ the version they compute on the host; it takes precedence. They do this
+ byte-identically across repos:
+
+ ```sh
+ # The version and the tag each get their own line: a failing command
+ # substitution inside an argument does not trip `set -e`, so the inline
+ # form degrades to an empty constant.
+ version="$(git describe --tags --always --dirty 2>/dev/null || true)"
+ [ -n "$version" ] || version="unknown"
+ tag="$(script/projectname)"
+ docker build --build-arg CACHEBUST="$(date +%s)-$$" \
+ --build-arg VERSION="$version" \
+ -t "$tag" .
+ ```
+
+ `--always` makes an untagged repo yield an abbreviated commit hash rather
+ than failing, and the `[ -n "$version" ]` line is the single place the
+ fallback is applied — a live check that fires on a build from an export with
+ no `.git` and on a repository with no commits yet. Do not fold it into the
+ substitution as `|| echo unknown`, which makes the guard unreachable. The
+ Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
+ there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
+ Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
+ the scripts stay byte-identical. One consequence for CI: the standard
+ checkout action clones shallow and fetches no tags, so the canonical
+ `.gitea/workflows/check.yml` sets `fetch-depth: 0` on its checkout step.
+
+- **Verify `.dockerignore` by enumerating the image, not by reading the
+ patterns.** Plant files at the root _and_ at least two directories deep, build
+ a probe image that does `COPY . .`, and list what actually landed
+ (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
+ size is not a substitute: a nested secret is a few bytes, and BuildKit
+ transfers only the delta from the previous build.
- **No build artifacts in version control.** Code-derived data (compiled
bundles, minified output, generated assets) must never be committed to the
@@ -258,9 +577,61 @@ style conventions are in separate documents:
- Make all changes on a feature branch. You can do whatever you want on a
feature branch.
-- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
- manually by the user. Fetch from
- `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
+- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
+ _NEVER_ be modified by an agent: fetch it from
+ `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
+ byte-identical, so that no repo can quietly loosen its own linting. Linter
+ configuration changes are made to the canonical copy in the `prompts` repo and
+ reach consuming repos by re-vendoring; an agent may open a PR against
+ canonical, which only the user merges. One list is exempt from byte-identity,
+ because it cannot be written once for every repo: the `deny` list of the
+ `test-support` depguard rule, where a repo names its own test-support packages
+ by full import path. A repo adds entries there and changes nothing else, and a
+ re-vendor carries its entries forward. The canonical golangci-lint version is
+ v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
+ image
+ (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
+ which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
+ directive must not name a newer Go minor version than the one golangci-lint
+ was built with, or golangci-lint refuses to lint it: this release lints
+ `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
+ installs golangci-lint on the host. A repo sets the lint phase digest to the
+ one named here and re-vendors `.golangci.yml` in the same commit, whichever of
+ the two prompted the change: the canonical copy can name linters that an older
+ golangci-lint rejects, and a newer golangci-lint can add linters that
+ `default: all` switches on until the canonical copy disables them.
+
+- **`script/bootstrap` installs a pinned tool by comparing versions, never by
+ testing presence.** An `if ! command -v ; then install; fi` guard tests
+ `PATH` only, so on an already-provisioned machine the pin is inert and a
+ version bump is a silent no-op — while the Dockerfile, installing into a clean
+ image, gets the pinned version, so a local `make check` and `make docker` can
+ disagree about what the tool even is. The canonical form:
+ - compares the installed version against the pin over the **whole** version
+ token; a parser that stops at the first `-` reports `2.12.2` for a host
+ running `2.12.2-rc1` and skips the install;
+ - treats absent, non-zero, empty or unrecognised `--version` output as a
+ mismatch, so the failure direction is a redundant install and never a
+ skipped one;
+ - after installing, re-resolves the binary the way callers do — `hash -r`,
+ then through `PATH`, not through the directory the installer wrote to —
+ and fails naming the resolved path, since an install that a shadowing
+ binary hides succeeds while changing nothing any caller sees;
+ - is actually called, and prints the version on both success paths: a
+ function defined and never invoked has the same exit status and the same
+ empty output as one that worked.
+
+ Node is the exception to the whole-token comparison: the canonical
+ `script/bootstrap` compares only its major version, because the node 22
+ alpine image the `Dockerfile` stages start from is not the exact pin, and
+ nvm cannot install a prebuilt node on alpine.
+
+ Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
+
+ A Go tool a repo needs on the host is installed with `go install` pinned to
+ a commit hash (`go install @`). It is never tracked as
+ a `go.mod` tool dependency or through a `tools.go` file, either of which
+ pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
- When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD).
@@ -371,15 +742,21 @@ style conventions are in separate documents:
Never edit existing migrations after release.
- All repos should have an `.editorconfig` enforcing the project's indentation
- settings.
+ settings: the standard file from
+ `https://git.eeqj.de/sneak/prompts/raw/branch/main/.editorconfig`, which sets
+ tabs for `Makefile` and Go files, followed by the repo's own sections, such as
+ one for another language it uses. A re-vendor replaces the standard part and
+ keeps those sections.
- Avoid putting files in the repo root unless necessary. Root should contain
- only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
- `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
- language-specific config). Everything else goes in a subdirectory. Canonical
- subdirectory names:
+ only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
+ `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
+ and language-specific config). Everything else goes in a subdirectory.
+ Canonical subdirectory names:
- `bin/` — executable scripts and tools
- - `cmd/` — Go command entrypoints
+ - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
+ body is a single call into `internal/` or `pkg/`, no project logic in
+ `cmd/`
- `configs/` — configuration templates and examples
- `deploy/` — deployment manifests (k8s, compose, terraform)
- `docs/` — documentation and markdown (README.md stays in root)
@@ -406,3 +783,7 @@ style conventions are in separate documents:
- Go: `go.mod`, `go.sum`, `.golangci.yml`
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
- Python: `pyproject.toml`
+
+- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
+ is never committed under a file or directory named after one agent tool, such
+ as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
diff --git a/TODO.md b/TODO.md
index 56d0745..b30de34 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,282 +1,31 @@
# Workflow
-* branch per issue from `next`
-* do the work in Next Step
-* move Next Step to the top of Completed Steps
-* move the top item of Future Steps into Next Step
-* commit (`TODO.md` changes in the same commit as the work)
-* open a PR based on `next`
-* an independent reviewer who did not write the change gates it
-* the manager squash-merges the PR into `next` once review passes
-* `next` stays green and mergeable to `main` at any time; only the owner
- merges `next` into `main`, via the single milestone PR
-* push
+- branch per issue from `next`
+- do the work in Next Step
+- delete Next Step from `TODO.md`
+- `TODO.md` merges with git's union merge (`.gitattributes`), which never
+ reports a conflict: read the merged entries after every merge or rebase
+- move the top item of Future Steps into Next Step
+- commit (`TODO.md` changes in the same commit as the work)
+- open a PR based on `next`
+- an independent reviewer who did not write the change gates it
+- the manager squash-merges the PR into `next` once review passes
+- `next` stays green and mergeable to `main` at any time; only the owner merges
+ `next` into `main`, via the single milestone PR
+- push
# Status
-pre-1.0. No git tags exist. The `1.0.0` milestone is in progress; work
-lands on `next`, and `main` receives only the milestone PR that the
-owner merges. `next` is at the canonical `golangci-lint` v2.12.2 config
-and is green. Recent work extracted the internal/magic,
-internal/allowlist, internal/httpfetcher, and internal/signature
-packages. The gosec findings from the 2026-07-06 survey are resolved.
-The disk cache is now size-bounded with LRU eviction
-(`cache_max_bytes`), closing the unbounded disk growth DoS vector.
+pre-1.0. The `1.0.0` milestone is in progress.
# Next Step
-P1: rate limit global concurrent upstream fetches to prevent resource
-exhaustion
-
-# Completed Steps
-
-- 2026-09-28 strip metadata from processed images (closes #82): every output is
- exported with govips' `StripMetadata`, so it carries no EXIF, XMP, IPTC or ICC
- profile; the image is first turned upright with `AutoRotate` (before sizes are
- worked out) and, when it has an ICC profile, converted to sRGB; the `orig`
- format is re-encoded and stripped like any other, as pixa never serves the
- source bytes; there is no setting to keep metadata; documented in `README.md`.
-- 2026-09-28 rate limit the login form (closes #66): `POST /` is limited to 5
- attempts per minute per client address, and an attempt over the limit is
- refused with 429 and a `Retry-After` header; the address is the one
- `internal/clientip` resolves through `trusted_proxies`, an IPv6 client is
- counted by its /64, and an IPv4-mapped address as the IPv4 address it
- carries; the limit is a `RateLimit` middleware in `internal/middleware` on
- `github.com/go-chi/httprate`, which the image routes can reuse; the library
- keeps counts for the current and the previous minute only; documented in
- `README.md`.
-- 2026-09-28 refuse an unparseable `exp` on `/v1/image/` and log swallowed
- cache errors (closes #72): an `exp` in the URL that is not a whole
- number, an empty `exp=` included, is a 400 naming `exp` and the value,
- instead of being ignored and answered with 401 as if the URL had no
- `exp`; only an `exp` missing from the URL is unchanged; `README.md` says
- so where it documents `exp`. A failed variant `.meta` write, source
- metadata JSON write, `Stats` count query, stats counter update, negative
- cache write or expired negative cache delete is now logged at `warn`
- with the path or key and the error, and stays non-fatal.
-- 2026-09-28 refuse an empty `fit` on `/v1/image/` (closes #139): a
- `fit` in the URL with an empty value (`fit=`) is a 400 naming `fit`,
- instead of being served as `cover` and verified against a signature
- made for `cover`; only a `fit` missing from the URL is still `cover`;
- any other value still goes through the existing fit-mode check;
- `README.md` says so where it documents `fit`.
-- 2026-09-28 refuse an invalid `q` on `/v1/image/` (closes #134): a `q`
- that is not a whole number from 1 to 100, an empty `q` included, is a
- 400 naming `q` and the value, instead of being served at the default
- 85; the route reads `q` with the generator's quality check
- (`parseFormInt` with `minQuality` and `maxQuality`); only a `q` missing
- from the URL is still 85; a query string that cannot be decoded, such
- as `q=80%`, is a 400 showing it; any query parameter given more than
- once (`q`, `fit`, `sig`, `exp` alike) is a 400 naming it, so none is
- read from its first value only; `README.md` states the range and both
- query-string rules.
-- 2026-09-28 unknown `PIXA_` environment variables abort startup (closes
- #133): a variable whose name starts with `PIXA_` but is neither a
- setting's variable nor `PIXA_CONFIG_PATH` aborts startup naming it, as
- an unknown config key does, and `PIXA_PORT` is named with a pointer to
- `PORT`; the check runs after the config file loads, so the variables
- the file's `env:` section sets are checked too; documented in
- `README.md`.
-- 2026-09-28 start on a fresh upaas volume (closes #129): the image
- starts as root only to give `/var/lib/pixa` to `pixad` when `pixad`
- does not own it (`deploy/docker-entrypoint.sh`), then runs the server
- as `pixad` through `su-exec`, so a root-owned host directory
- bind-mounted there no longer stops the container at startup;
- `README.md` gains a "Running under upaas" section.
-- 2026-09-28 run all linting in Docker via `Dockerfile.lint` +
- `script/lint` (closes #104): `make lint` calls `script/lint`, the only
- way the linter is run; inside a container (both Dockerfiles set
- `container=docker`) it runs `golangci-lint`, anywhere else it builds the
- hash-pinned `Dockerfile.lint`, whose last step runs `script/lint` again;
- the `Dockerfile` lint stage runs `make lint`; no host or nix-shell
- `golangci-lint` path remains (`script/bootstrap` installs no linter);
- a per-run `CACHEBUST` build-arg keeps the lint step from being served
- from cache, and a tmpfs mount on that step keeps Go's and
- golangci-lint's caches out of its layer, so a run leaves no large build
- cache behind; `golangci-lint config verify` stays out, as it fetches its
- schema over an unpinned live HTTPS call
-- 2026-09-28 every setting as an environment variable (closes #128, also
- covers #99): each config key can be set by `PIXA_` plus the key in upper
- case (`.` written as `_`), and the port by `PORT`; a variable present in
- the environment, even empty, wins over the config file, which wins over
- the default; the typed getters read the variable first, so every existing
- check applies to it and a bad value aborts startup naming the variable;
- lists are comma-separated, and an empty variable (or `""` in the file) is
- an empty list; the Docker image no longer bakes in `config.docker.yml` or
- passes `--config`, and its `HEALTHCHECK` probes `PORT` (default `8080`);
- the config file is looked for under `/etc/pixa` and `~/.config/pixa`
- instead of the daemon name `pixad`; documented in `README.md` and
- `config.example.yml`.
-- 2026-09-28 quality and fit in the URL signature (closes #60): the signed
- data is now `host:path:query:width:height:format:expiration:quality:fit`,
- using `85` and `cover` when the URL has no `q` or `fit`, so one signed
- URL can no longer be replayed across other quality and fit values to
- create unauthorized cache entries and transcodes; the known-answer
- vectors in `internal/signature/golden_test.go` and the README signature
- specification describe the new format.
-- 2026-09-28 Docker image healthcheck (closes #111): a `HEALTHCHECK` in
- the runtime stage probing `/.well-known/healthcheck.json` with busybox
- `wget`; `script/docker-smoke` (`make docker-smoke`) builds the image,
- starts it with a throwaway `PIXA_SIGNING_KEY`, and passes only once
- Docker reports it healthy within 30 seconds, removing the container on
- exit; the Gitea workflow runs it after `script/cibuild`.
-- 2026-09-21 trusted-proxy client IP resolution (closes #94): a
- `trusted_proxies` config key taking a list of CIDRs, parsed by the same
- `net/netip` list parser as `blocked_networks` (an invalid entry aborts
- startup naming the key and value; an omitted key defaults to the RFC 1918
- private ranges, an explicitly empty list trusts no one, and an explicit
- list replaces the default); a new
- `internal/clientip` package resolves the client address by honoring
- `X-Forwarded-For` only when the direct peer is a trusted proxy, walking
- the chain right-to-left to the rightmost non-proxy entry, so a client
- connecting directly cannot spoof its address; the resolved address is
- stored in the request context by a new middleware and used by the
- request-logging middleware and the login-attempt logs in place of the
- raw peer address; documented in `README.md` and `config.example.yml`.
-- 2026-09-21 blocked networks configuration extending SSRF protection: a
- `blocked_networks` config key taking a list of CIDRs (parsed with
- `net/netip`, an invalid entry aborts startup naming the key and value),
- added to the built-in blocklist rather than replacing it; the built-in
- ranges extended to CGNAT `100.64.0.0/10`, IETF protocol assignments
- `192.0.0.0/24`, benchmark `198.18.0.0/15`, and NAT64 `64:ff9b::/96`
- (IPv4-mapped forms covered); enforcement stays in the dial-time
- re-resolution so the DNS-rebinding window remains closed; documented in
- `README.md` and `config.example.yml`.
-- 2026-09-21 validate dimensions and fit mode on the encrypted-URL
- route and the token generator (closes #62): `imgcache.ValidateDimension`
- alone holds the `MaxDimension` bound and is used by the path parser, by
- the new `ValidateImageRequest` (which also applies `ValidateFitMode`)
- and by the generator; both the `/v1/image/` and `/v1/e/` routes call
- `ValidateImageRequest`, so an over-limit size or an unknown fit mode is a
- 400 rather than an out-of-memory or a 500 from the processor; the URL
- generator answers 400 naming the field for a `width` or `height` that is
- not a number or fails the shared check, a `quality` that is not a number
- from 1 to 100, a `ttl` that is not a number from 0 to the largest number
- of seconds the expiry calculation can hold, or an unknown `fit`; an empty
- `quality` is 85 and
- an empty `ttl` never expires; the form's width and height inputs stop at
- 8192
-- 2026-09-21 http.Server hardening (closes #92): added
- `HTTPReadHeaderTimeout` (10s, bounds the slowloris header dribble) and
- `HTTPIdleTimeout` (120s, bounds keep-alive reuse) alongside the
- existing timeouts and wired them onto the server; added a `LimitBody`
- middleware capping the two form POST bodies (`POST /`, `POST /generate`)
- at `MaxFormBytes` (1 MiB) and returning 413, applied ahead of the CSRF
- middleware so an oversized body is refused as 413 rather than being read
- as a missing CSRF token (403); left `WriteTimeout` at 60s unchanged
-- 2026-08-07 update golangci-lint to v2.12.2 with the canonical
- `.golangci.yml` (v2 schema, `default: all` minus six disabled
- linters, `lll` 88, tests included): bumped the pinned
- `golangci/golangci-lint:v2.12.2-alpine` image in `Dockerfile` and the
- release-archive sha256 pins in `script/bootstrap`; fixed the findings
- the stricter config surfaced (notably `paralleltest`, `wsl_v5`,
- `goconst`, `lll`, `noinlineerr`, `err113`, `errcheck`, `testpackage` —
- white-box test files renamed to `*_internal_test.go`), including #55's
- code absorbed after it merged, iterating the pinned linter to
- `0 issues.`; no single finding total is substantiable, since
- golangci-lint's `uniq-by-line` reveals new findings on a line as
- others there are fixed — the documented re-measurements were 81 after
- the #53 merge and 149 after the #55 merge; three behavior changes, so
- not a pure no-op: `Cache.StoreVariant` now takes a `context.Context`
- (`noctx`), so a cancelled request skips its best-effort accounting
- row; `MetadataStorage.Store`'s cleanup defer was dead on `main` and
- leaked `.tmp-*.json` on failure, now fixed with explicit removals; and
- the `signing_key` validation error text gained `value too short: `;
- the eviction loop's uncancellable context is deferred to #102 under a
- `//nolint:contextcheck`; three `//nolint:tagliatelle` directives keep
- the snake_case JSON wire/disk formats unchanged; `make check` green
-- 2026-08-07 implement cache size management and eviction (closes
- #51): new `cache_max_bytes` config key validated by the startup
- framework (explicit values used exactly with no floor, `0` disables
- the disk cache entirely, omitted defaults to max(75% of free space
- on the filesystem containing `/cache/`, 500 MiB), logged
- at startup); processed variants are now tracked in the database (a
- new `variant_content` table and an LRU timestamp on `source_content`)
- so total usage is two SUMs, never a directory scan on the hot path; a
- background goroutine evicts globally least-recently-used entries
- (variants and source blobs merged) to the limit, woken by a periodic
- ticker and by write-pressure notifications from stores; a source
- blob and ALL of its `source_metadata` references are deleted in one
- transaction before the file is unlinked, so multi-referenced blobs
- are never removed while referenced and rows never point at deleted
- files; a startup and periodic reconciliation pass adopts untracked
- variant files, drops rows for missing files, removes unreachable
- source blobs, and sweeps stale temp files
-- 2026-08-07 validate configuration on startup, fail fast on bad
- config (closes #52): a config value that is set but unparseable or
- invalid aborts startup naming the key and value (defaults apply only
- to omitted keys), unknown config keys abort startup, a malformed
- config file aborts instead of being skipped, and `state_dir` is
- verified creatable and writable before the listener binds
-- 2026-08-07 manual test pass of the auth and encrypted URL flows
- against a locally built and running `pixad` (built from `main` at
- `6573b9d`, port 18099, local throwaway config); all six checks
- passed, plus all nine tests in `scripts/manual-test.sh` (closes #49):
- - [x] visit `/` and see the login form: HTTP 200, `Pixa - Login`
- page with `name="key"` password form
- - [x] wrong key shows an error: POST `/` with `key=wrong-key`
- returned HTTP 200 login page containing "Invalid signing key"
- - [x] correct signing key shows the generator form: POST `/`
- returned HTTP 303 to `/` with
- `Set-Cookie: pixa_session=...; HttpOnly; Secure; SameSite=Strict`;
- GET `/` with that cookie rendered `Pixa - URL Generator` with the
- `/generate` form and logout link
- - [x] a generated encrypted URL serves the image: POST `/generate`
- (ttl=3600) produced a `/v1/e//img.jpeg` URL that returned
- HTTP 200, `Content-Type: image/jpeg`, an 800x600 baseline JPEG of
- 61706 bytes
- - [x] an expired URL (short TTL) returns 410: a ttl=1 URL fetched
- after 3 s returned HTTP 410 Gone with
- `{"error":"URL has expired","status":410,...}`
- - [x] logout redirects back to login: GET `/logout` returned HTTP
- 303 to `/` with `Set-Cookie: pixa_session=; Max-Age=0`;
- subsequent GET `/` rendered the login form again
-- 2026-08-07 fix the two remaining gosec findings (G124 in
- internal/session): session cookies now always carry
- Secure/HttpOnly/SameSite=Strict on both the set and clear paths;
- `make check` green (closes #47)
-- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
- Makefile shims, README Entrypoints section
-- 2026-04-07 extract magic byte detection into internal/magic (#42)
-- 2026-03-25 extract allowlist package from internal/imgcache (#41)
-- 2026-03-25 move schema_migrations table creation into 000.sql (#36)
-- 2026-03-20 enforce and document exact-match-only signature
- verification (#40)
-- 2026-03-20 bound imageprocessor.Process input read to prevent
- unbounded memory use (#37); consolidate appname into an
- internal/globals constant (#34)
-- 2026-03-18 parse version prefix from migration filenames (#33)
-- 2026-03-15 QA audit fixes for 1.0/MVP readiness (#25)
-- 2026-03-02 split Dockerfile with pre-built golangci-lint stage for
- faster CI (#23)
-- 2026-02-25 repo policy compliance: CI workflow, hash-pinned images,
- golangci-lint and gosec fixes of that date (#14); arm64 Docker build
- fix (#16)
-- 2026-01-08 WebP and AVIF encoding support via govips (both former P0
- image processing items, now done)
+P2: security: per-origin rate limiting
# Future Steps
-- P2: security
- - referer blacklist
- - per-IP rate limiting on the image routes
- - per-origin rate limiting
- P2: HTTP response handling
- Last-Modified headers
- - Vary header for content negotiation
- - X-Request-ID propagation
-- P2: auto format selection (format=auto based on Accept header)
-- P2: configuration
- - add all configuration options from README
- - YAML config file support
- P2: operational
- - optional Sentry error reporting
- - comprehensive request logging
- - Prometheus performance metrics
- - integration tests for the image proxy flow
- - load tests to verify the 1k to 5k req/s target
-- P2: documentation
- - configuration options
- - API endpoints
- - deployment guide
- - example nginx or caddy reverse proxy config
+ - measure the 1k to 5k req/s target with `script/loadtest` on a machine not
+ shared with other work
diff --git a/cmd/loadtest-origin/main.go b/cmd/loadtest-origin/main.go
new file mode 100644
index 0000000..5a8a93c
--- /dev/null
+++ b/cmd/loadtest-origin/main.go
@@ -0,0 +1,9 @@
+// Command loadtest-origin is the upstream host script/loadtest points pixad
+// at; internal/loadtestorigin says what it does.
+package main
+
+import "sneak.berlin/go/pixa/internal/loadtestorigin"
+
+func main() {
+ loadtestorigin.Run()
+}
diff --git a/cmd/pixad/main.go b/cmd/pixad/main.go
index 601c1ee..16a5a48 100644
--- a/cmd/pixad/main.go
+++ b/cmd/pixad/main.go
@@ -1,61 +1,21 @@
-// Package main is the entry point for the pixad image proxy server.
+// Package main is the entry point for the pixad image proxy server. pixad
+// also runs as a child process of itself, to read the size of an SVG source
+// or to render it (see imageprocessor.RunAsSVGChildProcess).
package main
import (
- "fmt"
"os"
- "github.com/spf13/cobra"
- "go.uber.org/fx"
- "sneak.berlin/go/pixa/internal/config"
- "sneak.berlin/go/pixa/internal/database"
- "sneak.berlin/go/pixa/internal/globals"
- "sneak.berlin/go/pixa/internal/handlers"
- "sneak.berlin/go/pixa/internal/healthcheck"
- "sneak.berlin/go/pixa/internal/logger"
- "sneak.berlin/go/pixa/internal/middleware"
- "sneak.berlin/go/pixa/internal/server"
+ "sneak.berlin/go/pixa/internal/app"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
)
var Version string //nolint:gochecknoglobals // set by ldflags
-var configPath string //nolint:gochecknoglobals // cobra flag
-
func main() {
- rootCmd := &cobra.Command{
- Use: "pixad",
- Short: "Pixa image caching proxy server",
- Run: run,
+ if imageprocessor.IsSVGChildProcess(os.Args[1:]) {
+ os.Exit(imageprocessor.RunAsSVGChildProcess(os.Args[1:]))
}
- rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
-
- err := rootCmd.Execute()
- if err != nil {
- fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
- }
-}
-
-func run(_ *cobra.Command, _ []string) {
- globals.Version = Version
-
- // Set config path in environment if specified via flag
- if configPath != "" {
- _ = os.Setenv("PIXA_CONFIG_PATH", configPath)
- }
-
- fx.New(
- fx.Provide(
- config.New,
- database.New,
- globals.New,
- handlers.New,
- logger.New,
- server.New,
- middleware.New,
- healthcheck.New,
- ),
- fx.Invoke(func(*server.Server) {}),
- ).Run()
+ app.Run(Version)
}
diff --git a/config.example.yml b/config.example.yml
deleted file mode 100644
index 8df6a35..0000000
--- a/config.example.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-# Pixa Example Configuration
-#
-# Every key can also be set by an environment variable, which wins over
-# this file: PIXA_ plus the key in upper case, with "." written as "_"
-# (state_dir is PIXA_STATE_DIR, metrics.username is
-# PIXA_METRICS_USERNAME). The one exception is port, which is set by
-# PORT. In a variable, a list is comma-separated. A variable named in
-# this file's env: section is set while the file loads, so it overrides
-# both the environment the process was started with and this file's own
-# key.
-
-# Server settings
-port: 8080
-debug: false
-maintenance_mode: false
-
-# Data directory for SQLite database and cache files
-state_dir: ./data
-
-# Image proxy settings
-# HMAC signing key for URL signatures (required, at least 32 characters)
-# Generate with: openssl rand -base64 32
-signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
-
-# Hosts that don't require signatures
-# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
-allowlist_hosts:
- - s3.sneak.cloud
- - static.sneak.cloud
- - sneak.berlin
- - github.com
- - user-images.githubusercontent.com
-
-# Additional CIDR ranges to refuse when fetching upstream, extending the
-# SSRF protection. These are added to the always-enforced built-in ranges
-# (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and
-# similar), never replacing them. Each entry must be a valid CIDR in IPv4
-# or IPv6 form; an invalid entry aborts startup.
-# blocked_networks:
-# - 100.64.0.0/10
-# - 2001:db8::/32
-
-# CIDR ranges of the reverse proxies in front of pixa. X-Forwarded-For
-# is believed only when the direct peer is inside one of these ranges;
-# the client address in the access log and login records is then the
-# rightmost forwarded entry that is not itself a trusted proxy. A client
-# connecting directly (peer outside these ranges) cannot spoof its
-# address: the header is ignored and the peer address is used. When
-# omitted, this defaults to the RFC 1918 private ranges (10.0.0.0/8,
-# 172.16.0.0/12, 192.168.0.0/16), since pixa is deployed behind a proxy on
-# a private network. An explicitly empty list ([]) trusts no one; an
-# explicit list replaces the default. An invalid CIDR aborts startup.
-# Uncomment to override the defaults with your proxy's address range.
-# trusted_proxies:
-# - 10.0.0.0/8
-# - 2001:db8::/32
-
-# Allow HTTP upstream (only for testing, always use HTTPS in production)
-allow_http: false
-
-# Maximum concurrent connections per upstream host (default: 20)
-upstream_connections_per_host: 20
-
-# Maximum disk cache size in bytes. Explicit values are used exactly as
-# given; 0 disables the disk cache entirely (every request fetches and
-# processes uncached). When omitted, the default is 75% of the free
-# space on the filesystem containing /cache/ at startup,
-# with a minimum of 500 MiB.
-# cache_max_bytes: 10737418240
-
-# Sentry error reporting (optional)
-sentry_dsn: ""
-
-# Metrics endpoint authentication (optional)
-# metrics:
-# username: "admin"
-# password: "secret"
diff --git a/configs/Caddyfile b/configs/Caddyfile
new file mode 100644
index 0000000..df453b5
--- /dev/null
+++ b/configs/Caddyfile
@@ -0,0 +1,17 @@
+# Example Caddy config for running pixa behind Caddy; see "Deployment" in
+# README.md. Replace images.example.com with pixa's public host name, and
+# 127.0.0.1:8080 with the address Caddy reaches pixa on.
+#
+# Caddy gets and renews the TLS certificate for the host name, passes the
+# Host, Origin and Referer headers on unchanged, sets X-Forwarded-For to the
+# client's address, and waits for pixa's answer with no time limit of its
+# own, so pixa's downstream_timeout is what ends a slow request.
+
+images.example.com
+
+# pixa asks for metrics.username and metrics.password on /metrics. This
+# line also keeps it off the public address, for a scraper that reaches
+# pixa directly; remove it to read /metrics through Caddy.
+respond /metrics 404
+
+reverse_proxy 127.0.0.1:8080
diff --git a/configs/config.example.yml b/configs/config.example.yml
new file mode 100644
index 0000000..33a062e
--- /dev/null
+++ b/configs/config.example.yml
@@ -0,0 +1,171 @@
+# Pixa Example Configuration
+#
+# Every key can also be set by an environment variable, which wins over
+# this file: PIXA_ plus the key in upper case, with "." written as "_"
+# (state_dir is PIXA_STATE_DIR, metrics.username is
+# PIXA_METRICS_USERNAME). The one exception is port, which is set by
+# PORT. In a variable, a list is comma-separated. A variable named in
+# this file's env: section is set while the file loads, so it overrides
+# both the environment the process was started with and this file's own
+# key.
+#
+# Durations are Go duration strings such as 30s or 2m and must be
+# positive; a bare number has no unit and aborts startup. Sizes are a
+# whole number of bytes.
+#
+# A key left out takes the default its comment gives.
+
+# Port to listen on (default: 8080)
+port: 8080
+
+# Debug logging and plain-HTTP local development (default: false)
+debug: false
+
+# While true, the image routes (/v1/image/ and /v1/e/) answer every request
+# for an image with 503 and a Retry-After header. The health check keeps
+# answering 200 and reports maintenance_mode as true. It stays 200 because
+# the image's Docker HEALTHCHECK requests it: a 503 there would make the
+# container unhealthy, and upaas marks a deploy failed when its container is
+# unhealthy. (default: false)
+maintenance_mode: false
+
+# Data directory for SQLite database and cache files
+# (default: /var/lib/pixa)
+state_dir: ./data
+
+# SQLite database URL (default:
+# file:/state.sqlite3?_pragma=journal_mode(WAL)). pixa adds
+# _pragma=busy_timeout(5000) to it. An empty value aborts startup; leave the
+# key out to use the default.
+# db_url: "file:./data/state.sqlite3?_pragma=journal_mode(WAL)"
+
+# Image proxy settings
+# HMAC signing key for URL signatures (required, at least 32 characters)
+# Generate with: openssl rand -base64 32
+signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
+
+# Hosts that don't require signatures (default: none)
+# Every image on an allowlisted host is served to anyone without a
+# signature, at any size and in any format, so list only hosts you
+# control, where nobody else can put images.
+# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
+# An entry that is neither a host name nor an IP address (IPv6 without
+# brackets), such as one with a port or a "*." wildcard, aborts startup.
+# allowlist_hosts:
+# - images.example.com
+
+# Hosts whose pages may not show pixa's images, written as for
+# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
+# names one of them is answered 403 before anything is fetched, even when
+# the image is cached. A request with no Referer, or one that does not
+# parse, is served, so a site whose pages send no Referer is not stopped.
+# The login and generator pages are not covered. (default: none)
+# referer_blocklist:
+# - leech.example
+# - .hotlinker.example
+
+# Additional CIDR ranges to refuse when fetching upstream, extending the
+# SSRF protection. These are added to the always-enforced built-in ranges
+# (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and
+# similar), never replacing them. Each entry must be a valid CIDR in IPv4
+# or IPv6 form; an invalid entry aborts startup. (default: none)
+# blocked_networks:
+# - 100.64.0.0/10
+# - 2001:db8::/32
+
+# CIDR ranges of the reverse proxies in front of pixa. X-Forwarded-For
+# is believed only when the direct peer is inside one of these ranges;
+# the client address in the access log and login records is then the
+# rightmost forwarded entry that is not itself a trusted proxy. A client
+# connecting directly (peer outside these ranges) cannot spoof its
+# address: the header is ignored and the peer address is used. When
+# omitted, this defaults to the RFC 1918 private ranges (10.0.0.0/8,
+# 172.16.0.0/12, 192.168.0.0/16), since pixa is deployed behind a proxy on
+# a private network. An explicitly empty list ([]) trusts no one; an
+# explicit list replaces the default. An invalid CIDR aborts startup.
+# Uncomment to override the defaults with the address pixa sees for
+# requests that come through your proxy. That is not always the proxy's own
+# address: a proxy on the Docker host that connects over 127.0.0.1 is seen
+# as the gateway of the container's Docker network (172.17.0.1 on the
+# default bridge), and one that connects through another host address is
+# seen with that address. To be sure, look it up in the request log as the
+# trusted_proxies entry in README.md describes.
+# trusted_proxies:
+# - 10.0.0.0/8
+# - 2001:db8::/32
+
+# Allow HTTP upstream (only for testing, always use HTTPS in production)
+# (default: false)
+allow_http: false
+
+# Maximum concurrent connections per upstream host (default: 20)
+upstream_connections_per_host: 20
+
+# Maximum concurrent connections to all upstream hosts together, on top of
+# the per-host limit (default: 64). A fetch holds its connection until its
+# image has been processed. A fetch that finds none free waits up to 10
+# seconds for one, and if none frees up the request is answered 503, unless
+# downstream_timeout has ended first.
+upstream_connections: 64
+
+# Maximum number of images decoded and encoded at once (default: half the
+# number of CPUs pixa can use, at least 1; the number of CPUs follows a
+# container's CPU limit). A request that finds none free waits up to 10
+# seconds for one, and if none frees up it is answered 503, unless
+# downstream_timeout has ended first.
+# max_concurrent_processing: 4
+
+# How many requests to the image routes (/v1/image/ and /v1/e/) one client
+# may send per minute; a request over the limit is answered 429 with a
+# Retry-After header. 0 turns the limit off. Clients are told apart as for
+# the login limit, so behind a CDN or proxy whose addresses are not in
+# trusted_proxies all requests arriving from one proxy or CDN address count
+# as one client: raise the limit there, or set 0. (default: 600)
+image_requests_per_minute: 600
+
+# Time allowed for one fetch from an upstream host (default: 30s)
+upstream_fetch_timeout: 30s
+
+# Largest upstream response accepted, in bytes, at most 1073741824
+# (1 GiB) (default: 52428800, 50 MiB)
+upstream_max_response_size: 52428800
+
+# Time allowed for answering one client request (default: 60s). The
+# upstream fetch counts toward it, and so do the waits for an upstream
+# connection and for a processing slot (up to 10 seconds each), so keep it
+# longer than upstream_fetch_timeout plus 20 seconds.
+downstream_timeout: 60s
+
+# The origin a browser lets read the responses of the image routes,
+# /v1/image/ and /v1/e/, sent as the CORS Access-Control-Allow-Origin
+# header; no other route sends it. "*" (the default) is any site;
+# otherwise one http or https origin such as https://example.com, whose
+# host is a lowercase host name (letters, digits, hyphens and dots, with a
+# letter in its last part) or an IP address (IPv6 in brackets, in its
+# shortest form), with an optional port 1-65535 that has no leading zero
+# and is not the scheme's default. Any other value, including another
+# scheme such as a browser extension's, aborts startup.
+access_control_allow_origin: "*"
+
+# Maximum disk cache size in bytes. Explicit values are used exactly as
+# given; 0 disables the disk cache entirely (every request fetches and
+# processes uncached). When omitted, the default is 75% of the sum of
+# the free space on the filesystem containing /cache/ and
+# the bytes of images the cache already holds, worked out at startup,
+# with a minimum of 500 MiB.
+# cache_max_bytes: 10737418240
+
+# Sentry DSN for error reporting (default: empty, which turns it off)
+sentry_dsn: ""
+
+# Username and password for /metrics, set together (default: unset). pixa
+# serves /metrics, and measures HTTP requests, only when both are set. It
+# measures the pixa_ metrics either way.
+# metrics:
+# username: "admin"
+# password: "secret"
+
+# Environment variables set while this file loads, as described at the top
+# (default: none)
+# env:
+# PIXA_DEBUG: "true"
diff --git a/deploy/docker-entrypoint.sh b/deploy/docker-entrypoint.sh
index 691ff2f..abb3f16 100755
--- a/deploy/docker-entrypoint.sh
+++ b/deploy/docker-entrypoint.sh
@@ -1,14 +1,22 @@
#!/bin/sh
# deploy/docker-entrypoint.sh: the Docker image's ENTRYPOINT. It runs as
-# root only to give /var/lib/pixa to pixad: a host directory
-# bind-mounted there keeps its host owner, often root, and pixad could
-# not write to it. The server itself always runs as pixad.
+# root only to make /var/lib/pixa usable by pixad: a host directory
+# bind-mounted there keeps its host owner, often root, and data from an
+# earlier run may belong to another uid. The server itself always runs
+# as pixad.
set -eu
main() {
- if [ "$(stat -c %U /var/lib/pixa)" != pixad ]; then
- chown pixad:pixad /var/lib/pixa
+ mkdir -p /var/lib/pixa
+ # Only the directory and its top-level entries are checked, so a
+ # normal start does not walk the cache. -depth gives each directory
+ # to pixad after its contents, so a start stopped part way leaves
+ # something at the top for the next start to find; -h changes a
+ # symlink itself, never the file it points to.
+ if [ -n "$(find /var/lib/pixa -maxdepth 1 \( ! -user pixad -o ! -group pixad \))" ]; then
+ find /var/lib/pixa -depth -exec chown -h pixad:pixad {} +
fi
+ chmod 750 /var/lib/pixa
exec su-exec pixad /usr/local/bin/pixad "$@"
}
diff --git a/go.mod b/go.mod
index f5c4c98..2707998 100644
--- a/go.mod
+++ b/go.mod
@@ -17,8 +17,10 @@ require (
github.com/prometheus/client_golang v1.23.2
github.com/slok/go-http-metrics v0.13.0
github.com/spf13/cobra v1.10.2
+ go.uber.org/dig v1.19.0
go.uber.org/fx v1.24.0
golang.org/x/crypto v0.41.0
+ golang.org/x/sync v0.19.0
modernc.org/sqlite v1.42.2
)
@@ -125,7 +127,6 @@ require (
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
- go.uber.org/dig v1.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
@@ -133,7 +134,6 @@ require (
golang.org/x/image v0.34.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
- golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/term v0.34.0 // indirect
golang.org/x/text v0.32.0 // indirect
diff --git a/internal/allowlist/allowlist.go b/internal/allowlist/allowlist.go
index 6dc2c4b..256c764 100644
--- a/internal/allowlist/allowlist.go
+++ b/internal/allowlist/allowlist.go
@@ -72,13 +72,3 @@ func (w *HostAllowList) IsAllowed(u *url.URL) bool {
return false
}
-
-// IsEmpty returns true if the allow list has no entries.
-func (w *HostAllowList) IsEmpty() bool {
- return len(w.exactHosts) == 0 && len(w.suffixHosts) == 0
-}
-
-// Count returns the total number of allow list entries.
-func (w *HostAllowList) Count() int {
- return len(w.exactHosts) + len(w.suffixHosts)
-}
diff --git a/internal/allowlist/allowlist_test.go b/internal/allowlist/allowlist_test.go
index 1d68a14..a1730b9 100644
--- a/internal/allowlist/allowlist_test.go
+++ b/internal/allowlist/allowlist_test.go
@@ -139,87 +139,3 @@ func TestHostAllowList_IsAllowed_SuffixMatch(t *testing.T) {
},
})
}
-
-func TestHostAllowList_IsEmpty(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- name string
- patterns []string
- want bool
- }{
- {
- name: "empty",
- patterns: []string{},
- want: true,
- },
- {
- name: "nil",
- patterns: nil,
- want: true,
- },
- {
- name: "whitespace only",
- patterns: []string{" ", ""},
- want: true,
- },
- {
- name: "has entries",
- patterns: []string{"example.com"},
- want: false,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- w := allowlist.New(tt.patterns)
- if got := w.IsEmpty(); got != tt.want {
- t.Errorf("IsEmpty() = %v, want %v", got, tt.want)
- }
- })
- }
-}
-
-func TestHostAllowList_Count(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- name string
- patterns []string
- want int
- }{
- {
- name: "empty",
- patterns: []string{},
- want: 0,
- },
- {
- name: "exact hosts only",
- patterns: []string{"a.com", "b.com", "c.com"},
- want: 3,
- },
- {
- name: "suffix hosts only",
- patterns: []string{".a.com", ".b.com"},
- want: 2,
- },
- {
- name: "mixed",
- patterns: []string{"exact.com", ".suffix.com"},
- want: 2,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- w := allowlist.New(tt.patterns)
- if got := w.Count(); got != tt.want {
- t.Errorf("Count() = %v, want %v", got, tt.want)
- }
- })
- }
-}
diff --git a/internal/app/app.go b/internal/app/app.go
new file mode 100644
index 0000000..1481ef7
--- /dev/null
+++ b/internal/app/app.go
@@ -0,0 +1,116 @@
+// Package app reads the pixad command line and runs the server.
+package app
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/spf13/cobra"
+ "go.uber.org/dig"
+ "go.uber.org/fx"
+ "sneak.berlin/go/pixa/internal/config"
+ "sneak.berlin/go/pixa/internal/database"
+ "sneak.berlin/go/pixa/internal/globals"
+ "sneak.berlin/go/pixa/internal/handlers"
+ "sneak.berlin/go/pixa/internal/healthcheck"
+ "sneak.berlin/go/pixa/internal/logger"
+ "sneak.berlin/go/pixa/internal/middleware"
+ "sneak.berlin/go/pixa/internal/providers"
+ "sneak.berlin/go/pixa/internal/server"
+)
+
+var configPath string //nolint:gochecknoglobals // cobra flag
+
+// Run reads the command line and runs the server until it stops, with
+// version as the version pixad logs and reports. It exits the process
+// with status 1 when the command line is not valid.
+func Run(version string) {
+ globals.Version = version
+
+ rootCmd := &cobra.Command{
+ Use: "pixad",
+ Short: "Pixa image caching proxy server",
+ Run: run,
+ }
+
+ rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
+
+ err := rootCmd.Execute()
+ if err != nil {
+ fmt.Fprintln(os.Stderr, err)
+ os.Exit(1)
+ }
+}
+
+func run(_ *cobra.Command, _ []string) {
+ // Set config path in environment if specified via flag
+ if configPath != "" {
+ _ = os.Setenv("PIXA_CONFIG_PATH", configPath)
+ }
+
+ // A write to a closed stdout or stderr must not end the process.
+ signal.Ignore(syscall.SIGPIPE)
+
+ app := fx.New(
+ // pixad logs with slog only; runApp prints a startup or shutdown
+ // error itself.
+ fx.NopLogger,
+ fx.Provide(
+ config.New,
+ database.New,
+ globals.New,
+ handlers.New,
+ logger.New,
+ server.New,
+ middleware.New,
+ healthcheck.New,
+ // The registry /metrics serves
+ func() prometheus.Registerer { return prometheus.DefaultRegisterer },
+ ),
+ fx.Provide(providers.Constructors()...),
+ fx.Invoke(
+ func(log *logger.Logger) { log.Identify() },
+ func(*server.Server) {},
+ ),
+ )
+
+ exitCode := runApp(app, os.Stderr)
+ if exitCode != 0 {
+ os.Exit(exitCode)
+ }
+}
+
+// runApp starts app, waits for a signal or a shutdown request, stops app,
+// and returns the status pixad exits with, as fx.App.Run does. It prints
+// a start or stop error to stderr, once, as the error that caused it:
+// dig.RootCause drops the frames fx and dig wrap around it.
+func runApp(app *fx.App, stderr io.Writer) int {
+ startCtx, cancelStart := context.WithTimeout(context.Background(), app.StartTimeout())
+ defer cancelStart()
+
+ err := app.Start(startCtx)
+ if err != nil {
+ _, _ = fmt.Fprintln(stderr, dig.RootCause(err))
+
+ return 1
+ }
+
+ shutdown := <-app.Wait()
+
+ stopCtx, cancelStop := context.WithTimeout(context.Background(), app.StopTimeout())
+ defer cancelStop()
+
+ err = app.Stop(stopCtx)
+ if err != nil {
+ _, _ = fmt.Fprintln(stderr, dig.RootCause(err))
+
+ return 1
+ }
+
+ return shutdown.ExitCode
+}
diff --git a/internal/app/app_internal_test.go b/internal/app/app_internal_test.go
new file mode 100644
index 0000000..d59b25d
--- /dev/null
+++ b/internal/app/app_internal_test.go
@@ -0,0 +1,65 @@
+package app
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "testing"
+
+ "go.uber.org/fx"
+)
+
+var errTestStopFailed = errors.New("test stop hook failed")
+
+// TestRunAppPrintsStopErrorOnce checks that when a stop hook fails, runApp
+// prints the hook's error once and returns exit status 1.
+func TestRunAppPrintsStopErrorOnce(t *testing.T) {
+ t.Parallel()
+
+ app := fx.New(
+ fx.NopLogger,
+ fx.Invoke(func(lifecycle fx.Lifecycle, shutdowner fx.Shutdowner) {
+ lifecycle.Append(fx.Hook{
+ // Ask to shut down as soon as the app has started, so
+ // that runApp goes on to stop it.
+ OnStart: func(context.Context) error { return shutdowner.Shutdown() },
+ OnStop: func(context.Context) error { return errTestStopFailed },
+ })
+ }),
+ )
+
+ var stderr bytes.Buffer
+
+ exitCode := runApp(app, &stderr)
+ if exitCode != 1 {
+ t.Errorf("runApp() = %d, want 1", exitCode)
+ }
+
+ want := errTestStopFailed.Error() + "\n"
+ if stderr.String() != want {
+ t.Errorf("stderr = %q, want %q", stderr.String(), want)
+ }
+}
+
+// TestRunAppReturnsShutdownExitCode checks that runApp returns the exit
+// code a shutdown request names.
+func TestRunAppReturnsShutdownExitCode(t *testing.T) {
+ t.Parallel()
+
+ app := fx.New(
+ fx.NopLogger,
+ fx.Invoke(func(lifecycle fx.Lifecycle, shutdowner fx.Shutdowner) {
+ lifecycle.Append(fx.Hook{
+ OnStart: func(context.Context) error {
+ return shutdowner.Shutdown(fx.ExitCode(3))
+ },
+ })
+ }),
+ )
+
+ exitCode := runApp(app, io.Discard)
+ if exitCode != 3 {
+ t.Errorf("runApp() = %d, want 3", exitCode)
+ }
+}
diff --git a/internal/app/app_test.go b/internal/app/app_test.go
new file mode 100644
index 0000000..a6d94bf
--- /dev/null
+++ b/internal/app/app_test.go
@@ -0,0 +1,177 @@
+package app_test
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "strconv"
+ "syscall"
+ "testing"
+ "time"
+
+ "sneak.berlin/go/pixa/internal/app"
+)
+
+// These tests run pixad in a child process: the test binary itself,
+// started again with runPixadInChildEnvVar set, which makes TestMain run
+// pixad instead of the tests.
+const runPixadInChildEnvVar = "TEST_RUN_PIXAD_IN_CHILD"
+
+const validTestSigningKey = "test-signing-key-for-app-tests-0123456789"
+
+func TestMain(m *testing.M) {
+ if os.Getenv(runPixadInChildEnvVar) == "1" {
+ os.Args = []string{"pixad"}
+
+ app.Run("test")
+ os.Exit(0)
+ }
+
+ os.Exit(m.Run())
+}
+
+// pixadCommand returns a command that runs pixad with only the given
+// environment, in an empty directory with an empty HOME, so that no
+// config file is found.
+func pixadCommand(t *testing.T, env ...string) (*exec.Cmd, *bytes.Buffer) {
+ t.Helper()
+
+ testBinary := os.Args[0]
+ cmd := exec.CommandContext(t.Context(), testBinary) //nolint:gosec // not tainted
+ cmd.Dir = t.TempDir()
+ // A test binary built with -cover warns on stderr at exit when
+ // GOCOVERDIR is not set.
+ cmd.Env = append([]string{
+ runPixadInChildEnvVar + "=1",
+ "HOME=" + t.TempDir(),
+ "GOCOVERDIR=" + t.TempDir(),
+ }, env...)
+
+ var stderr bytes.Buffer
+
+ cmd.Stderr = &stderr
+
+ return cmd, &stderr
+}
+
+// TestRunPrintsStartupErrorOnce checks that an invalid setting is
+// printed once, as the line that names it, and nothing else goes to
+// stderr.
+func TestRunPrintsStartupErrorOnce(t *testing.T) {
+ t.Parallel()
+
+ cmd, stderr := pixadCommand(t, "PIXA_SIGNING_KEY=short")
+
+ err := cmd.Run()
+
+ var exitErr *exec.ExitError
+ if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 {
+ t.Fatalf("pixad: got %v, want exit status 1; stderr:\n%s", err, stderr)
+ }
+
+ want := `config key "signing_key" (environment variable PIXA_SIGNING_KEY): ` +
+ "value too short: must be at least 32 characters, got 5\n"
+ if stderr.String() != want {
+ t.Errorf("stderr:\n%s\nwant:\n%s", stderr, want)
+ }
+}
+
+// TestRunWritesNothingToStderrOnStartAndStop starts pixad, waits until
+// it answers its health check, stops it with SIGTERM, and checks that it
+// wrote nothing to stderr.
+func TestRunWritesNothingToStderrOnStartAndStop(t *testing.T) {
+ t.Parallel()
+
+ port := freePort(t)
+ cmd, stderr := pixadCommand(t,
+ "PORT="+strconv.Itoa(port),
+ "PIXA_STATE_DIR="+t.TempDir(),
+ "PIXA_SIGNING_KEY="+validTestSigningKey,
+ )
+
+ err := cmd.Start()
+ if err != nil {
+ t.Fatalf("starting pixad: %v", err)
+ }
+
+ waitForHealthCheck(t, cmd, port, stderr)
+
+ err = cmd.Process.Signal(syscall.SIGTERM)
+ if err != nil {
+ t.Fatalf("signalling pixad: %v", err)
+ }
+
+ err = cmd.Wait()
+ if err != nil {
+ t.Fatalf("pixad: got %v, want exit status 0; stderr:\n%s", err, stderr)
+ }
+
+ if stderr.Len() != 0 {
+ t.Errorf("stderr is not empty:\n%s", stderr)
+ }
+}
+
+// freePort returns a TCP port on the loopback address that nothing
+// listens on.
+func freePort(t *testing.T) int {
+ t.Helper()
+
+ var listenConfig net.ListenConfig
+
+ listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("finding a free port: %v", err)
+ }
+
+ addr, ok := listener.Addr().(*net.TCPAddr)
+ if !ok {
+ t.Fatalf("listener address %v is not a TCP address", listener.Addr())
+ }
+
+ err = listener.Close()
+ if err != nil {
+ t.Fatalf("closing listener: %v", err)
+ }
+
+ return addr.Port
+}
+
+// waitForHealthCheck waits up to 30 seconds for pixad, started as cmd, to
+// answer its health check on port. If it does not, it stops pixad and
+// fails the test with what pixad wrote to stderr.
+func waitForHealthCheck(t *testing.T, cmd *exec.Cmd, port int, stderr *bytes.Buffer) {
+ t.Helper()
+
+ url := fmt.Sprintf("http://127.0.0.1:%d/.well-known/healthcheck.json", port)
+ deadline := time.Now().Add(30 * time.Second)
+
+ for time.Now().Before(deadline) {
+ req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, nil)
+ if err != nil {
+ t.Fatalf("health check request: %v", err)
+ }
+
+ resp, err := http.DefaultClient.Do(req)
+ if err == nil {
+ _ = resp.Body.Close()
+
+ if resp.StatusCode == http.StatusOK {
+ return
+ }
+ }
+
+ time.Sleep(100 * time.Millisecond)
+ }
+
+ // Stop pixad first: stderr is complete, and safe to read, only once
+ // cmd.Wait has returned.
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+
+ t.Fatalf("pixad did not answer its health check on port %d; stderr:\n%s",
+ port, stderr)
+}
diff --git a/internal/config/blocked_networks_internal_test.go b/internal/config/blocked_networks_internal_test.go
index 25373be..475b1d4 100644
--- a/internal/config/blocked_networks_internal_test.go
+++ b/internal/config/blocked_networks_internal_test.go
@@ -88,3 +88,56 @@ func TestBlockedNetworksInvalidAbortsStartup(t *testing.T) {
},
})
}
+
+// TestBlockedNetworksMappedEntryIsIPv4Range checks that an entry written in
+// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
+// since the fetcher matches an IPv4-mapped address as the IPv4 address it
+// holds. /96, the shortest such entry, covers every IPv4 address.
+func TestBlockedNetworksMappedEntryIsIPv4Range(t *testing.T) {
+ t.Parallel()
+
+ yamlContent := signingKeyLine + `blocked_networks:
+ - "::ffff:10.0.0.0/104"
+ - "::ffff:0:0/96"
+`
+
+ c, err := configFromYAML(t, yamlContent)
+ if err != nil {
+ t.Fatalf("IPv4-mapped blocked_networks entries should load, got error: %v", err)
+ }
+
+ want := []string{"10.0.0.0/8", "0.0.0.0/0"}
+ if len(c.BlockedNetworks) != len(want) {
+ t.Fatalf("BlockedNetworks = %v, want %d entries", c.BlockedNetworks, len(want))
+ }
+
+ for i, w := range want {
+ if got := c.BlockedNetworks[i].String(); got != w {
+ t.Errorf("BlockedNetworks[%d] = %q, want %q", i, got, w)
+ }
+ }
+}
+
+// TestBlockedNetworksMappedEntryShorterThan96AbortsStartup checks that an
+// entry in the IPv4-mapped form shorter than /96, which covers more than the
+// IPv4 addresses, aborts startup with an error naming the key and the value.
+func TestBlockedNetworksMappedEntryShorterThan96AbortsStartup(t *testing.T) {
+ t.Parallel()
+
+ runAbortCases(t, []abortCase{
+ {
+ name: "mapped-shorter-than-96",
+ yaml: signingKeyLine + `blocked_networks:
+ - "::ffff:0.0.0.0/80"
+`,
+ wantErrSubstrings: []string{keyBlockedNetworks, "::ffff:0.0.0.0/80"},
+ },
+ {
+ name: "mapped-95",
+ yaml: signingKeyLine + `blocked_networks:
+ - "::ffff:0:0/95"
+`,
+ wantErrSubstrings: []string{keyBlockedNetworks},
+ },
+ })
+}
diff --git a/internal/config/cache_max_bytes_internal_test.go b/internal/config/cache_max_bytes_internal_test.go
index 50933c2..3538530 100644
--- a/internal/config/cache_max_bytes_internal_test.go
+++ b/internal/config/cache_max_bytes_internal_test.go
@@ -1,26 +1,10 @@
package config
import (
- "errors"
- "log/slog"
- "os"
- "path/filepath"
"strings"
"testing"
)
-// Static errors returned by the stub free-space probes below.
-var (
- errTestStatfsFailed = errors.New("statfs failed")
- errTestProbeNotExpected = errors.New("probe must not be called")
-)
-
-// discardLogger returns a logger that swallows all output, for tests
-// that exercise code paths which log.
-func discardLogger() *slog.Logger {
- return slog.New(slog.DiscardHandler)
-}
-
// TestCacheMaxBytesExplicitValueUsedWithoutFloor verifies that an
// explicitly configured cache_max_bytes value is used exactly as
// given: the 500 MiB floor applies only to the computed default, never
@@ -151,30 +135,10 @@ func TestCacheMaxBytesInvalidValuesAbortStartup(t *testing.T) {
}
}
-// TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace verifies the
-// computed default is 75% of the probed free space when that exceeds
-// the floor.
-func TestComputeDefaultCacheMaxBytesUses75PercentOfFreeSpace(t *testing.T) {
- t.Parallel()
-
- // 4 GiB free -> 3 GiB default.
- probe := func(string) (uint64, error) { return 4294967296, nil }
-
- got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe)
- if err != nil {
- t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err)
- }
-
- if got != 3221225472 {
- t.Errorf("ComputeDefaultCacheMaxBytes = %d, want 3221225472 (75%% of 4 GiB)",
- got)
- }
-}
-
-// TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault
-// verifies that when 75% of free space is below 500 MiB, the computed
-// default is floored at DefaultCacheMaxBytesFloor.
-func TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault(t *testing.T) {
+// TestDefaultCacheMaxBytesAppliesFloor verifies that when 75% of the
+// free space plus what the cache holds is below 500 MiB, the default is
+// floored at DefaultCacheMaxBytesFloor.
+func TestDefaultCacheMaxBytesAppliesFloor(t *testing.T) {
t.Parallel()
cases := []struct {
@@ -190,116 +154,39 @@ func TestComputeDefaultCacheMaxBytesAppliesFloorToComputedDefault(t *testing.T)
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
- probe := func(string) (uint64, error) { return tc.freeBytes, nil }
-
- got, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe)
- if err != nil {
- t.Fatalf("ComputeDefaultCacheMaxBytes returned error: %v", err)
- }
-
+ got := DefaultCacheMaxBytes(tc.freeBytes, 0)
if got != DefaultCacheMaxBytesFloor {
- t.Errorf("ComputeDefaultCacheMaxBytes = %d, want floor %d",
+ t.Errorf("DefaultCacheMaxBytes = %d, want floor %d",
got, DefaultCacheMaxBytesFloor)
}
})
}
}
-// TestComputeDefaultCacheMaxBytesPropagatesProbeError verifies that a
-// failing free-space probe produces an error naming the config key,
-// instead of a silently wrong default.
-func TestComputeDefaultCacheMaxBytesPropagatesProbeError(t *testing.T) {
+// TestCacheMaxBytesExplicitIsRecorded verifies that an omitted
+// cache_max_bytes is recorded as not explicit, so pixad works out the
+// default when it builds the cache, and that an explicit zero is recorded
+// as explicit, so it disables the disk cache instead.
+func TestCacheMaxBytesExplicitIsRecorded(t *testing.T) {
t.Parallel()
- probe := func(string) (uint64, error) { return 0, errTestStatfsFailed }
+ signingKeyLine := "signing_key: " + validTestSigningKey + "\n"
- _, err := ComputeDefaultCacheMaxBytes(t.TempDir(), probe)
- if err == nil {
- t.Fatal("probe failure must produce an error, got nil")
- }
-
- t.Logf("got expected error: %v", err)
-
- if !strings.Contains(err.Error(), keyCacheMaxBytes) {
- t.Errorf("error %q does not name the config key cache_max_bytes", err.Error())
- }
-}
-
-// TestResolveCacheMaxBytesComputesDefaultWhenOmitted verifies that an
-// omitted cache_max_bytes key resolves to the computed default, that
-// the probe is pointed at /cache/ (which must be created
-// first so statfs measures the right filesystem), and that the result
-// lands on the Config.
-func TestResolveCacheMaxBytesComputesDefaultWhenOmitted(t *testing.T) {
- t.Parallel()
-
- c, err := configFromYAML(t, "signing_key: "+validTestSigningKey+"\n")
+ omitted, err := configFromYAML(t, signingKeyLine)
if err != nil {
t.Fatalf("minimal config should be valid, got error: %v", err)
}
- c.StateDir = t.TempDir()
- wantCacheDir := filepath.Join(c.StateDir, "cache")
-
- var probedPath string
-
- // 4 GiB free -> 3 GiB default.
- probe := func(path string) (uint64, error) {
- probedPath = path
-
- return 4294967296, nil
+ if omitted.CacheMaxBytesExplicit {
+ t.Error("omitted cache_max_bytes recorded as explicit")
}
- err = c.resolveCacheMaxBytes(discardLogger(), probe)
+ zero, err := configFromYAML(t, signingKeyLine+"cache_max_bytes: 0\n")
if err != nil {
- t.Fatalf("resolveCacheMaxBytes returned error: %v", err)
+ t.Fatalf("cache_max_bytes: 0 must be accepted, got error: %v", err)
}
- if c.CacheMaxBytes != 3221225472 {
- t.Errorf("CacheMaxBytes = %d, want computed default 3221225472",
- c.CacheMaxBytes)
- }
-
- if probedPath != wantCacheDir {
- t.Errorf("free space probed at %q, want cache directory %q",
- probedPath, wantCacheDir)
- }
-
- info, err := os.Stat(wantCacheDir)
- if err != nil || !info.IsDir() {
- t.Errorf("cache directory %q was not created before probing: info=%v err=%v",
- wantCacheDir, info, err)
- }
-}
-
-// TestResolveCacheMaxBytesDoesNotOverrideExplicitValue verifies that
-// an explicitly configured value survives resolution untouched and
-// that the free-space probe is never consulted for it.
-func TestResolveCacheMaxBytesDoesNotOverrideExplicitValue(t *testing.T) {
- t.Parallel()
-
- yamlContent := "signing_key: " + validTestSigningKey + "\ncache_max_bytes: 1024\n"
-
- c, err := configFromYAML(t, yamlContent)
- if err != nil {
- t.Fatalf("explicit cache_max_bytes must be accepted, got error: %v", err)
- }
-
- c.StateDir = t.TempDir()
-
- probe := func(string) (uint64, error) {
- t.Error("free-space probe must not be consulted for explicit values")
-
- return 0, errTestProbeNotExpected
- }
-
- err = c.resolveCacheMaxBytes(discardLogger(), probe)
- if err != nil {
- t.Fatalf("resolveCacheMaxBytes returned error: %v", err)
- }
-
- if c.CacheMaxBytes != 1024 {
- t.Errorf("CacheMaxBytes = %d, want explicit 1024 (no floor, no recompute)",
- c.CacheMaxBytes)
+ if !zero.CacheMaxBytesExplicit {
+ t.Error("cache_max_bytes: 0 not recorded as explicit")
}
}
diff --git a/internal/config/cachesize.go b/internal/config/cachesize.go
deleted file mode 100644
index 8f7a014..0000000
--- a/internal/config/cachesize.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package config
-
-import (
- "fmt"
- "log/slog"
- "math"
- "os"
- "path/filepath"
- "syscall"
-)
-
-// DefaultCacheMaxBytesFloor is the minimum computed default for the
-// cache_max_bytes setting: 500 MiB. The floor applies only to the
-// computed default (when the key is omitted from the configuration),
-// never to explicitly configured values.
-const DefaultCacheMaxBytesFloor int64 = 524288000
-
-// cacheDirPerms is the permission mode for the cache directory created
-// before probing free space, matching the state directory permissions.
-const cacheDirPerms = 0o750
-
-// freeSpaceFractionNumerator and freeSpaceFractionDenominator express
-// the 75% share of free space used for the computed default limit as
-// integer arithmetic (dividing before multiplying avoids overflow).
-const (
- freeSpaceFractionNumerator uint64 = 3
- freeSpaceFractionDenominator uint64 = 4
-)
-
-// FreeSpaceProbeFunc reports the number of free bytes available on the
-// filesystem containing path. It is a function type so tests can
-// inject a fake probe instead of depending on the host disk.
-type FreeSpaceProbeFunc func(path string) (uint64, error)
-
-// defaultFreeSpaceProbe reports free filesystem bytes via statfs on
-// the given path, as available to unprivileged processes.
-func defaultFreeSpaceProbe(path string) (uint64, error) {
- var stat syscall.Statfs_t
-
- err := syscall.Statfs(path, &stat)
- if err != nil {
- return 0, err
- }
-
- if stat.Bsize < 0 {
- return 0, fmt.Errorf("%w %d for %q", errNegativeBlockSize, stat.Bsize, path)
- }
-
- blockSize := uint64(stat.Bsize)
-
- return stat.Bavail * blockSize, nil
-}
-
-// ComputeDefaultCacheMaxBytes returns the default cache size limit for
-// the filesystem containing cacheDir: 75% of the free bytes reported
-// by probe, with a floor of DefaultCacheMaxBytesFloor.
-func ComputeDefaultCacheMaxBytes(
- cacheDir string, probe FreeSpaceProbeFunc,
-) (int64, error) {
- freeBytes, err := probe(cacheDir)
- if err != nil {
- return 0, fmt.Errorf("config key %q: cannot determine free space for %q: %w",
- "cache_max_bytes", cacheDir, err)
- }
-
- computed := freeBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
- computed = min(computed, math.MaxInt64)
-
- // gosec cannot see that min() above bounds computed, so it reads
- // this conversion as potentially overflowing. It cannot: computed is
- // at most math.MaxInt64 on every path here.
- //nolint:gosec // G115: clamped to MaxInt64 by min above
- limit := int64(computed)
- limit = max(limit, DefaultCacheMaxBytesFloor)
-
- return limit, nil
-}
-
-// resolveCacheMaxBytes finalizes CacheMaxBytes after state_dir
-// validation: an explicitly configured value is kept as-is (no floor
-// applies), while an omitted key receives the computed default based
-// on free space in /cache/. The cache directory is created
-// first so statfs measures the filesystem that will actually hold the
-// cache. The effective limit is logged either way.
-func (c *Config) resolveCacheMaxBytes(
- log *slog.Logger, probe FreeSpaceProbeFunc,
-) error {
- if !c.cacheMaxBytesExplicit {
- cacheDir := filepath.Join(c.StateDir, "cache")
-
- err := os.MkdirAll(cacheDir, cacheDirPerms)
- if err != nil {
- return fmt.Errorf("config key %q: cannot create cache directory %q: %w",
- keyCacheMaxBytes, cacheDir, err)
- }
-
- limit, err := ComputeDefaultCacheMaxBytes(cacheDir, probe)
- if err != nil {
- return err
- }
-
- c.CacheMaxBytes = limit
-
- log.Info("computed default cache size limit from free space",
- "cache_max_bytes", limit,
- "cache_dir", cacheDir,
- )
- }
-
- log.Info("effective cache size limit",
- "cache_max_bytes", c.CacheMaxBytes,
- "cache_disabled", c.CacheMaxBytes == 0,
- )
-
- return nil
-}
diff --git a/internal/config/concurrency_limits_internal_test.go b/internal/config/concurrency_limits_internal_test.go
new file mode 100644
index 0000000..51406e3
--- /dev/null
+++ b/internal/config/concurrency_limits_internal_test.go
@@ -0,0 +1,209 @@
+package config
+
+import (
+ "fmt"
+ "runtime"
+ "testing"
+)
+
+// The variables that set the two concurrency limits.
+const (
+ testMaxConcurrentProcessingVar = "PIXA_MAX_CONCURRENT_PROCESSING"
+ testUpstreamConnectionsVar = "PIXA_UPSTREAM_CONNECTIONS"
+)
+
+// TestOmittedConcurrencyLimitsUseDefaults checks that an omitted
+// max_concurrent_processing is half the CPUs Go uses, and at least 1, and
+// an omitted upstream_connections is 64.
+func TestOmittedConcurrencyLimitsUseDefaults(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine)
+ if err != nil {
+ t.Fatalf("minimal config should be valid, got error: %v", err)
+ }
+
+ if c.MaxConcurrentProcessing != max(runtime.GOMAXPROCS(0)/2, 1) {
+ t.Errorf("MaxConcurrentProcessing = %d, want %d, half the CPUs",
+ c.MaxConcurrentProcessing, max(runtime.GOMAXPROCS(0)/2, 1))
+ }
+
+ if c.UpstreamConnections != 64 {
+ t.Errorf("UpstreamConnections = %d, want 64", c.UpstreamConnections)
+ }
+}
+
+// TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne checks
+// that an omitted max_concurrent_processing is half the CPUs Go uses,
+// rounded down, and 1 when Go uses one CPU.
+//
+//nolint:paralleltest // it sets GOMAXPROCS, which other tests read
+func TestOmittedMaxConcurrentProcessingIsHalfTheCPUsAndAtLeastOne(t *testing.T) {
+ cpusBefore := runtime.GOMAXPROCS(0)
+
+ t.Cleanup(func() { runtime.GOMAXPROCS(cpusBefore) })
+
+ for _, tc := range []struct{ cpus, want int }{
+ {1, 1}, {2, 1}, {3, 1}, {8, 4},
+ } {
+ runtime.GOMAXPROCS(tc.cpus)
+
+ c, err := configFromYAML(t, signingKeyLine)
+ if err != nil {
+ t.Fatalf("minimal config should be valid, got error: %v", err)
+ }
+
+ t.Logf("GOMAXPROCS %d: MaxConcurrentProcessing %d",
+ tc.cpus, c.MaxConcurrentProcessing)
+
+ if c.MaxConcurrentProcessing != tc.want {
+ t.Errorf("GOMAXPROCS %d: MaxConcurrentProcessing = %d, want %d",
+ tc.cpus, c.MaxConcurrentProcessing, tc.want)
+ }
+ }
+}
+
+// TestExplicitConcurrencyLimitsAreUsed checks that valid values for the
+// two limits are used as given.
+func TestExplicitConcurrencyLimitsAreUsed(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+
+ "max_concurrent_processing: 3\nupstream_connections: 10\n")
+ if err != nil {
+ t.Fatalf("valid config should load, got error: %v", err)
+ }
+
+ if c.MaxConcurrentProcessing != 3 {
+ t.Errorf("MaxConcurrentProcessing = %d, want 3", c.MaxConcurrentProcessing)
+ }
+
+ if c.UpstreamConnections != 10 {
+ t.Errorf("UpstreamConnections = %d, want 10", c.UpstreamConnections)
+ }
+}
+
+// TestInvalidConcurrencyLimitAbortsStartup checks that a limit that is
+// not a whole number of at least 1, or is null, aborts startup naming the
+// key and the value, and the variable too where the value could have come
+// from it.
+func TestInvalidConcurrencyLimitAbortsStartup(t *testing.T) {
+ t.Parallel()
+
+ processing := keyMaxConcurrentProcessing
+ connections := keyUpstreamConnections
+
+ runAbortCases(t, []abortCase{
+ {
+ name: "max_concurrent_processing zero",
+ yaml: signingKeyLine + processing + ": 0\n",
+ wantErrSubstrings: []string{
+ processing, testMaxConcurrentProcessingVar, "value 0",
+ },
+ },
+ {
+ name: "max_concurrent_processing negative",
+ yaml: signingKeyLine + processing + ": -2\n",
+ wantErrSubstrings: []string{
+ processing, testMaxConcurrentProcessingVar, "value -2",
+ },
+ },
+ {
+ name: "max_concurrent_processing not a number",
+ yaml: signingKeyLine + processing + ": lots\n",
+ wantErrSubstrings: []string{
+ processing, testMaxConcurrentProcessingVar, "lots",
+ },
+ },
+ {
+ name: "max_concurrent_processing fractional",
+ yaml: signingKeyLine + processing + ": 1.5\n",
+ wantErrSubstrings: []string{processing, "1.5"},
+ },
+ {
+ name: "max_concurrent_processing null",
+ yaml: signingKeyLine + processing + ": null\n",
+ wantErrSubstrings: []string{processing, nullValueText},
+ },
+ {
+ name: "upstream_connections zero",
+ yaml: signingKeyLine + connections + ": 0\n",
+ wantErrSubstrings: []string{
+ connections, testUpstreamConnectionsVar, "value 0",
+ },
+ },
+ {
+ name: "upstream_connections negative",
+ yaml: signingKeyLine + connections + ": -5\n",
+ wantErrSubstrings: []string{
+ connections, testUpstreamConnectionsVar, "value -5",
+ },
+ },
+ {
+ name: "upstream_connections not a number",
+ yaml: signingKeyLine + connections + ": many\n",
+ wantErrSubstrings: []string{
+ connections, testUpstreamConnectionsVar, "many",
+ },
+ },
+ {
+ name: "upstream_connections null",
+ yaml: signingKeyLine + connections + ": null\n",
+ wantErrSubstrings: []string{connections, nullValueText},
+ },
+ })
+}
+
+// TestConcurrencyLimitsFromEnvironment checks that the two variables set
+// the limits over the config file, and that an invalid value in either
+// aborts startup naming the variable and the value.
+func TestConcurrencyLimitsFromEnvironment(t *testing.T) {
+ t.Setenv(testMaxConcurrentProcessingVar, "3")
+ t.Setenv(testUpstreamConnectionsVar, "10")
+
+ c, err := configFromYAML(t, signingKeyLine+
+ "max_concurrent_processing: 5\nupstream_connections: 50\n")
+ if err != nil {
+ t.Fatalf("limits from the environment should load: %v", err)
+ }
+
+ if c.MaxConcurrentProcessing != 3 || c.UpstreamConnections != 10 {
+ t.Errorf("limits = %d and %d, want 3 and 10 from the environment",
+ c.MaxConcurrentProcessing, c.UpstreamConnections)
+ }
+
+ cases := []struct {
+ variable string
+ value string
+ }{
+ {testMaxConcurrentProcessingVar, "lots"},
+ {testMaxConcurrentProcessingVar, "0"},
+ {testUpstreamConnectionsVar, "-1"},
+ {testUpstreamConnectionsVar, "ten"},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.variable+"="+tc.value, func(t *testing.T) {
+ t.Setenv(tc.variable, tc.value)
+
+ _, err := configFromYAML(t, signingKeyLine)
+ wantStartupError(t, err, tc.variable, tc.value)
+ })
+ }
+}
+
+// TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup checks
+// that a PIXA_MAX_CONCURRENT_PROCESSING that is not a whole number of at
+// least 1, an empty one included, aborts startup naming the variable and
+// the key instead of taking the default.
+func TestInvalidMaxConcurrentProcessingFromEnvironmentAbortsStartup(t *testing.T) {
+ for _, value := range []string{"", " ", "0.5", "half", "-1", "4 images"} {
+ t.Run(fmt.Sprintf("value %q", value), func(t *testing.T) {
+ t.Setenv(testMaxConcurrentProcessingVar, value)
+
+ _, err := configFromYAML(t, signingKeyLine)
+ wantStartupError(t, err,
+ testMaxConcurrentProcessingVar, keyMaxConcurrentProcessing)
+ })
+ }
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 3d948be..2f433c7 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -4,15 +4,20 @@ package config
import (
"errors"
"fmt"
+ "io/fs"
"log/slog"
"math"
"net/netip"
"net/url"
"os"
"path/filepath"
+ "regexp"
+ "runtime"
"sort"
"strconv"
"strings"
+ "syscall"
+ "time"
"git.eeqj.de/sneak/smartconfig"
"go.uber.org/fx"
@@ -24,30 +29,55 @@ const (
DefaultPort = 8080
DefaultStateDir = "/var/lib/pixa"
DefaultUpstreamConnectionsPerHost = 20
+ DefaultUpstreamConnections = 64
+ DefaultAccessControlAllowOrigin = "*"
+ DefaultUpstreamFetchTimeout = 30 * time.Second
+ DefaultUpstreamMaxResponseSize = 50 << 20 // 50 MiB
+ DefaultDownstreamTimeout = 60 * time.Second
+ DefaultImageRequestsPerMinute = 600
)
-// Configuration key names.
+// DefaultMaxConcurrentProcessing returns the most images processed at once
+// when max_concurrent_processing is omitted: half the CPUs Go uses
+// (runtime.GOMAXPROCS(0), which follows a container's CPU limit), rounded
+// down, and at least 1.
+func DefaultMaxConcurrentProcessing() int {
+ return max(runtime.GOMAXPROCS(0)/2, 1) //nolint:mnd // half the CPUs
+}
+
+// DefaultCacheMaxBytesFloor is the least DefaultCacheMaxBytes returns:
+// 500 MiB. It applies only to the default, never to an explicit
+// cache_max_bytes.
+const DefaultCacheMaxBytesFloor int64 = 500 << 20
+
+// freeSpaceFractionNumerator and freeSpaceFractionDenominator express
+// the 75% share of DefaultCacheMaxBytes as integer arithmetic (dividing
+// before multiplying avoids overflow).
const (
- keyDebug = "debug"
- keyMaintenanceMode = "maintenance_mode"
- keyPort = "port"
- keyStateDir = "state_dir"
- keySentryDSN = "sentry_dsn"
- keyDBURL = "db_url"
- keyMetrics = "metrics"
- keyMetricsUsername = "metrics.username"
- keyMetricsPassword = "metrics.password"
- keySigningKey = "signing_key"
- keyAllowlistHosts = "allowlist_hosts"
- keyAllowHTTP = "allow_http"
- keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
- keyCacheMaxBytes = "cache_max_bytes"
- keyBlockedNetworks = "blocked_networks"
- keyTrustedProxies = "trusted_proxies"
+ freeSpaceFractionNumerator uint64 = 3
+ freeSpaceFractionDenominator uint64 = 4
)
+// DefaultCacheMaxBytes returns the disk cache limit when cache_max_bytes
+// is omitted: 75% of the sum of freeBytes, the space free on the
+// filesystem holding the cache, and heldBytes, the bytes the cache
+// already holds, and at least DefaultCacheMaxBytesFloor. Counting what
+// the cache holds keeps the limit from shrinking as the cache fills.
+// pixad works it out when it builds the cache, as both numbers are known
+// only then.
+func DefaultCacheMaxBytes(freeBytes uint64, heldBytes int64) int64 {
+ // Both terms are at most math.MaxInt64, so the sum cannot overflow.
+ //nolint:gosec // G115: heldBytes is the cache's total, never negative
+ spaceBytes := min(freeBytes, math.MaxInt64) + uint64(heldBytes)
+
+ computed := spaceBytes / freeSpaceFractionDenominator * freeSpaceFractionNumerator
+ computed = min(computed, math.MaxInt64)
+
+ return max(int64(computed), DefaultCacheMaxBytesFloor)
+}
+
// placeholderSigningKey is the dummy signing_key shipped in
-// config.example.yml. It is 45 characters, so it passes the length
+// configs/config.example.yml. It is 45 characters, so it passes the length
// check, but it is public in this repository and must be rejected at
// startup so no deployment ever signs URLs with it.
const placeholderSigningKey = "CHANGE_ME_generate_with_openssl_rand_base64_32"
@@ -64,28 +94,31 @@ var (
errNotABoolean = errors.New("not a boolean")
errNotAStringList = errors.New("not a list of strings")
errNotAValidCIDR = errors.New("not a valid CIDR network")
- errNotAMetricsMap = errors.New("not a map of metrics settings")
+ errMappedTooShort = errors.New("in the IPv4-mapped form but shorter than /96")
+ errNotAMap = errors.New("not a map")
errEmptyListEntry = errors.New("list contains an empty entry")
errEmptyEntry = errors.New("contains an empty entry")
errNotAValidURL = errors.New("not a valid URL")
errPortOutOfRange = errors.New("outside the valid port range")
- errTooFewConnections = errors.New("must be at least 1")
+ errSizeOutOfRange = errors.New("outside the accepted range")
+ errMustBeAtLeastOne = errors.New("must be at least 1")
errValueTooShort = errors.New("value too short")
errPlaceholderKey = errors.New(
- "is the placeholder from config.example.yml; " +
+ "is the placeholder from configs/config.example.yml; " +
"generate a real key with: openssl rand -base64 32")
errMustBeSetTogether = errors.New("must be set together")
errMustNotBeNegative = errors.New("must not be negative")
errOverflowsInt64 = errors.New("overflows a 64-bit integer")
- errNegativeBlockSize = errors.New(
- "statfs reported negative block size")
- errValueNull = errors.New(
+ errValueNull = errors.New(
"value is null; omit the key entirely to use the default")
errValuesNull = errors.New(
"value is null; omit a key entirely to use its default")
- errNotBareHostname = errors.New(
- "must be a bare hostname without scheme, path, or whitespace")
- errNoHostnameLabels = errors.New("contains no hostname labels")
+ errNotAHost = errors.New("must be a host name such as " +
+ "cdn.example.com or .example.com, or an IP address")
+ errNotADuration = errors.New("not a duration such as 30s or 2m")
+ errMustBePositive = errors.New("must be positive")
+ errNotAnOrigin = errors.New(
+ `not "*" or an origin such as https://example.com`)
)
// Params defines dependencies for Config.
@@ -112,6 +145,38 @@ type Config struct {
AllowHTTP bool // Allow non-TLS upstream (testing only)
UpstreamConnectionsPerHost int // Max concurrent connections per upstream host
+ // RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
+ // image routes refuse a request whose Referer names a matching host.
+ RefererBlocklist []string
+
+ // UpstreamConnections is the most concurrent connections to all
+ // upstream hosts together, on top of the per-host limit.
+ // MaxConcurrentProcessing is the most images processed at once.
+ UpstreamConnections int
+ MaxConcurrentProcessing int
+
+ // UpstreamFetchTimeout is the time allowed for one fetch from an
+ // upstream host. UpstreamMaxResponseSize is the largest upstream
+ // response accepted, in bytes, and also the image processor's input
+ // limit.
+ UpstreamFetchTimeout time.Duration
+ UpstreamMaxResponseSize int64
+
+ // AccessControlAllowOrigin is the origin the CORS middleware allows
+ // to read responses: "*" for any, or one origin such as
+ // https://example.com.
+ AccessControlAllowOrigin string
+
+ // DownstreamTimeout bounds how long answering one client request may
+ // take. It is both the HTTP server's write timeout and the deadline
+ // of the per-request timeout middleware.
+ DownstreamTimeout time.Duration
+
+ // ImageRequestsPerMinute is how many requests one client may make to
+ // the image routes per minute; the next is refused with 429 Too Many
+ // Requests. 0 turns the limit off.
+ ImageRequestsPerMinute int
+
// BlockedNetworks are operator-supplied CIDR ranges to refuse in
// addition to the built-in SSRF blocklist. Enforced by the upstream
// fetcher's dialer; the built-in ranges always apply.
@@ -128,18 +193,15 @@ type Config struct {
// address, and an explicit list replaces the default.
TrustedProxies []netip.Prefix
- // CacheMaxBytes is the disk cache size limit in bytes. Zero
- // disables the disk cache entirely. When cache_max_bytes is
- // omitted from the configuration, this holds the computed default
- // (75% of free space on the filesystem containing
- // /cache/, floored at DefaultCacheMaxBytesFloor).
+ // CacheMaxBytes is the disk cache size limit in bytes, as set; 0
+ // turns the disk cache off.
CacheMaxBytes int64
- // cacheMaxBytesExplicit records whether cache_max_bytes was
+ // CacheMaxBytesExplicit records whether cache_max_bytes was
// explicitly set, in the environment or the configuration file.
- // Explicit values are used exactly as given; only an omitted key
- // gets the computed default (and its floor) in resolveCacheMaxBytes.
- cacheMaxBytesExplicit bool
+ // Explicit values are used exactly as given. For an omitted key
+ // pixad works out DefaultCacheMaxBytes when it builds the cache.
+ CacheMaxBytesExplicit bool
}
// New creates a new Config instance from the environment and the
@@ -176,9 +238,13 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) {
return nil, err
}
- err = c.resolveCacheMaxBytes(log, defaultFreeSpaceProbe)
- if err != nil {
- return nil, err
+ // An omitted cache_max_bytes is worked out and logged when pixad
+ // builds the cache.
+ if c.CacheMaxBytesExplicit {
+ log.Info("effective cache size limit",
+ "cache_max_bytes", c.CacheMaxBytes,
+ "cache_disabled", c.CacheMaxBytes == 0,
+ )
}
if c.Debug {
@@ -190,136 +256,268 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) {
// newFromSmartConfig constructs a Config from the environment and a
// loaded smartconfig instance, and validates it. A nil sc means no
-// config file was found, in which case every option the environment
+// config file was found, in which case every setting the environment
// does not set takes its default value. A key that is present but
// unparseable or invalid is an error: defaults apply only to omitted
// keys, never to invalid explicit values.
func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
- if sc != nil {
- err := validateKnownKeys(sc)
+ err := validateKnownKeys(sc)
+ if err != nil {
+ return nil, err
+ }
+
+ c := &Config{}
+
+ for _, s := range c.settings() {
+ err := s.load(lookupValue(sc, s))
if err != nil {
return nil, err
}
}
- err := validateAllowlistHostsValue(sc)
- if err != nil {
- return nil, err
- }
-
- blockedNetworks, err := parseCIDRList(sc, keyBlockedNetworks)
- if err != nil {
- return nil, err
- }
-
- trustedProxies, err := parseCIDRList(sc, keyTrustedProxies)
- if err != nil {
- return nil, err
- }
-
- // parseCIDRList returns a nil slice only when the key is absent; an
- // explicitly empty list ([]) comes back non-nil and empty. An omitted
- // key takes the RFC 1918 default, while an explicit empty list is left
- // as trust-nothing.
- if trustedProxies == nil {
- trustedProxies = defaultTrustedProxies()
- }
-
- loader := &strictLoader{sc: sc}
-
- c := &Config{
- Debug: loader.boolVal(keyDebug, false),
- MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
- Port: loader.intVal(keyPort, DefaultPort),
- StateDir: loader.stringVal(keyStateDir, DefaultStateDir),
- SentryDSN: loader.stringVal(keySentryDSN, ""),
- MetricsUsername: loader.stringVal(keyMetricsUsername, ""),
- MetricsPassword: loader.stringVal(keyMetricsPassword, ""),
- SigningKey: loader.stringVal(keySigningKey, ""),
- AllowlistHosts: getStringSlice(sc),
- AllowHTTP: loader.boolVal(keyAllowHTTP, false),
- UpstreamConnectionsPerHost: loader.intVal(
- keyUpstreamConnectionsPerHost, DefaultUpstreamConnectionsPerHost),
- CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
- BlockedNetworks: blockedNetworks,
- TrustedProxies: trustedProxies,
- }
-
- // The computed default for cache_max_bytes needs a validated
- // state_dir, so it is resolved later (resolveCacheMaxBytes); here
- // we only record whether the operator set the key explicitly.
- if _, present := lookupValue(sc, keyCacheMaxBytes); present {
- c.cacheMaxBytesExplicit = true
- }
-
- // Build DBURL from StateDir if not explicitly set. The derived URL
- // is a default: it applies only when db_url is omitted, never to an
- // explicitly empty value.
- c.DBURL = loader.stringVal(keyDBURL, "")
- if c.DBURL == "" && loader.err == nil {
- if _, present := lookupValue(sc, keyDBURL); present {
- return nil, fmt.Errorf("%s: %w; omit it to derive it from state_dir",
- settingName(keyDBURL), errValueEmpty)
- }
-
- c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_journal_mode=WAL", c.StateDir)
- }
-
- if loader.err != nil {
- return nil, loader.err
- }
-
- err = c.validate()
- if err != nil {
- return nil, err
+ // The metrics username and password are checked together, once both
+ // have loaded.
+ if (c.MetricsUsername == "") != (c.MetricsPassword == "") {
+ return nil, fmt.Errorf("%s and %s %w",
+ settingName("metrics.username"), settingName("metrics.password"),
+ errMustBeSetTogether)
}
return c, nil
}
+// A setting is one row of the settings table. key is its key in the
+// config file, and envVar is the environment variable that also sets it.
+// load reads the value set for it into its Config field.
+type setting struct {
+ key string
+ envVar string
+ load func(v value) error
+}
+
+// A value is what the environment or the config file sets for one
+// setting. present is false when neither sets it.
+type value struct {
+ key string
+ raw any
+ present bool
+}
+
+// settings returns the settings table: every setting pixa reads, once
+// each, in the order they load into c. Most rows fill their Config field
+// with field, whose arguments give the setting's type, its default and
+// its check. Code that needs only the keys and the variables reads the
+// table of a new Config.
+func (c *Config) settings() []setting {
+ return []setting{
+ {"signing_key", "PIXA_SIGNING_KEY",
+ field(&c.SigningKey, getString, "", validateSigningKey)},
+ {"port", "PORT", field(&c.Port, getInt, DefaultPort, validatePort)},
+ {"state_dir", "PIXA_STATE_DIR",
+ field(&c.StateDir, getString, DefaultStateDir, validateNotEmpty)},
+ {"db_url", "PIXA_DB_URL", c.loadDBURL},
+ {"cache_max_bytes", "PIXA_CACHE_MAX_BYTES", c.loadCacheMaxBytes},
+ {"allowlist_hosts", "PIXA_ALLOWLIST_HOSTS",
+ field(&c.AllowlistHosts, parseHostList, nil, nil)},
+ {"referer_blocklist", "PIXA_REFERER_BLOCKLIST",
+ field(&c.RefererBlocklist, parseHostList, nil, nil)},
+ {"blocked_networks", "PIXA_BLOCKED_NETWORKS",
+ field(&c.BlockedNetworks, parseCIDRList, nil, nil)},
+ {"trusted_proxies", "PIXA_TRUSTED_PROXIES",
+ field(&c.TrustedProxies, parseCIDRList, defaultTrustedProxies(), nil)},
+ {"allow_http", "PIXA_ALLOW_HTTP", field(&c.AllowHTTP, getBool, false, nil)},
+ {"upstream_connections_per_host", "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
+ field(&c.UpstreamConnectionsPerHost, getInt,
+ DefaultUpstreamConnectionsPerHost, validateAtLeastOne)},
+ {"upstream_connections", "PIXA_UPSTREAM_CONNECTIONS",
+ field(&c.UpstreamConnections, getInt,
+ DefaultUpstreamConnections, validateAtLeastOne)},
+ {"max_concurrent_processing", "PIXA_MAX_CONCURRENT_PROCESSING",
+ field(&c.MaxConcurrentProcessing, getInt,
+ DefaultMaxConcurrentProcessing(), validateAtLeastOne)},
+ {"image_requests_per_minute", "PIXA_IMAGE_REQUESTS_PER_MINUTE",
+ field(&c.ImageRequestsPerMinute, getInt,
+ DefaultImageRequestsPerMinute, validateNotNegative)},
+ {"upstream_fetch_timeout", "PIXA_UPSTREAM_FETCH_TIMEOUT",
+ field(&c.UpstreamFetchTimeout, getDuration,
+ DefaultUpstreamFetchTimeout, nil)},
+ {"upstream_max_response_size", "PIXA_UPSTREAM_MAX_RESPONSE_SIZE",
+ field(&c.UpstreamMaxResponseSize, getInt64,
+ DefaultUpstreamMaxResponseSize, validateUpstreamMaxResponseSize)},
+ {"downstream_timeout", "PIXA_DOWNSTREAM_TIMEOUT",
+ field(&c.DownstreamTimeout, getDuration,
+ DefaultDownstreamTimeout, nil)},
+ {"access_control_allow_origin", "PIXA_ACCESS_CONTROL_ALLOW_ORIGIN",
+ field(&c.AccessControlAllowOrigin, getString,
+ DefaultAccessControlAllowOrigin, validateAccessControlAllowOrigin)},
+ {"metrics.username", "PIXA_METRICS_USERNAME",
+ field(&c.MetricsUsername, getString, "", nil)},
+ {"metrics.password", "PIXA_METRICS_PASSWORD",
+ field(&c.MetricsPassword, getString, "", nil)},
+ {"sentry_dsn", "PIXA_SENTRY_DSN",
+ field(&c.SentryDSN, getString, "", validateSentryDSN)},
+ {"debug", "PIXA_DEBUG", field(&c.Debug, getBool, false, nil)},
+ {"maintenance_mode", "PIXA_MAINTENANCE_MODE",
+ field(&c.MaintenanceMode, getBool, false, nil)},
+ }
+}
+
+// field returns the load function of a setting that fills *dst. read
+// reads a value that is set, and so gives the setting's type. A setting
+// that is not set takes defaultValue. The value, a default too, must
+// then pass check, unless check is nil.
+func field[T any](
+ dst *T,
+ read func(v value) (T, error),
+ defaultValue T,
+ check func(key string, val T) error,
+) func(v value) error {
+ return func(v value) error {
+ val := defaultValue
+
+ if v.present {
+ var err error
+
+ val, err = read(v)
+ if err != nil {
+ return err
+ }
+ }
+
+ if check != nil {
+ err := check(v.key, val)
+ if err != nil {
+ return err
+ }
+ }
+
+ *dst = val
+
+ return nil
+ }
+}
+
+// loadDBURL loads db_url. An omitted db_url is derived from state_dir,
+// so db_url comes after state_dir in the table. An empty db_url is
+// refused, since defaults apply only to omitted keys.
+func (c *Config) loadDBURL(v value) error {
+ if !v.present {
+ c.DBURL = defaultDBURL(c.StateDir)
+
+ return nil
+ }
+
+ dbURL, err := getString(v)
+ if err != nil {
+ return err
+ }
+
+ if dbURL == "" {
+ return fmt.Errorf("%s: %w; omit it to derive it from state_dir",
+ settingName(v.key), errValueEmpty)
+ }
+
+ c.DBURL = dbURL
+
+ return nil
+}
+
+// loadCacheMaxBytes loads cache_max_bytes, and records whether it is set.
+// For an omitted cache_max_bytes pixad works out DefaultCacheMaxBytes
+// when it builds the cache.
+func (c *Config) loadCacheMaxBytes(v value) error {
+ c.CacheMaxBytesExplicit = v.present
+
+ return field(&c.CacheMaxBytes, getInt64, 0, validateNotNegative)(v)
+}
+
+// defaultDBURL returns the db_url used when db_url is omitted: the file
+// state.sqlite3 in stateDir, in WAL mode, which the driver sets only
+// through a _pragma parameter. String() percent-encodes any %, ? or # in
+// the directory name, which SQLite decodes back, so the URL names exactly
+// this file. OmitHost writes file:/var/lib/pixa/... instead of
+// file:///var/lib/pixa/..., and keeps a relative stateDir from being read
+// as a host name. The path written into the URL is stateDir followed by
+// /state.sqlite3; when it starts with //, as it does for a stateDir of /
+// or one starting with //, the empty host is kept, so the URL starts
+// file://// and SQLite does not read the first name in the path as a
+// host.
+func defaultDBURL(stateDir string) string {
+ dbPath := stateDir + "/state.sqlite3"
+
+ dbURL := url.URL{
+ Scheme: "file",
+ OmitHost: !strings.HasPrefix(dbPath, "//"),
+ Path: dbPath,
+ RawQuery: "_pragma=journal_mode(WAL)",
+ }
+
+ return dbURL.String()
+}
+
// validateKnownKeys rejects configuration files containing keys the
// application does not understand, so typos fail at startup instead of
// being silently ignored, and rejects keys that are explicitly set to
// null: a null is a SET value, never an omission, so it must not
-// silently take the default. The env section is permitted because
-// smartconfig consumes it for environment variable injection.
+// silently take the default. A setting whose key has a dot, such as
+// metrics.username, is written inside the map of the key's first part.
+// The env section is permitted because smartconfig consumes it for
+// environment variable injection. A nil sc means no config file, which
+// has no keys to check.
func validateKnownKeys(sc *smartconfig.Config) error {
+ if sc == nil {
+ return nil
+ }
+
+ topLevelKeys := map[string]bool{"env": true}
+ mapKeys := map[string]bool{}
+ keysInMaps := map[string]bool{}
+
+ for _, s := range new(Config).settings() {
+ mapKey, _, inMap := strings.Cut(s.key, ".")
+ topLevelKeys[mapKey] = true
+
+ if inMap {
+ mapKeys[mapKey] = true
+ keysInMaps[s.key] = true
+ }
+ }
+
var unknown, nullKeys []string
- for key, value := range sc.Data() {
- if !isKnownConfigKey(key) {
+ for key, val := range sc.Data() {
+ switch {
+ case !topLevelKeys[key]:
unknown = append(unknown, key)
-
- continue
- }
-
- if value == nil {
+ case val == nil:
nullKeys = append(nullKeys, key)
-
- continue
- }
-
- if key == keyMetrics {
- metricsMap, ok := value.(map[string]any)
+ case mapKeys[key]:
+ settingsMap, ok := val.(map[string]any)
if !ok {
- return fmt.Errorf("config key %q: value %v is %w",
- keyMetrics, value, errNotAMetricsMap)
+ return fmt.Errorf("config key %q: value %v is %w of %s settings",
+ key, val, errNotAMap, key)
}
- for subkey, subvalue := range metricsMap {
- if subkey != "username" && subkey != "password" {
- unknown = append(unknown, keyMetrics+"."+subkey)
+ for subkey, subvalue := range settingsMap {
+ fullKey := key + "." + subkey
- continue
- }
-
- if subvalue == nil {
- nullKeys = append(nullKeys, keyMetrics+"."+subkey)
+ switch {
+ case !keysInMaps[fullKey]:
+ unknown = append(unknown, fullKey)
+ case subvalue == nil:
+ nullKeys = append(nullKeys, fullKey)
}
}
}
}
+ return unknownOrNullKeysError(unknown, nullKeys)
+}
+
+// unknownOrNullKeysError reports the unknown keys validateKnownKeys
+// found, or else the keys set to null. It returns nil when there are
+// neither.
+func unknownOrNullKeysError(unknown, nullKeys []string) error {
if len(unknown) > 0 {
sort.Strings(unknown)
@@ -347,44 +545,6 @@ func errNullConfigValue(key string) error {
return fmt.Errorf("config key %q: %w", key, errValueNull)
}
-// isKnownConfigKey reports whether key is a permitted top-level
-// configuration key.
-func isKnownConfigKey(key string) bool {
- switch key {
- case keyDebug, keyMaintenanceMode, keyPort, keyStateDir, keySentryDSN,
- keyDBURL, keyMetrics, keySigningKey, keyAllowlistHosts, keyAllowHTTP,
- keyUpstreamConnectionsPerHost, keyCacheMaxBytes, keyBlockedNetworks,
- keyTrustedProxies, "env":
- return true
- }
-
- return false
-}
-
-// envVarNames returns, for each configuration key, the environment
-// variable that also sets it: PIXA_ plus the key in upper case, with "."
-// written as "_", except the port, which REPO_POLICIES.md requires to be
-// PORT. metrics is set through its two subkeys; env has no variable.
-func envVarNames() map[string]string {
- return map[string]string{ //nolint:gosec // G101: variable names, not secrets
- keyDebug: "PIXA_DEBUG",
- keyMaintenanceMode: "PIXA_MAINTENANCE_MODE",
- keyPort: "PORT",
- keyStateDir: "PIXA_STATE_DIR",
- keySentryDSN: "PIXA_SENTRY_DSN",
- keyDBURL: "PIXA_DB_URL",
- keyMetricsUsername: "PIXA_METRICS_USERNAME",
- keyMetricsPassword: "PIXA_METRICS_PASSWORD",
- keySigningKey: "PIXA_SIGNING_KEY",
- keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS",
- keyAllowHTTP: "PIXA_ALLOW_HTTP",
- keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
- keyCacheMaxBytes: "PIXA_CACHE_MAX_BYTES",
- keyBlockedNetworks: "PIXA_BLOCKED_NETWORKS",
- keyTrustedProxies: "PIXA_TRUSTED_PROXIES",
- }
-}
-
// validateKnownEnvVars rejects environment variables whose names start
// with PIXA_ but that are neither a setting's variable nor
// PIXA_CONFIG_PATH, so a misspelled variable fails at startup instead of
@@ -394,8 +554,8 @@ func envVarNames() map[string]string {
func validateKnownEnvVars() error {
known := map[string]bool{"PIXA_CONFIG_PATH": true}
- for _, name := range envVarNames() {
- known[name] = true
+ for _, s := range new(Config).settings() {
+ known[s.envVar] = true
}
var unknown []string
@@ -422,28 +582,37 @@ func validateKnownEnvVars() error {
return nil
}
-// lookupValue returns the value set for key and whether one is set. The
-// key's environment variable wins when it is present, even when empty;
-// its value is a string, read exactly as the same text quoted in the
-// config file would be. Otherwise the config file's value is used.
-func lookupValue(sc *smartconfig.Config, key string) (any, bool) {
- value, present := os.LookupEnv(envVarNames()[key])
+// lookupValue returns the value set for s. Its environment variable wins
+// when it is present, even when empty; its value is a string, read
+// exactly as the same text quoted in the config file would be. Otherwise
+// the config file's value is used.
+func lookupValue(sc *smartconfig.Config, s setting) value {
+ envValue, present := os.LookupEnv(s.envVar)
if present {
- return value, true
+ return value{key: s.key, raw: envValue, present: true}
}
if sc == nil {
- return nil, false
+ return value{key: s.key}
}
- return sc.Get(key)
+ fileValue, present := sc.Get(s.key)
+
+ return value{key: s.key, raw: fileValue, present: present}
}
-// settingName names key in an error message together with its
-// environment variable, since either one may have set the value.
+// settingName names the setting key in an error message together with
+// its environment variable, since either one may have set the value.
func settingName(key string) string {
- return fmt.Sprintf("config key %q (environment variable %s)",
- key, envVarNames()[key])
+ var envVar string
+
+ for _, s := range new(Config).settings() {
+ if s.key == key {
+ envVar = s.envVar
+ }
+ }
+
+ return fmt.Sprintf("config key %q (environment variable %s)", key, envVar)
}
// ensureStateDirWritable verifies at startup that StateDir can be
@@ -452,16 +621,18 @@ func settingName(key string) string {
func (c *Config) ensureStateDirWritable() error {
const stateDirPerms = 0o750
+ name := settingName("state_dir")
+
err := os.MkdirAll(c.StateDir, stateDirPerms)
if err != nil {
return fmt.Errorf("%s: cannot create directory %q: %w",
- settingName(keyStateDir), c.StateDir, err)
+ name, c.StateDir, err)
}
probe, err := os.CreateTemp(c.StateDir, ".startup-write-probe-*")
if err != nil {
return fmt.Errorf("%s: directory %q is not writable: %w",
- settingName(keyStateDir), c.StateDir, err)
+ name, c.StateDir, err)
}
probePath := probe.Name()
@@ -469,117 +640,192 @@ func (c *Config) ensureStateDirWritable() error {
err = probe.Close()
if err != nil {
return fmt.Errorf("%s: cannot close probe file %q: %w",
- settingName(keyStateDir), probePath, err)
+ name, probePath, err)
}
err = os.Remove(probePath)
if err != nil {
return fmt.Errorf("%s: cannot remove probe file %q: %w",
- settingName(keyStateDir), probePath, err)
+ name, probePath, err)
}
return nil
}
// validateSigningKey checks that the signing key is present, long
-// enough, and not the public placeholder from config.example.yml. The
-// key value itself is never echoed in error messages.
-func (c *Config) validateSigningKey() error {
- if c.SigningKey == "" {
- return fmt.Errorf("%s: %w", settingName(keySigningKey), errValueRequired)
+// enough, and not the public placeholder from configs/config.example.yml.
+// The key value itself is never echoed in error messages.
+func validateSigningKey(key, signingKey string) error {
+ if signingKey == "" {
+ return fmt.Errorf("%s: %w", settingName(key), errValueRequired)
}
// Minimum key length for security (32 bytes = 256 bits)
const minKeyLength = 32
- if len(c.SigningKey) < minKeyLength {
+ if len(signingKey) < minKeyLength {
return fmt.Errorf("%s: %w: must be at least %d characters, got %d",
- settingName(keySigningKey), errValueTooShort, minKeyLength,
- len(c.SigningKey))
+ settingName(key), errValueTooShort, minKeyLength, len(signingKey))
}
- if c.SigningKey == placeholderSigningKey {
- return fmt.Errorf("%s: %w", settingName(keySigningKey), errPlaceholderKey)
+ if signingKey == placeholderSigningKey {
+ return fmt.Errorf("%s: %w", settingName(key), errPlaceholderKey)
}
return nil
}
-// validate checks that all required configuration values are set and
-// that every value is within its valid range.
-func (c *Config) validate() error {
- err := c.validateSigningKey()
- if err != nil {
- return err
- }
-
+// validatePort checks that a port is from 1 to 65535.
+func validatePort(key string, port int) error {
const maxPort = 65535
- if c.Port < 1 || c.Port > maxPort {
+ if port < 1 || port > maxPort {
return fmt.Errorf("%s: value %d is %w 1-%d",
- settingName(keyPort), c.Port, errPortOutOfRange, maxPort)
- }
-
- if c.UpstreamConnectionsPerHost < 1 {
- return fmt.Errorf("%s: value %d %w",
- settingName(keyUpstreamConnectionsPerHost),
- c.UpstreamConnectionsPerHost, errTooFewConnections)
- }
-
- if c.StateDir == "" {
- return fmt.Errorf("%s: %w", settingName(keyStateDir), errValueEmpty)
- }
-
- // Zero is valid (it disables the disk cache); only negative
- // values are rejected. No floor applies to explicit values.
- if c.CacheMaxBytes < 0 {
- return fmt.Errorf("%s: value %d %w",
- settingName(keyCacheMaxBytes), c.CacheMaxBytes, errMustNotBeNegative)
- }
-
- for _, host := range c.AllowlistHosts {
- err := validateAllowlistHost(host)
- if err != nil {
- return err
- }
- }
-
- if c.SentryDSN != "" {
- parsed, err := url.Parse(c.SentryDSN)
- if err != nil || parsed.Scheme == "" || parsed.Host == "" {
- return fmt.Errorf("%s: value %q is %w",
- settingName(keySentryDSN), c.SentryDSN, errNotAValidURL)
- }
- }
-
- if (c.MetricsUsername == "") != (c.MetricsPassword == "") {
- return fmt.Errorf("%s and %s %w",
- settingName(keyMetricsUsername), settingName(keyMetricsPassword),
- errMustBeSetTogether)
+ settingName(key), port, errPortOutOfRange, maxPort)
}
return nil
}
-// validateAllowlistHost checks that an allowlist_hosts entry is a bare
-// hostname, optionally with a leading dot for suffix matching. URLs,
-// paths, and whitespace indicate a misconfigured entry. An entry with
-// no hostname labels (such as ".") is rejected: the allowlist matcher
-// treats a leading dot as a suffix pattern, so a bare "." would match
-// any upstream host written in FQDN trailing-dot form and effectively
-// disable URL signing.
-func validateAllowlistHost(host string) error {
- if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
- return fmt.Errorf("%s: entry %q %w",
- settingName(keyAllowlistHosts), host, errNotBareHostname)
- }
-
- if strings.Trim(host, ".") == "" {
- return fmt.Errorf("%s: entry %q %w",
- settingName(keyAllowlistHosts), host, errNoHostnameLabels)
+// validateNotEmpty checks that a value is not empty.
+func validateNotEmpty(key, val string) error {
+ if val == "" {
+ return fmt.Errorf("%s: %w", settingName(key), errValueEmpty)
}
return nil
}
+// validateAtLeastOne checks that a limit, such as the most images
+// processed at once, is at least 1.
+func validateAtLeastOne(key string, limit int) error {
+ if limit < 1 {
+ return fmt.Errorf("%s: value %d %w",
+ settingName(key), limit, errMustBeAtLeastOne)
+ }
+
+ return nil
+}
+
+// validateNotNegative checks that a number is not negative. Zero is valid
+// for the two settings that use it: it turns the disk cache or the limit
+// on image requests off.
+func validateNotNegative[T int | int64](key string, number T) error {
+ if number < 0 {
+ return fmt.Errorf("%s: value %d %w",
+ settingName(key), number, errMustNotBeNegative)
+ }
+
+ return nil
+}
+
+// validateSentryDSN checks that sentry_dsn, when set, is a URL with a scheme
+// and a host.
+func validateSentryDSN(key, dsn string) error {
+ if dsn == "" {
+ return nil
+ }
+
+ parsed, err := url.Parse(dsn)
+ if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+ return fmt.Errorf("%s: value %q is %w",
+ settingName(key), dsn, errNotAValidURL)
+ }
+
+ return nil
+}
+
+// validateUpstreamMaxResponseSize checks that upstream_max_response_size
+// is from 1 byte to 1 GiB. An upstream response is read whole into
+// memory, and the image processor reads one byte past this limit, which
+// must not overflow.
+func validateUpstreamMaxResponseSize(key string, size int64) error {
+ const maxUpstreamMaxResponseSize = 1 << 30 // 1 GiB
+ if size < 1 || size > maxUpstreamMaxResponseSize {
+ return fmt.Errorf("%s: value %d is %w 1-%d",
+ settingName(key), size, errSizeOutOfRange, maxUpstreamMaxResponseSize)
+ }
+
+ return nil
+}
+
+// validateAccessControlAllowOrigin accepts "*" or an origin exactly as a browser
+// sends it: http or https, an IP address as netip writes it or a lowercase name
+// with a letter in its last part, and an optional port 1-65535, not the default.
+func validateAccessControlAllowOrigin(key, origin string) error {
+ if origin == "*" {
+ return nil
+ }
+
+ errOrigin := fmt.Errorf("%s: value %q is %w",
+ settingName(key), origin, errNotAnOrigin)
+
+ parsed, err := url.Parse(origin)
+ if err != nil {
+ return errOrigin
+ }
+
+ defaultPort := map[string]string{"http": "80", "https": "443"}[parsed.Scheme]
+ if defaultPort == "" {
+ return errOrigin
+ }
+
+ const letters = "abcdefghijklmnopqrstuvwxyz"
+
+ host := parsed.Hostname()
+ lastPart := host[strings.LastIndex(host, ".")+1:]
+
+ addr, err := netip.ParseAddr(host)
+
+ switch {
+ case err == nil && addr.Is6():
+ host = "[" + addr.String() + "]"
+ case err == nil:
+ host = addr.String()
+ case strings.Trim(host, letters+"0123456789-.") != "": // a character other than these
+ return errOrigin
+ case !strings.ContainsAny(lastPart, letters):
+ return errOrigin
+ }
+
+ // The value must be exactly the origin rebuilt from its parts.
+ rebuilt := parsed.Scheme + "://" + host
+
+ port := parsed.Port()
+ if port != "" {
+ _, err := strconv.ParseUint(port, 10, 16)
+ if err != nil || port[0] == '0' || port == defaultPort {
+ return errOrigin
+ }
+
+ rebuilt += ":" + port
+ }
+
+ if rebuilt != origin {
+ return errOrigin
+ }
+
+ return nil
+}
+
+// hostNamePattern matches a host name: letters, digits, hyphens, underscores
+// and dots, optionally after one leading dot.
+var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
+
+// validateHostPattern checks that an entry of the named key, allowlist_hosts
+// or referer_blocklist, is an IP address or a host name, the host name
+// optionally with one leading dot for suffix matching. Anything else, such as
+// a URL, a port or a "*." wildcard, can never match a host name that resolves,
+// so it is refused.
+// So is "." alone: the allowlist matcher would match it against any host
+// written with a trailing dot, which in allowlist_hosts disables URL signing.
+func validateHostPattern(key, host string) error {
+ _, err := netip.ParseAddr(host)
+ if err == nil || hostNamePattern.MatchString(host) {
+ return nil
+ }
+
+ return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
+}
+
// loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
// or standard locations.
func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, error) {
@@ -608,120 +854,58 @@ func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, erro
for _, path := range configPaths {
cleanPath := filepath.Clean(path)
+ // Only a config file that does not exist is skipped, including
+ // one whose path runs through a file, such as under a HOME of
+ // /dev/null. One that cannot be read or does not parse is a
+ // fatal startup error.
+ //nolint:gosec // G703: a fixed config location; HOME is the operator's
_, statErr := os.Stat(cleanPath)
- if statErr == nil {
- // A config file that exists but does not parse is a fatal
- // startup error, never something to skip over.
- sc, err := smartconfig.NewFromConfigPath(path)
- if err != nil {
- return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
- }
-
- log.Info("loaded config file", "path", path)
-
- return sc, nil
+ if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
+ continue
}
+
+ if statErr != nil {
+ return nil, fmt.Errorf("failed to read config file %s: %w", path, statErr)
+ }
+
+ sc, err := smartconfig.NewFromConfigPath(path)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
+ }
+
+ log.Info("loaded config file", "path", path)
+
+ return sc, nil
}
return nil, nil //nolint:nilnil // nil config is valid (use defaults)
}
-// strictLoader accumulates the first error encountered while reading
-// typed values out of a smartconfig instance, so Config construction
-// can stay a single struct literal.
-type strictLoader struct {
- sc *smartconfig.Config
- err error
-}
-
-func (l *strictLoader) stringVal(key, defaultVal string) string {
- if l.err != nil {
- return ""
+// getString returns the string v holds. A value that is not a string, or
+// is explicitly null, is an error.
+func getString(v value) (string, error) {
+ if v.raw == nil {
+ return "", errNullConfigValue(v.key)
}
- val, err := getString(l.sc, key, defaultVal)
- if err != nil {
- l.err = err
- }
-
- return val
-}
-
-func (l *strictLoader) intVal(key string, defaultVal int) int {
- if l.err != nil {
- return 0
- }
-
- val, err := getInt(l.sc, key, defaultVal)
- if err != nil {
- l.err = err
- }
-
- return val
-}
-
-func (l *strictLoader) int64Val(key string, defaultVal int64) int64 {
- if l.err != nil {
- return 0
- }
-
- val, err := getInt64(l.sc, key, defaultVal)
- if err != nil {
- l.err = err
- }
-
- return val
-}
-
-func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
- if l.err != nil {
- return false
- }
-
- val, err := getBool(l.sc, key, defaultVal)
- if err != nil {
- l.err = err
- }
-
- return val
-}
-
-// getString returns the string value for key, or defaultVal if the key
-// is omitted. A present value that is not a string, or is explicitly
-// null, is an error.
-func getString(sc *smartconfig.Config, key, defaultVal string) (string, error) {
- raw, ok := lookupValue(sc, key)
- if !ok {
- return defaultVal, nil
- }
-
- if raw == nil {
- return "", errNullConfigValue(key)
- }
-
- str, ok := raw.(string)
+ str, ok := v.raw.(string)
if !ok {
return "", fmt.Errorf("config key %q: value %v (%T) is %w",
- key, raw, raw, errNotAString)
+ v.key, v.raw, v.raw, errNotAString)
}
return str, nil
}
-// getInt returns the integer value for key, or defaultVal if the key is
-// omitted. A present value that is not a whole number, or is explicitly
-// null, is an error; fractional values are never truncated.
-func getInt(sc *smartconfig.Config, key string, defaultVal int) (int, error) {
- raw, ok := lookupValue(sc, key)
- if !ok {
- return defaultVal, nil
+// getInt returns the integer v holds. A value that is not a whole
+// number, or is explicitly null, is an error; fractional values are
+// never truncated.
+func getInt(v value) (int, error) {
+ if v.raw == nil {
+ return 0, errNullConfigValue(v.key)
}
- if raw == nil {
- return 0, errNullConfigValue(key)
- }
-
- switch val := raw.(type) {
+ switch val := v.raw.(type) {
case int:
return val, nil
case int64:
@@ -729,7 +913,7 @@ func getInt(sc *smartconfig.Config, key string, defaultVal int) (int, error) {
case float64:
if val != math.Trunc(val) {
return 0, fmt.Errorf("config key %q: value %v is %w",
- key, val, errNotAnInteger)
+ v.key, val, errNotAnInteger)
}
return int(val), nil
@@ -737,31 +921,25 @@ func getInt(sc *smartconfig.Config, key string, defaultVal int) (int, error) {
parsed, err := strconv.Atoi(strings.TrimSpace(val))
if err != nil {
return 0, fmt.Errorf("%s: value %q is %w",
- settingName(key), val, errNotAnInteger)
+ settingName(v.key), val, errNotAnInteger)
}
return parsed, nil
default:
return 0, fmt.Errorf("config key %q: value %v (%T) is %w",
- key, raw, raw, errNotAnInteger)
+ v.key, v.raw, v.raw, errNotAnInteger)
}
}
-// getInt64 returns the 64-bit integer value for key, or defaultVal if
-// the key is omitted. A present value that is not a whole number, or
-// is explicitly null, is an error; fractional values are never
-// truncated and out-of-range values are never clamped.
-func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, error) {
- raw, ok := lookupValue(sc, key)
- if !ok {
- return defaultVal, nil
+// getInt64 returns the 64-bit integer v holds. A value that is not a
+// whole number, or is explicitly null, is an error; fractional values
+// are never truncated and out-of-range values are never clamped.
+func getInt64(v value) (int64, error) {
+ if v.raw == nil {
+ return 0, errNullConfigValue(v.key)
}
- if raw == nil {
- return 0, errNullConfigValue(key)
- }
-
- switch val := raw.(type) {
+ switch val := v.raw.(type) {
case int:
return int64(val), nil
case int64:
@@ -769,14 +947,14 @@ func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, erro
case uint64:
if val > math.MaxInt64 {
return 0, fmt.Errorf("config key %q: value %d %w",
- key, val, errOverflowsInt64)
+ v.key, val, errOverflowsInt64)
}
return int64(val), nil
case float64:
if val != math.Trunc(val) {
return 0, fmt.Errorf("config key %q: value %v is %w",
- key, val, errNotAnInteger)
+ v.key, val, errNotAnInteger)
}
return int64(val), nil
@@ -784,138 +962,70 @@ func getInt64(sc *smartconfig.Config, key string, defaultVal int64) (int64, erro
parsed, err := strconv.ParseInt(strings.TrimSpace(val), 10, 64)
if err != nil {
return 0, fmt.Errorf("%s: value %q is %w",
- settingName(key), val, errNotAnInteger)
+ settingName(v.key), val, errNotAnInteger)
}
return parsed, nil
default:
return 0, fmt.Errorf("config key %q: value %v (%T) is %w",
- key, raw, raw, errNotAnInteger)
+ v.key, v.raw, v.raw, errNotAnInteger)
}
}
-// getBool returns the boolean value for key, or defaultVal if the key
-// is omitted. A present value that is not a boolean (or a ParseBool-able
-// string), or is explicitly null, is an error; numbers are not accepted
-// as booleans.
-func getBool(sc *smartconfig.Config, key string, defaultVal bool) (bool, error) {
- raw, ok := lookupValue(sc, key)
+// getDuration returns the duration v holds. The value must be a positive
+// Go duration string such as "30s" or "2m", read with
+// time.ParseDuration; a bare number has no unit and is an error, as is
+// an explicit null.
+func getDuration(v value) (time.Duration, error) {
+ if v.raw == nil {
+ return 0, errNullConfigValue(v.key)
+ }
+
+ str, ok := v.raw.(string)
if !ok {
- return defaultVal, nil
+ return 0, fmt.Errorf("config key %q: value %v (%T) is %w",
+ v.key, v.raw, v.raw, errNotADuration)
}
- if raw == nil {
- return false, errNullConfigValue(key)
+ parsed, err := time.ParseDuration(strings.TrimSpace(str))
+ if err != nil {
+ return 0, fmt.Errorf("%s: value %q is %w",
+ settingName(v.key), str, errNotADuration)
}
- switch val := raw.(type) {
+ if parsed <= 0 {
+ return 0, fmt.Errorf("%s: value %q %w",
+ settingName(v.key), str, errMustBePositive)
+ }
+
+ return parsed, nil
+}
+
+// getBool returns the boolean v holds. A value that is not a boolean (or
+// a ParseBool-able string), or is explicitly null, is an error; numbers
+// are not accepted as booleans.
+func getBool(v value) (bool, error) {
+ if v.raw == nil {
+ return false, errNullConfigValue(v.key)
+ }
+
+ switch val := v.raw.(type) {
case bool:
return val, nil
case string:
parsed, err := strconv.ParseBool(strings.TrimSpace(val))
if err != nil {
return false, fmt.Errorf("%s: value %q is %w",
- settingName(key), val, errNotABoolean)
+ settingName(v.key), val, errNotABoolean)
}
return parsed, nil
default:
return false, fmt.Errorf("config key %q: value %v (%T) is %w",
- key, raw, raw, errNotABoolean)
+ v.key, v.raw, v.raw, errNotABoolean)
}
}
-// validateAllowlistHostsValue checks the raw shape of the
-// allowlist_hosts value before the lenient extraction in getStringSlice
-// runs: an explicitly null value, a value that is not a list of strings
-// (or a comma-separated string), a non-string entry, or an empty entry
-// is an error, never silently skipped.
-func validateAllowlistHostsValue(sc *smartconfig.Config) error {
- raw, ok := lookupValue(sc, keyAllowlistHosts)
- if !ok {
- return nil
- }
-
- if raw == nil {
- return errNullConfigValue(keyAllowlistHosts)
- }
-
- switch val := raw.(type) {
- case []any:
- for _, item := range val {
- str, ok := item.(string)
- if !ok {
- return fmt.Errorf("config key %q: list entry %v (%T) is %w",
- keyAllowlistHosts, item, item, errNotAString)
- }
-
- if strings.TrimSpace(str) == "" {
- return fmt.Errorf("config key %q: %w",
- keyAllowlistHosts, errEmptyListEntry)
- }
- }
- case string:
- if strings.TrimSpace(val) == "" {
- return nil
- }
-
- for part := range strings.SplitSeq(val, ",") {
- if strings.TrimSpace(part) == "" {
- return fmt.Errorf("%s: value %q %w",
- settingName(keyAllowlistHosts), val, errEmptyEntry)
- }
- }
- default:
- return fmt.Errorf("config key %q: value %v (%T) is %w",
- keyAllowlistHosts, raw, raw, errNotAStringList)
- }
-
- return nil
-}
-
-// getStringSlice returns the allowlist_hosts list of strings, or nil if
-// the key is omitted. It accepts a YAML list of strings or a
-// comma-separated string (backwards compatibility). Malformed entries
-// are rejected beforehand by validateAllowlistHostsValue.
-func getStringSlice(sc *smartconfig.Config) []string {
- val, ok := lookupValue(sc, keyAllowlistHosts)
- if !ok || val == nil {
- return nil
- }
-
- // Handle YAML list format
- if slice, ok := val.([]any); ok {
- result := make([]string, 0, len(slice))
- for _, item := range slice {
- if str, ok := item.(string); ok {
- trimmed := strings.TrimSpace(str)
- if trimmed != "" {
- result = append(result, trimmed)
- }
- }
- }
-
- return result
- }
-
- // Fall back to comma-separated string for backwards compatibility
- if str, ok := val.(string); ok && str != "" {
- parts := strings.Split(str, ",")
- result := make([]string, 0, len(parts))
-
- for _, part := range parts {
- trimmed := strings.TrimSpace(part)
- if trimmed != "" {
- result = append(result, trimmed)
- }
- }
-
- return result
- }
-
- return nil
-}
-
// defaultTrustedProxies returns the trusted_proxies default: the three RFC
// 1918 private ranges. pixa is always deployed behind a TLS-terminating
// reverse proxy, which in practice sits on a private network, so its
@@ -931,34 +1041,48 @@ func defaultTrustedProxies() []netip.Prefix {
}
}
-// parseCIDRList parses the value of the named config key into CIDR
-// prefixes, or returns nil if the key is omitted. It accepts a YAML list
-// of strings or a comma-separated string. An explicitly null value, a
-// wrong type, an empty entry, a non-string entry, or an unparseable CIDR
-// aborts startup naming the key and the offending value; the default
-// (an empty list) applies only to an omitted key.
-func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
- raw, ok := lookupValue(sc, key)
- if !ok {
- return nil, nil
+// parseCIDRList parses the list v holds into CIDR prefixes. It accepts a
+// YAML list of strings or a comma-separated string. An explicitly null
+// value, a wrong type, an empty entry, a non-string entry, or an
+// unparseable CIDR aborts startup naming the key and the offending value.
+//
+// An entry written in the IPv4-mapped form (::ffff:a.b.c.d) is kept as the
+// IPv4 range it covers: ::ffff:10.0.0.0/104 is kept as 10.0.0.0/8. Both
+// blocked_networks and trusted_proxies are matched against an IPv4-mapped
+// address as the IPv4 address it holds, so the entry as written would match
+// nothing. Such an entry shorter than /96 covers more than the IPv4
+// addresses and aborts startup.
+func parseCIDRList(v value) ([]netip.Prefix, error) {
+ if v.raw == nil {
+ return nil, errNullConfigValue(v.key)
}
- if raw == nil {
- return nil, errNullConfigValue(key)
- }
-
- entries, err := cidrListEntries(raw, key)
+ entries, err := listEntries(v.raw, v.key)
if err != nil {
return nil, err
}
+ // An IPv4-mapped address is the 96 bits of ::ffff:0:0/96 followed by
+ // the 32 bits of the IPv4 address it holds.
+ const mappedPrefixBits = 96
+
prefixes := make([]netip.Prefix, 0, len(entries))
for _, entry := range entries {
prefix, err := netip.ParsePrefix(entry)
if err != nil {
return nil, fmt.Errorf("%s: value %q is %w",
- settingName(key), entry, errNotAValidCIDR)
+ settingName(v.key), entry, errNotAValidCIDR)
+ }
+
+ if prefix.Addr().Is4In6() && prefix.Bits() < mappedPrefixBits {
+ return nil, fmt.Errorf("%s: value %q is %w",
+ settingName(v.key), entry, errMappedTooShort)
+ }
+
+ if prefix.Addr().Is4In6() {
+ prefix = netip.PrefixFrom(prefix.Addr().Unmap(),
+ prefix.Bits()-mappedPrefixBits)
}
prefixes = append(prefixes, prefix)
@@ -967,11 +1091,36 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
return prefixes, nil
}
-// cidrListEntries extracts the raw entries of the named CIDR-list key as
-// trimmed, non-empty strings, from either a YAML list of strings or a
-// comma-separated string; an empty string is an empty list, as for
-// allowlist_hosts. Any other shape is a configuration error.
-func cidrListEntries(raw any, key string) ([]string, error) {
+// parseHostList parses the list v holds into host patterns. It accepts a
+// YAML list of strings or a comma-separated string. An explicitly null
+// value, a wrong type, an empty entry, a non-string entry, or an entry
+// validateHostPattern rejects aborts startup naming the key and the
+// offending value.
+func parseHostList(v value) ([]string, error) {
+ if v.raw == nil {
+ return nil, errNullConfigValue(v.key)
+ }
+
+ entries, err := listEntries(v.raw, v.key)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, entry := range entries {
+ err := validateHostPattern(v.key, entry)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return entries, nil
+}
+
+// listEntries extracts the raw entries of the named list key as trimmed,
+// non-empty strings, from either a YAML list of strings or a comma-separated
+// string; an empty string is an empty list. Any other shape is a
+// configuration error.
+func listEntries(raw any, key string) ([]string, error) {
switch val := raw.(type) {
case []any:
entries := make([]string, 0, len(val))
diff --git a/internal/config/config_internal_test.go b/internal/config/config_internal_test.go
index de33d40..bae94c7 100644
--- a/internal/config/config_internal_test.go
+++ b/internal/config/config_internal_test.go
@@ -1,94 +1,35 @@
package config
import (
- "os"
"path/filepath"
"testing"
"git.eeqj.de/sneak/smartconfig"
)
-// writeTestConfig writes yamlContent to a temp config file and returns
-// the file path.
-func writeTestConfig(t *testing.T, yamlContent string) string {
- t.Helper()
+// TestExampleConfigAllowlistsNoHost loads configs/config.example.yml
+// with a real signing key, as Getting Started in README.md has it copied
+// and edited, and checks that it allowlists no host: every image on an
+// allowlisted host is served to anyone without a signature.
+func TestExampleConfigAllowlistsNoHost(t *testing.T) {
+ t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
- tmpDir := t.TempDir()
- configPath := filepath.Join(tmpDir, "config.yml")
+ examplePath := filepath.Join("..", "..", "configs", "config.example.yml")
- err := os.WriteFile(configPath, []byte(yamlContent), 0o600)
+ sc, err := loadTestConfig(examplePath)
if err != nil {
- t.Fatalf("failed to write test config: %v", err)
+ t.Fatalf("failed to load %s: %v", examplePath, err)
}
- return configPath
-}
-
-// checkAllowlistHosts loads the config at configPath and asserts that
-// getStringSlice returns the three expected hosts.
-func checkAllowlistHosts(t *testing.T, configPath string) {
- t.Helper()
-
- sc, err := loadTestConfig(configPath)
+ c, err := newFromSmartConfig(sc)
if err != nil {
- t.Fatalf("failed to load config: %v", err)
+ t.Fatalf("%s is not a valid config: %v", examplePath, err)
}
- hosts := getStringSlice(sc)
+ t.Logf("allowlist_hosts in %s: %v", examplePath, c.AllowlistHosts)
- if len(hosts) != 3 {
- t.Errorf("expected 3 hosts, got %d: %v", len(hosts), hosts)
- }
-
- expected := []string{"static.sneak.cloud", "sneak.berlin", testHostS3}
- for i, want := range expected {
- if i >= len(hosts) {
- t.Errorf("missing host at index %d: want %q", i, want)
-
- continue
- }
-
- if hosts[i] != want {
- t.Errorf("host[%d] = %q, want %q", i, hosts[i], want)
- }
- }
-}
-
-func TestGetStringSlice_YAMLList(t *testing.T) {
- t.Parallel()
-
- yamlContent := `
-allowlist_hosts:
- - static.sneak.cloud
- - sneak.berlin
- - s3.sneak.cloud
-`
-
- checkAllowlistHosts(t, writeTestConfig(t, yamlContent))
-}
-
-func TestGetStringSlice_CommaSeparated(t *testing.T) {
- t.Parallel()
-
- // Backwards compatibility with comma-separated string values.
- yamlContent := `allowlist_hosts: "static.sneak.cloud, sneak.berlin, s3.sneak.cloud"`
-
- checkAllowlistHosts(t, writeTestConfig(t, yamlContent))
-}
-
-func TestGetStringSlice_Empty(t *testing.T) {
- t.Parallel()
-
- configPath := writeTestConfig(t, `port: 8080`)
-
- sc, err := loadTestConfig(configPath)
- if err != nil {
- t.Fatalf("failed to load config: %v", err)
- }
-
- hosts := getStringSlice(sc)
- if len(hosts) != 0 {
- t.Errorf("expected nil or empty slice, got %v", hosts)
+ if len(c.AllowlistHosts) != 0 {
+ t.Errorf("%s allowlists %v, want no host", examplePath, c.AllowlistHosts)
}
}
diff --git a/internal/config/config_validation_internal_test.go b/internal/config/config_validation_internal_test.go
index c12d27d..1aaac63 100644
--- a/internal/config/config_validation_internal_test.go
+++ b/internal/config/config_validation_internal_test.go
@@ -1,17 +1,50 @@
package config
import (
+ "database/sql"
"log/slog"
"os"
"path/filepath"
+ "slices"
"strings"
"testing"
+ "time"
"git.eeqj.de/sneak/smartconfig"
+
+ _ "modernc.org/sqlite" // SQLite driver registration
+)
+
+// The config keys the tests name in the errors they expect.
+const (
+ keyDebug = "debug"
+ keyMaintenanceMode = "maintenance_mode"
+ keyPort = "port"
+ keyStateDir = "state_dir"
+ keySentryDSN = "sentry_dsn"
+ keyDBURL = "db_url"
+ keyMetrics = "metrics"
+ keyMetricsUsername = "metrics.username"
+ keyMetricsPassword = "metrics.password"
+ keySigningKey = "signing_key"
+ keyAllowlistHosts = "allowlist_hosts"
+ keyRefererBlocklist = "referer_blocklist"
+ keyAllowHTTP = "allow_http"
+ keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
+ keyUpstreamConnections = "upstream_connections"
+ keyMaxConcurrentProcessing = "max_concurrent_processing"
+ keyCacheMaxBytes = "cache_max_bytes"
+ keyBlockedNetworks = "blocked_networks"
+ keyTrustedProxies = "trusted_proxies"
+ keyAccessControlAllowOrigin = "access_control_allow_origin"
+ keyUpstreamFetchTimeout = "upstream_fetch_timeout"
+ keyUpstreamMaxResponseSize = "upstream_max_response_size"
+ keyDownstreamTimeout = "downstream_timeout"
+ keyImageRequestsPerMinute = "image_requests_per_minute"
)
// validTestSigningKey is a 32-character signing key that satisfies the
-// minimum length requirement in validate().
+// minimum length requirement in validateSigningKey.
const validTestSigningKey = "0123456789abcdef0123456789abcdef"
// signingKeyLine is a valid signing_key config line used as the base of
@@ -93,12 +126,141 @@ func TestOmittedValuesUseDefaults(t *testing.T) {
t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts)
}
- wantDBURL := "file:" + DefaultStateDir + "/state.sqlite3?_journal_mode=WAL"
+ wantDBURL := "file:" + DefaultStateDir +
+ "/state.sqlite3?_pragma=journal_mode(WAL)"
if c.DBURL != wantDBURL {
t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL)
}
}
+// TestDefaultDBURLOpensTheDatabaseInWALMode opens the db_url derived from
+// state_dir with the SQLite driver pixad uses and checks that the database
+// is in WAL mode: the driver ignores any parameter it does not know.
+func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+"state_dir: "+t.TempDir()+"\n")
+ if err != nil {
+ t.Fatalf("config with only state_dir set should be valid, got: %v", err)
+ }
+
+ db, err := sql.Open("sqlite", c.DBURL)
+ if err != nil {
+ t.Fatalf("failed to open %q: %v", c.DBURL, err)
+ }
+
+ t.Cleanup(func() { _ = db.Close() })
+
+ var journalMode string
+
+ err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
+ if err != nil {
+ t.Fatalf("failed to read the journal mode of %q: %v", c.DBURL, err)
+ }
+
+ if journalMode != "wal" {
+ t.Errorf("journal mode of %q = %q, want wal", c.DBURL, journalMode)
+ }
+}
+
+// TestDerivedDBURLOpensStateDirHoldingURLCharacters opens the db_url
+// derived from a state_dir whose name holds %41, ? and #, and checks that
+// the database is state.sqlite3 in that directory, in WAL mode. Put into
+// a file: URL as it is, the name would lose its meaning: %41 is read as
+// A, and the path ends at the ?, which also loses the journal mode.
+func TestDerivedDBURLOpensStateDirHoldingURLCharacters(t *testing.T) {
+ t.Parallel()
+
+ stateDir := filepath.Join(t.TempDir(), "a%41b?c#d")
+
+ err := os.Mkdir(stateDir, 0o750)
+ if err != nil {
+ t.Fatalf("failed to create %q: %v", stateDir, err)
+ }
+
+ // Quoted so YAML takes the whole name as it is.
+ c, err := configFromYAML(t, signingKeyLine+"state_dir: '"+stateDir+"'\n")
+ if err != nil {
+ t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
+ }
+
+ t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
+
+ checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(stateDir, "state.sqlite3"))
+}
+
+// TestDerivedDBURLOpensStateDirStartingWithTwoSlashes opens the db_url
+// derived from a state_dir that starts with //, and checks that the
+// database is state.sqlite3 in that directory, in WAL mode. Written right
+// after file:, such a name gives a URL starting file://, and SQLite reads
+// the first directory name as a host and refuses to open it.
+func TestDerivedDBURLOpensStateDirStartingWithTwoSlashes(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ stateDir := "/" + dir
+
+ c, err := configFromYAML(t, signingKeyLine+"state_dir: "+stateDir+"\n")
+ if err != nil {
+ t.Fatalf("config with state_dir %q should be valid, got: %v", stateDir, err)
+ }
+
+ t.Logf("state_dir %q gives db_url %q", stateDir, c.DBURL)
+
+ checkDBURLOpensFileInWALMode(t, c.DBURL, filepath.Join(dir, "state.sqlite3"))
+}
+
+// TestDerivedDBURLForRootStateDir checks the db_url derived from a
+// state_dir of /. The path written into the URL is then
+// //state.sqlite3, and written right after file: it gives a URL starting
+// file://, where SQLite reads state.sqlite3 as a host and refuses to open
+// it. The test only checks the URL, since it cannot write to /.
+func TestDerivedDBURLForRootStateDir(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+"state_dir: /\n")
+ if err != nil {
+ t.Fatalf("config with state_dir / should be valid, got: %v", err)
+ }
+
+ t.Logf("state_dir / gives db_url %q", c.DBURL)
+
+ wantDBURL := "file:////state.sqlite3?_pragma=journal_mode(WAL)"
+ if c.DBURL != wantDBURL {
+ t.Errorf("DBURL = %q, want %q", c.DBURL, wantDBURL)
+ }
+}
+
+// checkDBURLOpensFileInWALMode opens dbURL with the SQLite driver pixad
+// uses, and checks that the database is in WAL mode and is the file
+// dbFile.
+func checkDBURLOpensFileInWALMode(t *testing.T, dbURL, dbFile string) {
+ t.Helper()
+
+ db, err := sql.Open("sqlite", dbURL)
+ if err != nil {
+ t.Fatalf("failed to open %q: %v", dbURL, err)
+ }
+
+ t.Cleanup(func() { _ = db.Close() })
+
+ var journalMode string
+
+ err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
+ if err != nil {
+ t.Fatalf("failed to read the journal mode of %q: %v", dbURL, err)
+ }
+
+ if journalMode != "wal" {
+ t.Errorf("journal mode of %q = %q, want wal", dbURL, journalMode)
+ }
+
+ _, err = os.Stat(dbFile)
+ if err != nil {
+ t.Errorf("opening %q did not create %q: %v", dbURL, dbFile, err)
+ }
+}
+
func TestExplicitValidValuesAreUsed(t *testing.T) {
t.Parallel()
@@ -181,6 +343,25 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) {
}
}
+// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with
+// an underscore, which pixa can fetch from, is accepted as an entry.
+func TestAllowlistHostsAcceptsUnderscore(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts:
+ - my_bucket.example.com
+ - .my_bucket.example.org
+`)
+ if err != nil {
+ t.Fatalf("host names with an underscore should load, got error: %v", err)
+ }
+
+ want := []string{"my_bucket.example.com", ".my_bucket.example.org"}
+ if !slices.Equal(c.AllowlistHosts, want) {
+ t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want)
+ }
+}
+
// runAbortCases asserts that each case's config aborts startup with an
// error message mentioning every expected substring.
func runAbortCases(t *testing.T, cases []abortCase) {
@@ -283,6 +464,20 @@ func invalidHostAndCredentialCases() []abortCase {
keyAllowlistHosts, "example.com/images",
},
},
+ {
+ name: "allowlist host with wildcard",
+ yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n",
+ wantErrSubstrings: []string{
+ keyAllowlistHosts, "*.example.com",
+ },
+ },
+ {
+ name: "allowlist host with port",
+ yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n",
+ wantErrSubstrings: []string{
+ keyAllowlistHosts, "example.com:8443",
+ },
+ },
{
name: "allowlist host with whitespace",
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
@@ -563,6 +758,116 @@ func TestMalformedConfigFileAbortsStartup(t *testing.T) {
t.Logf("got expected error: %v", err)
}
+// TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup checks that a
+// config file pixa cannot read because it may not enter its directory
+// aborts startup instead of being passed over.
+func TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root may enter any directory")
+ }
+
+ home := t.TempDir()
+ configDir := filepath.Join(home, ".config", "pixa-test-nonexistent-app")
+ configPath := filepath.Join(configDir, "config.yml")
+
+ err := os.MkdirAll(configDir, 0o700)
+ if err != nil {
+ t.Fatalf("failed to create config directory: %v", err)
+ }
+
+ err = os.WriteFile(configPath, []byte(signingKeyLine), 0o600)
+ if err != nil {
+ t.Fatalf("failed to write config: %v", err)
+ }
+
+ err = os.Chmod(configDir, 0)
+ if err != nil {
+ t.Fatalf("failed to remove the config directory's permissions: %v", err)
+ }
+
+ // Give the directory back its permissions so t.TempDir can remove it.
+ t.Cleanup(func() {
+ //nolint:gosec // G302: a directory needs its execute bit to be removed
+ _ = os.Chmod(configDir, 0o700)
+ })
+
+ // The ~/.config candidate is the only one that exists: the appname
+ // rules out /etc, and the working directory is empty.
+ t.Setenv("PIXA_CONFIG_PATH", "")
+ t.Setenv("HOME", home)
+ t.Chdir(t.TempDir())
+
+ log := slog.New(slog.DiscardHandler)
+
+ sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
+ if err == nil {
+ t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
+ sc)
+ }
+
+ t.Logf("got expected error: %v", err)
+
+ if !strings.Contains(err.Error(), configPath) {
+ t.Errorf("error %q does not name the config file %s", err.Error(), configPath)
+ }
+}
+
+// TestConfigFileLinkingToItselfAbortsStartup checks that a config file
+// pixa cannot read for a reason other than not existing aborts startup,
+// as root too: a symbolic link to itself fails with "too many levels of
+// symbolic links".
+func TestConfigFileLinkingToItselfAbortsStartup(t *testing.T) {
+ workDir := t.TempDir()
+
+ err := os.Symlink("config.yml", filepath.Join(workDir, "config.yml"))
+ if err != nil {
+ t.Fatalf("failed to create symbolic link: %v", err)
+ }
+
+ // Only the working directory's config.yml is there: the appname rules
+ // out /etc, and HOME is empty.
+ t.Setenv("PIXA_CONFIG_PATH", "")
+ t.Setenv("HOME", t.TempDir())
+ t.Chdir(workDir)
+
+ log := slog.New(slog.DiscardHandler)
+
+ sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
+ if err == nil {
+ t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
+ sc)
+ }
+
+ t.Logf("got expected error: %v", err)
+
+ if !strings.Contains(err.Error(), "config.yml") {
+ t.Errorf("error %q does not name the config file config.yml", err.Error())
+ }
+}
+
+// TestConfigPathThroughFileIsPassedOver checks that a config file path
+// that runs through a file, such as one under a HOME of /dev/null, is
+// passed over like one that does not exist, since no file can be there.
+func TestConfigPathThroughFileIsPassedOver(t *testing.T) {
+ // No config file is there: the appname rules out /etc, HOME is
+ // /dev/null, and the working directory is empty.
+ t.Setenv("PIXA_CONFIG_PATH", "")
+ t.Setenv("HOME", os.DevNull)
+ t.Chdir(t.TempDir())
+
+ log := slog.New(slog.DiscardHandler)
+
+ sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
+ if err != nil {
+ t.Fatalf("a config path through a file must be passed over, got error: %v",
+ err)
+ }
+
+ if sc != nil {
+ t.Errorf("expected no config file, got config: %v", sc)
+ }
+}
+
func TestEnsureStateDirCreatesDirectory(t *testing.T) {
t.Parallel()
@@ -599,3 +904,244 @@ func TestEnsureStateDirFailsOnUncreatablePath(t *testing.T) {
t.Errorf("error %q does not name the offending key state_dir", err.Error())
}
}
+
+// TestOmittedOriginTimeoutsAndSizeUseDefaults checks that the CORS
+// origin, the upstream fetch timeout, the upstream response size limit
+// and the downstream timeout default to the values pixa used before they
+// could be configured.
+func TestOmittedOriginTimeoutsAndSizeUseDefaults(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine)
+ if err != nil {
+ t.Fatalf("minimal config should be valid, got error: %v", err)
+ }
+
+ if c.AccessControlAllowOrigin != "*" {
+ t.Errorf("AccessControlAllowOrigin = %q, want *", c.AccessControlAllowOrigin)
+ }
+
+ if c.UpstreamFetchTimeout != 30*time.Second {
+ t.Errorf("UpstreamFetchTimeout = %v, want 30s", c.UpstreamFetchTimeout)
+ }
+
+ if c.UpstreamMaxResponseSize != 50<<20 {
+ t.Errorf("UpstreamMaxResponseSize = %d, want %d (50 MiB)",
+ c.UpstreamMaxResponseSize, 50<<20)
+ }
+
+ if c.DownstreamTimeout != 60*time.Second {
+ t.Errorf("DownstreamTimeout = %v, want 60s", c.DownstreamTimeout)
+ }
+}
+
+// TestExplicitOriginTimeoutsAndSizeAreUsed checks that valid values for
+// the CORS origin, the two timeouts and the response size limit are used
+// as given. The size is the largest accepted, 1 GiB.
+func TestExplicitOriginTimeoutsAndSizeAreUsed(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+`
+access_control_allow_origin: https://app.example.com
+upstream_fetch_timeout: 10s
+upstream_max_response_size: 1073741824
+downstream_timeout: 2m
+`)
+ if err != nil {
+ t.Fatalf("valid config should load, got error: %v", err)
+ }
+
+ if c.AccessControlAllowOrigin != "https://app.example.com" {
+ t.Errorf("AccessControlAllowOrigin = %q, want https://app.example.com",
+ c.AccessControlAllowOrigin)
+ }
+
+ if c.UpstreamFetchTimeout != 10*time.Second {
+ t.Errorf("UpstreamFetchTimeout = %v, want 10s", c.UpstreamFetchTimeout)
+ }
+
+ if c.UpstreamMaxResponseSize != 1073741824 {
+ t.Errorf("UpstreamMaxResponseSize = %d, want 1073741824",
+ c.UpstreamMaxResponseSize)
+ }
+
+ if c.DownstreamTimeout != 2*time.Minute {
+ t.Errorf("DownstreamTimeout = %v, want 2m", c.DownstreamTimeout)
+ }
+}
+
+// TestOriginWithPortOrAnyOriginIsAccepted checks the other accepted forms
+// of access_control_allow_origin: "*", an origin with a port, and origins
+// whose host is an IPv4 or IPv6 address.
+func TestOriginWithPortOrAnyOriginIsAccepted(t *testing.T) {
+ t.Parallel()
+
+ for _, origin := range []string{
+ "*", "http://localhost:3000", "http://192.0.2.1", "http://[2001:db8::1]:8080",
+ } {
+ c, err := configFromYAML(t, signingKeyLine+
+ "access_control_allow_origin: \""+origin+"\"\n")
+ if err != nil {
+ t.Fatalf("origin %q should be accepted, got error: %v", origin, err)
+ }
+
+ if c.AccessControlAllowOrigin != origin {
+ t.Errorf("AccessControlAllowOrigin = %q, want %q",
+ c.AccessControlAllowOrigin, origin)
+ }
+ }
+}
+
+// invalidTimeoutCases are configs where upstream_fetch_timeout or
+// downstream_timeout is not a positive Go duration string; each must
+// abort startup naming the key and the value.
+func invalidTimeoutCases() []abortCase {
+ return []abortCase{
+ {
+ name: "upstream_fetch_timeout not a duration",
+ yaml: signingKeyLine + "upstream_fetch_timeout: soon\n",
+ wantErrSubstrings: []string{keyUpstreamFetchTimeout, "soon"},
+ },
+ {
+ name: "upstream_fetch_timeout number without a unit",
+ yaml: signingKeyLine + "upstream_fetch_timeout: 45\n",
+ wantErrSubstrings: []string{keyUpstreamFetchTimeout, "45"},
+ },
+ {
+ name: "upstream_fetch_timeout zero",
+ yaml: signingKeyLine + "upstream_fetch_timeout: 0s\n",
+ wantErrSubstrings: []string{keyUpstreamFetchTimeout, "0s"},
+ },
+ {
+ name: "upstream_fetch_timeout negative",
+ yaml: signingKeyLine + "upstream_fetch_timeout: -5s\n",
+ wantErrSubstrings: []string{keyUpstreamFetchTimeout, "-5s"},
+ },
+ {
+ name: "upstream_fetch_timeout null",
+ yaml: signingKeyLine + "upstream_fetch_timeout: null\n",
+ wantErrSubstrings: []string{keyUpstreamFetchTimeout, nullValueText},
+ },
+ {
+ name: "downstream_timeout not a duration",
+ yaml: signingKeyLine + "downstream_timeout: 1 minute\n",
+ wantErrSubstrings: []string{keyDownstreamTimeout, "1 minute"},
+ },
+ {
+ name: "downstream_timeout zero",
+ yaml: signingKeyLine + "downstream_timeout: 0s\n",
+ wantErrSubstrings: []string{keyDownstreamTimeout, "0s"},
+ },
+ {
+ name: "downstream_timeout negative",
+ yaml: signingKeyLine + "downstream_timeout: -1m\n",
+ wantErrSubstrings: []string{keyDownstreamTimeout, "-1m"},
+ },
+ {
+ name: "downstream_timeout null",
+ yaml: signingKeyLine + "downstream_timeout:\n",
+ wantErrSubstrings: []string{keyDownstreamTimeout, nullValueText},
+ },
+ }
+}
+
+// invalidSizeAndOriginCases are configs where upstream_max_response_size
+// is not a whole number of bytes from 1 to 1 GiB, or
+// access_control_allow_origin is neither "*" nor an origin; each must
+// abort startup naming the key and the value.
+func invalidSizeAndOriginCases() []abortCase {
+ badOrigins := []string{
+ "", // empty
+ "example.com", // no scheme
+ "https://example.com/images", // a path
+ "https://example.com/", // a trailing slash
+ // The CORS middleware reads a * inside an origin as a pattern
+ // that lets other sites read responses.
+ "https://*",
+ "https://*.example.com",
+ "https://*example.com",
+ "https://a.com,b.com", // two hosts
+ "https://example.com:", // an empty port
+ "https://:8443", // no host
+ "https://example.com:0", // a port below 1
+ "https://example.com:99999", // a port above 65535
+ "https://exämple.com", // a host name that is not ASCII
+ "https://example.com:443", // the default port for https
+ "http://example.com:80", // the default port for http
+ "https://example.com:08080", // a port with a leading zero
+ "https://01.2.3.4", // an IPv4 address with a leading zero
+ "https://10.0.0", // an IPv4 address with three parts
+ "https://192.168.1.256", // an IPv4 address part above 255
+ "https://example.123", // a host name whose last part is a number
+ "https://[0:0:0:0:0:0:0:1]", // an IPv6 address not in its shortest form
+ "file://example.com", // a scheme other than http or https
+ "HTTPS://example.com", // a scheme in upper case
+ "https://Example.com", // a host name in upper case
+ }
+
+ cases := make([]abortCase, 0, len(badOrigins))
+ for _, origin := range badOrigins {
+ cases = append(cases, abortCase{
+ name: "access_control_allow_origin " + origin,
+ yaml: signingKeyLine +
+ "access_control_allow_origin: \"" + origin + "\"\n",
+ wantErrSubstrings: []string{keyAccessControlAllowOrigin, origin},
+ })
+ }
+
+ return append(cases, []abortCase{
+ {
+ name: "access_control_allow_origin null",
+ yaml: signingKeyLine + "access_control_allow_origin: null\n",
+ wantErrSubstrings: []string{keyAccessControlAllowOrigin, nullValueText},
+ },
+ {
+ name: "upstream_max_response_size with a unit",
+ yaml: signingKeyLine + "upstream_max_response_size: 50MB\n",
+ wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "50MB"},
+ },
+ {
+ name: "upstream_max_response_size fractional",
+ yaml: signingKeyLine + "upstream_max_response_size: 1.5\n",
+ wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "1.5"},
+ },
+ {
+ name: "upstream_max_response_size zero",
+ yaml: signingKeyLine + "upstream_max_response_size: 0\n",
+ wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "0"},
+ },
+ {
+ name: "upstream_max_response_size negative",
+ yaml: signingKeyLine + "upstream_max_response_size: -1\n",
+ wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "-1"},
+ },
+ {
+ name: "upstream_max_response_size null",
+ yaml: signingKeyLine + "upstream_max_response_size: null\n",
+ wantErrSubstrings: []string{keyUpstreamMaxResponseSize, nullValueText},
+ },
+ {
+ name: "upstream_max_response_size above 1 GiB",
+ yaml: signingKeyLine + "upstream_max_response_size: 1073741825\n",
+ wantErrSubstrings: []string{keyUpstreamMaxResponseSize, "1073741825"},
+ },
+ {
+ name: "upstream_max_response_size largest 64-bit integer",
+ yaml: signingKeyLine +
+ "upstream_max_response_size: 9223372036854775807\n",
+ wantErrSubstrings: []string{
+ keyUpstreamMaxResponseSize, "9223372036854775807",
+ },
+ },
+ }...)
+}
+
+// TestInvalidOriginTimeoutOrSizeAbortsStartup verifies the
+// no-silent-fallback rule for the CORS origin, the two timeouts and the
+// response size limit: a value that does not parse or is out of range
+// aborts startup naming the key and the value.
+func TestInvalidOriginTimeoutOrSizeAbortsStartup(t *testing.T) {
+ t.Parallel()
+
+ runAbortCases(t, append(invalidTimeoutCases(), invalidSizeAndOriginCases()...))
+}
diff --git a/internal/config/documented_settings_internal_test.go b/internal/config/documented_settings_internal_test.go
new file mode 100644
index 0000000..6670252
--- /dev/null
+++ b/internal/config/documented_settings_internal_test.go
@@ -0,0 +1,127 @@
+package config
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+// TestReadmeNamesEverySetting checks that the list of settings in
+// README.md names exactly the keys of the settings table, one item each.
+// The list is the first one in the Configuration section, and it ends at
+// the first blank line. Each item starts with its key in backticks.
+func TestReadmeNamesEverySetting(t *testing.T) {
+ t.Parallel()
+
+ readme, err := os.ReadFile(filepath.Join("..", "..", "README.md"))
+ if err != nil {
+ t.Fatalf("failed to read README.md: %v", err)
+ }
+
+ _, section, found := strings.Cut(string(readme), "\n### Configuration\n")
+ if !found {
+ t.Fatal("README.md has no Configuration section")
+ }
+
+ // The section ends at the next heading of its level or above.
+ section, _, _ = strings.Cut(section, "\n## ")
+ section, _, _ = strings.Cut(section, "\n### ")
+
+ _, list, found := strings.Cut(section, "\n- ")
+ if !found {
+ t.Fatal("README.md's Configuration section has no list")
+ }
+
+ list, _, _ = strings.Cut("- "+list, "\n\n")
+
+ itemKey := regexp.MustCompile("^- `([^`]+)`")
+ named := map[string]bool{}
+
+ for line := range strings.SplitSeq(list, "\n") {
+ // A line that does not start with "- " goes on with the item above.
+ if !strings.HasPrefix(line, "- ") {
+ continue
+ }
+
+ match := itemKey.FindStringSubmatch(line)
+ if match == nil {
+ t.Errorf("README.md's list of settings has an item "+
+ "that does not start with a key in backticks: %s", line)
+
+ continue
+ }
+
+ if named[match[1]] {
+ t.Errorf("README.md's list of settings names %s twice", match[1])
+ }
+
+ named[match[1]] = true
+ }
+
+ for _, s := range new(Config).settings() {
+ if !named[s.key] {
+ t.Errorf("README.md's list of settings does not name %s", s.key)
+ }
+
+ delete(named, s.key)
+ }
+
+ for key := range named {
+ t.Errorf("README.md's list of settings names %s, "+
+ "which is not a setting", key)
+ }
+}
+
+// TestExampleConfigNamesEverySetting checks that
+// configs/config.example.yml names exactly the keys of the settings
+// table, each one set or commented out with "# ". A key line holds the
+// key, a colon and at most a one-word value, which tells it from a
+// comment line that starts with a word and a colon. An indented key is
+// inside the map of the key above it, as username is inside metrics.
+func TestExampleConfigNamesEverySetting(t *testing.T) {
+ t.Parallel()
+
+ const examplePath = "configs/config.example.yml"
+
+ example, err := os.ReadFile(filepath.Join("..", "..", examplePath))
+ if err != nil {
+ t.Fatalf("failed to read %s: %v", examplePath, err)
+ }
+
+ keyLine := regexp.MustCompile(`(?m)^(?:# )?( *)([a-z_]+):(?: \S+)?$`)
+ named := map[string]bool{}
+
+ var mapKey string
+
+ for _, match := range keyLine.FindAllStringSubmatch(string(example), -1) {
+ indent, key := match[1], match[2]
+ if indent == "" {
+ mapKey = key
+ named[key] = true
+
+ continue
+ }
+
+ // The key holding the map is not itself a setting.
+ delete(named, mapKey)
+
+ named[mapKey+"."+key] = true
+ }
+
+ // The env section sets environment variables; it is not a setting.
+ delete(named, "env")
+
+ for _, s := range new(Config).settings() {
+ if !named[s.key] {
+ t.Errorf("%s does not name %s", examplePath, s.key)
+ }
+
+ delete(named, s.key)
+ }
+
+ for key := range named {
+ t.Errorf("%s names %s, which is not a setting", examplePath, key)
+ }
+}
diff --git a/internal/config/env_internal_test.go b/internal/config/env_internal_test.go
index 57f1f15..435c825 100644
--- a/internal/config/env_internal_test.go
+++ b/internal/config/env_internal_test.go
@@ -8,6 +8,7 @@ import (
"slices"
"strings"
"testing"
+ "time"
"sneak.berlin/go/pixa/internal/globals"
"sneak.berlin/go/pixa/internal/logger"
@@ -64,11 +65,19 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
+ t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example")
t.Setenv("PIXA_ALLOW_HTTP", "true")
t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
+ t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
+ t.Setenv("PIXA_MAX_CONCURRENT_PROCESSING", "3")
+ t.Setenv("PIXA_IMAGE_REQUESTS_PER_MINUTE", "30")
t.Setenv("PIXA_CACHE_MAX_BYTES", "1024")
t.Setenv("PIXA_BLOCKED_NETWORKS", "203.0.113.0/24")
t.Setenv("PIXA_TRUSTED_PROXIES", "192.0.2.0/24")
+ t.Setenv("PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", "https://app.example.com")
+ t.Setenv("PIXA_UPSTREAM_FETCH_TIMEOUT", "10s")
+ t.Setenv("PIXA_UPSTREAM_MAX_RESPONSE_SIZE", "1048576")
+ t.Setenv("PIXA_DOWNSTREAM_TIMEOUT", "2m")
c, err := newFromSmartConfig(nil)
if err != nil {
@@ -86,12 +95,20 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
MetricsPassword: "metricspass",
SigningKey: validTestSigningKey,
AllowlistHosts: []string{testHostS3, ".example.com"},
+ RefererBlocklist: []string{"hotlinker.example", ".leech.example"},
AllowHTTP: true,
UpstreamConnectionsPerHost: 5,
+ UpstreamConnections: 10,
+ MaxConcurrentProcessing: 3,
+ ImageRequestsPerMinute: 30,
CacheMaxBytes: 1024,
- cacheMaxBytesExplicit: true,
+ CacheMaxBytesExplicit: true,
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")},
TrustedProxies: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
+ AccessControlAllowOrigin: "https://app.example.com",
+ UpstreamFetchTimeout: 10 * time.Second,
+ UpstreamMaxResponseSize: 1048576,
+ DownstreamTimeout: 2 * time.Minute,
}
if !reflect.DeepEqual(*c, want) {
@@ -121,7 +138,8 @@ func TestPixaPortAbortsStartupPointingToPort(t *testing.T) {
// TestSettingVariablesAndConfigPathAreAccepted checks that every
// setting's variable and PIXA_CONFIG_PATH pass the check for unknown
-// PIXA_ variables. TestEnvironmentSetsEveryKey pins the names in the list.
+// PIXA_ variables. TestEnvironmentSetsEveryKey pins the names in the
+// settings table.
func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) {
// A config file's env section loaded by another test can leave a
// PIXA_ variable set for the whole process, so every one is unset
@@ -142,8 +160,8 @@ func TestSettingVariablesAndConfigPathAreAccepted(t *testing.T) {
t.Setenv("PIXA_CONFIG_PATH", "/etc/pixa/config.yml")
- for _, name := range envVarNames() {
- t.Setenv(name, "")
+ for _, s := range new(Config).settings() {
+ t.Setenv(s.envVar, "")
}
err := validateKnownEnvVars()
@@ -280,6 +298,30 @@ func TestInvalidDebugFromEnvironmentAbortsStartup(t *testing.T) {
wantStartupError(t, err, "PIXA_DEBUG", "maybe")
}
+// TestInvalidOriginTimeoutOrSizeFromEnvironmentAbortsStartup checks that
+// an invalid CORS origin, timeout or response size limit in its variable
+// aborts startup naming the variable and the value.
+func TestInvalidOriginTimeoutOrSizeFromEnvironmentAbortsStartup(t *testing.T) {
+ cases := []struct {
+ variable string
+ value string
+ }{
+ {"PIXA_ACCESS_CONTROL_ALLOW_ORIGIN", "example.com"},
+ {"PIXA_UPSTREAM_FETCH_TIMEOUT", "soon"},
+ {"PIXA_UPSTREAM_MAX_RESPONSE_SIZE", "50MB"},
+ {"PIXA_DOWNSTREAM_TIMEOUT", "0s"},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.variable, func(t *testing.T) {
+ t.Setenv(tc.variable, tc.value)
+
+ _, err := configFromYAML(t, signingKeyLine)
+ wantStartupError(t, err, tc.variable, tc.value)
+ })
+ }
+}
+
// TestConfigFileAloneBehavesAsBefore checks that with no variables set
// (TestMain unsets them) the config file's values are used and omitted
// keys take their defaults.
diff --git a/internal/config/image_requests_per_minute_internal_test.go b/internal/config/image_requests_per_minute_internal_test.go
new file mode 100644
index 0000000..12f63ab
--- /dev/null
+++ b/internal/config/image_requests_per_minute_internal_test.go
@@ -0,0 +1,107 @@
+package config
+
+import "testing"
+
+// testImageRequestsPerMinuteVar is the variable that sets
+// image_requests_per_minute.
+const testImageRequestsPerMinuteVar = "PIXA_IMAGE_REQUESTS_PER_MINUTE"
+
+// TestImageRequestsPerMinuteValues checks that an omitted
+// image_requests_per_minute is 600, and that 0, which turns the limit off,
+// and any larger whole number are used as given.
+func TestImageRequestsPerMinuteValues(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine)
+ if err != nil {
+ t.Fatalf("minimal config should be valid, got error: %v", err)
+ }
+
+ if c.ImageRequestsPerMinute != 600 {
+ t.Errorf("ImageRequestsPerMinute = %d, want 600", c.ImageRequestsPerMinute)
+ }
+
+ explicit := []struct {
+ value string
+ want int
+ }{
+ {"0", 0},
+ {"30", 30},
+ }
+
+ for _, tc := range explicit {
+ c, err := configFromYAML(t,
+ signingKeyLine+"image_requests_per_minute: "+tc.value+"\n")
+ if err != nil {
+ t.Fatalf("image_requests_per_minute: %s should load, got error: %v",
+ tc.value, err)
+ }
+
+ if c.ImageRequestsPerMinute != tc.want {
+ t.Errorf("ImageRequestsPerMinute = %d, want %d",
+ c.ImageRequestsPerMinute, tc.want)
+ }
+ }
+}
+
+// TestInvalidImageRequestsPerMinuteAbortsStartup checks that a value that is
+// negative, not a whole number, or null aborts startup naming the key and
+// the value, and the variable too where the value could have come from it.
+func TestInvalidImageRequestsPerMinuteAbortsStartup(t *testing.T) {
+ t.Parallel()
+
+ key := keyImageRequestsPerMinute
+
+ runAbortCases(t, []abortCase{
+ {
+ name: "negative",
+ yaml: signingKeyLine + key + ": -1\n",
+ wantErrSubstrings: []string{
+ key, testImageRequestsPerMinuteVar, "value -1",
+ },
+ },
+ {
+ name: "not a number",
+ yaml: signingKeyLine + key + ": plenty\n",
+ wantErrSubstrings: []string{
+ key, testImageRequestsPerMinuteVar, "plenty",
+ },
+ },
+ {
+ name: "fractional",
+ yaml: signingKeyLine + key + ": 2.5\n",
+ wantErrSubstrings: []string{key, "2.5"},
+ },
+ {
+ name: "explicit null",
+ yaml: signingKeyLine + key + ": null\n",
+ wantErrSubstrings: []string{key, nullValueText},
+ },
+ })
+}
+
+// TestImageRequestsPerMinuteFromEnvironment checks that the variable sets the
+// limit over the config file, and that an invalid value in it aborts startup
+// naming the variable and the value.
+func TestImageRequestsPerMinuteFromEnvironment(t *testing.T) {
+ t.Setenv(testImageRequestsPerMinuteVar, "0")
+
+ c, err := configFromYAML(t, signingKeyLine+"image_requests_per_minute: 30\n")
+ if err != nil {
+ t.Fatalf("limit from the environment should load: %v", err)
+ }
+
+ if c.ImageRequestsPerMinute != 0 {
+ t.Errorf("ImageRequestsPerMinute = %d, want 0 from the environment",
+ c.ImageRequestsPerMinute)
+ }
+
+ for _, value := range []string{"-5", "ten"} {
+ t.Run(value, func(t *testing.T) {
+ t.Setenv(testImageRequestsPerMinuteVar, value)
+
+ _, err := configFromYAML(t, signingKeyLine)
+ wantStartupError(t, err, testImageRequestsPerMinuteVar, value)
+ })
+ }
+}
diff --git a/internal/config/referer_blocklist_internal_test.go b/internal/config/referer_blocklist_internal_test.go
new file mode 100644
index 0000000..43d5753
--- /dev/null
+++ b/internal/config/referer_blocklist_internal_test.go
@@ -0,0 +1,166 @@
+package config
+
+import (
+ "slices"
+ "testing"
+)
+
+// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
+// pattern starting with "." and checks both are kept in order.
+func TestRefererBlocklistParsed(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+ - leech.example
+ - .hotlinker.example
+`)
+ if err != nil {
+ t.Fatalf("valid referer_blocklist should load, got error: %v", err)
+ }
+
+ want := []string{"leech.example", ".hotlinker.example"}
+ if !slices.Equal(c.RefererBlocklist, want) {
+ t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+ }
+}
+
+// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
+// the IPv6 one written without brackets, are accepted as entries.
+func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+ - 192.0.2.7
+ - "2001:db8::7"
+`)
+ if err != nil {
+ t.Fatalf("IP address entries should load, got error: %v", err)
+ }
+
+ want := []string{"192.0.2.7", "2001:db8::7"}
+ if !slices.Equal(c.RefererBlocklist, want) {
+ t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+ }
+}
+
+// TestRefererBlocklistAcceptsUnderscore checks that a host name with an
+// underscore, which a page can be served from, is accepted as an entry.
+func TestRefererBlocklistAcceptsUnderscore(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+ - my_site.leech.example
+ - .my_site.hotlinker.example
+`)
+ if err != nil {
+ t.Fatalf("host names with an underscore should load, got error: %v", err)
+ }
+
+ want := []string{"my_site.leech.example", ".my_site.hotlinker.example"}
+ if !slices.Equal(c.RefererBlocklist, want) {
+ t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+ }
+}
+
+// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
+// referer.
+func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine)
+ if err != nil {
+ t.Fatalf("minimal config should be valid, got error: %v", err)
+ }
+
+ if len(c.RefererBlocklist) != 0 {
+ t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
+ }
+}
+
+// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
+// host, or a value that is not a list of them, aborts startup with an error
+// naming the key and the entry.
+func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
+ t.Parallel()
+
+ runAbortCases(t, []abortCase{
+ {
+ name: "entry with a scheme",
+ yaml: signingKeyLine + "referer_blocklist:\n - https://leech.example\n",
+ wantErrSubstrings: []string{
+ keyRefererBlocklist, "https://leech.example",
+ },
+ },
+ {
+ name: "entry with a path",
+ yaml: signingKeyLine + "referer_blocklist:\n - leech.example/page\n",
+ wantErrSubstrings: []string{
+ keyRefererBlocklist, "leech.example/page",
+ },
+ },
+ {
+ name: "wildcard entry",
+ yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n",
+ wantErrSubstrings: []string{
+ keyRefererBlocklist, "*.leech.example",
+ },
+ },
+ {
+ name: "entry with a port",
+ yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n",
+ wantErrSubstrings: []string{
+ keyRefererBlocklist, "leech.example:8080",
+ },
+ },
+ {
+ name: "two leading dots",
+ yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n",
+ wantErrSubstrings: []string{
+ keyRefererBlocklist, "..leech.example",
+ },
+ },
+ {
+ name: "dot only",
+ yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
+ wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
+ },
+ {
+ name: "empty entry",
+ yaml: signingKeyLine + "referer_blocklist:\n - \"\"\n",
+ wantErrSubstrings: []string{keyRefererBlocklist},
+ },
+ {
+ name: "entry not a string",
+ yaml: signingKeyLine + "referer_blocklist:\n - 42\n",
+ wantErrSubstrings: []string{keyRefererBlocklist, "42"},
+ },
+ {
+ name: "null value",
+ yaml: signingKeyLine + "referer_blocklist:\n",
+ wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
+ },
+ })
+}
+
+// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
+// takes comma-separated entries, and that an entry in it that is not a host
+// aborts startup naming the variable and the entry.
+func TestRefererBlocklistFromEnvironment(t *testing.T) {
+ t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
+ t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
+
+ c, err := newFromSmartConfig(nil)
+ if err != nil {
+ t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
+ }
+
+ want := []string{"leech.example", ".hotlinker.example"}
+ if !slices.Equal(c.RefererBlocklist, want) {
+ t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+ }
+
+ t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
+
+ _, err = newFromSmartConfig(nil)
+ wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
+}
diff --git a/internal/config/trusted_proxies_internal_test.go b/internal/config/trusted_proxies_internal_test.go
index ae61bfd..41a184d 100644
--- a/internal/config/trusted_proxies_internal_test.go
+++ b/internal/config/trusted_proxies_internal_test.go
@@ -83,3 +83,41 @@ func TestTrustedProxiesInvalidAbortsStartup(t *testing.T) {
},
})
}
+
+// TestTrustedProxiesMappedEntryIsIPv4Range checks that an entry written in
+// the IPv4-mapped form (::ffff:a.b.c.d) is kept as the IPv4 range it covers,
+// since the client address is matched as the IPv4 address it holds. /96, the
+// shortest such entry, covers every IPv4 address.
+func TestTrustedProxiesMappedEntryIsIPv4Range(t *testing.T) {
+ t.Parallel()
+
+ c, err := configFromYAML(t, signingKeyLine+
+ `trusted_proxies: ["::ffff:10.0.0.0/104", "::ffff:0:0/96"]`+"\n")
+ if err != nil {
+ t.Fatalf("IPv4-mapped trusted_proxies entries should load: %v", err)
+ }
+
+ got := make([]string, len(c.TrustedProxies))
+ for i, p := range c.TrustedProxies {
+ got[i] = p.String()
+ }
+
+ if joined := strings.Join(got, ","); joined != "10.0.0.0/8,0.0.0.0/0" {
+ t.Errorf("TrustedProxies = %v, want 10.0.0.0/8 and 0.0.0.0/0", got)
+ }
+}
+
+// TestTrustedProxiesMappedEntryShorterThan96AbortsStartup checks that an
+// entry in the IPv4-mapped form shorter than /96, which covers more than the
+// IPv4 addresses, aborts startup with an error naming the key and the value.
+func TestTrustedProxiesMappedEntryShorterThan96AbortsStartup(t *testing.T) {
+ t.Parallel()
+
+ runAbortCases(t, []abortCase{
+ {
+ name: "mapped-95",
+ yaml: signingKeyLine + `trusted_proxies: ["::ffff:0:0/95"]` + "\n",
+ wantErrSubstrings: []string{keyTrustedProxies, "::ffff:0:0/95"},
+ },
+ })
+}
diff --git a/internal/database/concurrent_writes_internal_test.go b/internal/database/concurrent_writes_internal_test.go
new file mode 100644
index 0000000..8dcfdbf
--- /dev/null
+++ b/internal/database/concurrent_writes_internal_test.go
@@ -0,0 +1,143 @@
+package database
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log/slog"
+ "path/filepath"
+ "sync"
+ "testing"
+
+ "sneak.berlin/go/pixa/internal/config"
+)
+
+// TestConcurrentWritesAllSucceed opens a database the way pixad does and
+// writes to it from several goroutines at once, as one request's writes and
+// the background eviction pass do. Every write must succeed, none failing
+// with "database is locked", whether or not db_url already has parameters,
+// and the parameters it has must still apply.
+func TestConcurrentWritesAllSucceed(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ query string
+ wantJournalMode string
+ }{
+ {
+ name: "db_url without parameters",
+ query: "",
+ wantJournalMode: "delete",
+ },
+ {
+ name: "db_url with the WAL parameter",
+ query: "?_pragma=journal_mode(WAL)",
+ wantJournalMode: "wal",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ dbURL := "file:" + filepath.Join(t.TempDir(), "state.sqlite3") + tt.query
+
+ d := &Database{
+ log: slog.New(slog.DiscardHandler),
+ config: &config.Config{DBURL: dbURL},
+ }
+
+ err := d.connect(t.Context())
+ if err != nil {
+ t.Fatalf("failed to connect to %q: %v", dbURL, err)
+ }
+
+ t.Cleanup(func() { _ = d.db.Close() })
+
+ writeConcurrently(t, d.db)
+
+ var journalMode string
+
+ err = d.db.QueryRowContext(t.Context(), "PRAGMA journal_mode").
+ Scan(&journalMode)
+ if err != nil {
+ t.Fatalf("failed to read the journal mode: %v", err)
+ }
+
+ if journalMode != tt.wantJournalMode {
+ t.Errorf("journal mode = %q, want %q", journalMode, tt.wantJournalMode)
+ }
+ })
+ }
+}
+
+// writeConcurrently runs writeLikeOneRequest from several goroutines at once
+// and checks that every write was made.
+func writeConcurrently(t *testing.T, db *sql.DB) {
+ t.Helper()
+
+ const (
+ writers = 4
+ requestsEach = 20
+ totalRequests = writers * requestsEach
+ )
+
+ ctx := t.Context()
+
+ var wg sync.WaitGroup
+
+ for writer := range writers {
+ wg.Go(func() {
+ for request := range requestsEach {
+ key := fmt.Sprintf("%d-%d", writer, request)
+
+ err := writeLikeOneRequest(ctx, db, key)
+ if err != nil {
+ t.Errorf("writer %d: %v", writer, err)
+
+ return
+ }
+ }
+ })
+ }
+
+ wg.Wait()
+
+ var sources int
+
+ err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM source_content`).
+ Scan(&sources)
+ if err != nil {
+ t.Fatalf("failed to count the writes: %v", err)
+ }
+
+ if sources != totalRequests {
+ t.Errorf("%d source_content rows, want %d", sources, totalRequests)
+ }
+}
+
+// writeLikeOneRequest makes the writes one request and the eviction pass
+// make: it stores a source, records a transformed image and deletes that
+// record again.
+func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error {
+ _, err := db.ExecContext(ctx, `INSERT INTO source_content
+ (source_key, size_bytes) VALUES (?, 1)`, key)
+ if err != nil {
+ return fmt.Errorf("storing a source: %w", err)
+ }
+
+ _, err = db.ExecContext(ctx, `INSERT INTO variant_content
+ (cache_key, size_bytes) VALUES (?, 1)`, key)
+ if err != nil {
+ return fmt.Errorf("recording a transformed image: %w", err)
+ }
+
+ _, err = db.ExecContext(ctx,
+ `DELETE FROM variant_content WHERE cache_key = ?`, key)
+ if err != nil {
+ return fmt.Errorf("evicting a transformed image: %w", err)
+ }
+
+ return nil
+}
diff --git a/internal/database/database.go b/internal/database/database.go
index f29560c..31b74e2 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -4,9 +4,9 @@ package database
import (
"context"
"database/sql"
- "embed"
"errors"
"fmt"
+ "io/fs"
"log/slog"
"path/filepath"
"sort"
@@ -15,14 +15,12 @@ import (
"go.uber.org/fx"
"sneak.berlin/go/pixa/internal/config"
+ "sneak.berlin/go/pixa/internal/db/migrations"
"sneak.berlin/go/pixa/internal/logger"
_ "modernc.org/sqlite" // SQLite driver registration
)
-//go:embed schema/*.sql
-var schemaFS embed.FS
-
// bootstrapVersion is the migration that creates the schema_migrations
// table itself. It is applied before the normal migration loop.
const bootstrapVersion = 0
@@ -84,7 +82,11 @@ func ParseMigrationVersion(filename string) (int, error) {
return version, nil
}
-// New creates a new Database instance.
+// New opens the database and applies the migrations, so that the parts
+// fx builds after it can read the database while they are built. Its
+// stop hook closes the database. fx runs stop hooks in the reverse of the
+// order it built their parts in, so the stop hooks of those parts run
+// before this one.
func New(lc fx.Lifecycle, params Params) (*Database, error) {
s := &Database{
log: params.Logger.Get(),
@@ -93,49 +95,45 @@ func New(lc fx.Lifecycle, params Params) (*Database, error) {
s.log.Info("Database instantiated")
- lc.Append(fx.Hook{
- OnStart: func(ctx context.Context) error {
- s.log.Info("Database OnStart Hook")
+ err := s.connect(context.Background())
+ if err != nil {
+ return nil, err
+ }
- return s.connect(ctx)
- },
+ lc.Append(fx.Hook{
OnStop: func(_ context.Context) error {
s.log.Info("Database OnStop Hook")
- if s.db != nil {
- return s.db.Close()
- }
-
- return nil
+ return s.db.Close()
},
})
return s, nil
}
-// collectMigrations reads the embedded schema directory and returns
+// collectMigrations reads the embedded migrations directory and returns
// migration filenames sorted lexicographically.
func collectMigrations() ([]string, error) {
- entries, err := schemaFS.ReadDir("schema")
+ entries, err := fs.ReadDir(migrations.FS(), ".")
if err != nil {
- return nil, fmt.Errorf("failed to read schema directory: %w", err)
+ return nil, fmt.Errorf("failed to read migrations directory: %w", err)
}
- var migrations []string
+ var filenames []string
for _, entry := range entries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") {
- migrations = append(migrations, entry.Name())
+ filenames = append(filenames, entry.Name())
}
}
- sort.Strings(migrations)
+ sort.Strings(filenames)
- return migrations, nil
+ return filenames, nil
}
// bootstrapMigrationsTable ensures the schema_migrations table exists
-// by applying 000.sql if the table is missing.
+// by applying 000_migration.sql if the table is missing.
func bootstrapMigrationsTable(ctx context.Context, db *sql.DB, log *slog.Logger) error {
var tableExists int
@@ -150,9 +148,9 @@ func bootstrapMigrationsTable(ctx context.Context, db *sql.DB, log *slog.Logger)
return nil
}
- content, err := schemaFS.ReadFile("schema/000.sql")
+ content, err := fs.ReadFile(migrations.FS(), "000_migration.sql")
if err != nil {
- return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err)
+ return fmt.Errorf("failed to read bootstrap migration 000_migration.sql: %w", err)
}
if log != nil {
@@ -177,12 +175,12 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error {
return err
}
- migrations, err := collectMigrations()
+ filenames, err := collectMigrations()
if err != nil {
return err
}
- for _, migration := range migrations {
+ for _, migration := range filenames {
version, parseErr := ParseMigrationVersion(migration)
if parseErr != nil {
return parseErr
@@ -208,7 +206,7 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error {
}
// Read and apply migration.
- content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration))
+ content, readErr := fs.ReadFile(migrations.FS(), migration)
if readErr != nil {
return fmt.Errorf("failed to read migration %s: %w", migration, readErr)
}
@@ -239,13 +237,24 @@ func ApplyMigrations(ctx context.Context, db *sql.DB, log *slog.Logger) error {
return nil
}
-// DB returns the underlying sql.DB.
+// DB returns the underlying sql.DB. It has one connection, so close any
+// rows and end any transaction before running another query on it; a
+// query run while they are open waits forever.
func (s *Database) DB() *sql.DB {
return s.db
}
func (s *Database) connect(ctx context.Context) error {
- dbURL := s.config.DBURL
+ // With a busy timeout, a write that finds another program writing to
+ // the same database file waits up to five seconds for it instead of
+ // failing at once with "database is locked". The driver runs each
+ // _pragma parameter on every connection it opens.
+ separator := "?"
+ if strings.Contains(s.config.DBURL, "?") {
+ separator = "&"
+ }
+
+ dbURL := s.config.DBURL + separator + "_pragma=busy_timeout(5000)"
s.log.Info("connecting to database", "url", dbURL)
@@ -256,6 +265,11 @@ func (s *Database) connect(ctx context.Context) error {
return err
}
+ // One connection: pixa's own reads and writes run on it one at a
+ // time instead of competing for SQLite's lock, where a write that
+ // keeps losing can wait past the busy timeout and be lost.
+ db.SetMaxOpenConns(1)
+
err = db.PingContext(ctx)
if err != nil {
s.log.Error("failed to ping database", "error", err)
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index 4b70909..1339ac7 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -159,10 +159,7 @@ func TestApplyMigrations_CreatesSchemaAndTables(t *testing.T) {
}
// Verify that the application tables created by 001.sql exist.
- tables := []string{
- "source_content", "source_metadata", "output_content",
- "request_cache", "negative_cache", "cache_stats",
- }
+ tables := []string{"source_content", "negative_cache"}
for _, table := range tables {
var count int
diff --git a/internal/database/migrations_internal_test.go b/internal/database/migrations_internal_test.go
new file mode 100644
index 0000000..6fb7282
--- /dev/null
+++ b/internal/database/migrations_internal_test.go
@@ -0,0 +1,54 @@
+package database
+
+import (
+ "bytes"
+ "database/sql"
+ "log/slog"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ _ "modernc.org/sqlite" // SQLite driver registration
+)
+
+// TestApplyMigrations_SecondRunAppliesNothing applies the migrations twice
+// to one database file, as happens when pixad starts again on the database
+// it created, and checks that the second run applies none of them.
+// ApplyMigrations logs a message starting with "applying" before it runs
+// any migration, the bootstrap one included.
+func TestApplyMigrations_SecondRunAppliesNothing(t *testing.T) {
+ t.Parallel()
+
+ ctx := t.Context()
+
+ db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "state.sqlite3"))
+ if err != nil {
+ t.Fatalf("failed to open test db: %v", err)
+ }
+
+ t.Cleanup(func() { _ = db.Close() })
+
+ var firstLog bytes.Buffer
+
+ err = ApplyMigrations(ctx, db, slog.New(slog.NewTextHandler(&firstLog, nil)))
+ if err != nil {
+ t.Fatalf("first ApplyMigrations failed: %v", err)
+ }
+
+ if !strings.Contains(firstLog.String(), "applying") {
+ t.Fatalf("first ApplyMigrations logged no applied migration:\n%s",
+ firstLog.String())
+ }
+
+ var secondLog bytes.Buffer
+
+ err = ApplyMigrations(ctx, db, slog.New(slog.NewTextHandler(&secondLog, nil)))
+ if err != nil {
+ t.Fatalf("second ApplyMigrations failed: %v", err)
+ }
+
+ if strings.Contains(secondLog.String(), "applying") {
+ t.Errorf("second ApplyMigrations ran a migration again:\n%s",
+ secondLog.String())
+ }
+}
diff --git a/internal/database/schema/001_initial_schema.sql b/internal/database/schema/001_initial_schema.sql
deleted file mode 100644
index c4ca863..0000000
--- a/internal/database/schema/001_initial_schema.sql
+++ /dev/null
@@ -1,112 +0,0 @@
--- Migration 001: Initial schema
--- Creates all tables for the pixa caching image proxy
-
--- Source content blobs
--- Files stored at: cache/src-content///
--- last_accessed_at is NULL until the first LRU touch; eviction falls
--- back to fetched_at for rows that have never been touched.
-CREATE TABLE IF NOT EXISTS source_content (
- content_hash TEXT PRIMARY KEY,
- content_type TEXT NOT NULL,
- size_bytes INTEGER NOT NULL,
- fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
- last_accessed_at DATETIME
-);
-CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
- ON source_content(last_accessed_at);
-
--- Source URL metadata - maps URLs to content hashes
--- JSON stored at: cache/src-metadata//.json
-CREATE TABLE IF NOT EXISTS source_metadata (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- source_host TEXT NOT NULL,
- source_path TEXT NOT NULL,
- source_query TEXT NOT NULL DEFAULT '',
- path_hash TEXT NOT NULL,
- content_hash TEXT,
- status_code INTEGER NOT NULL,
- content_type TEXT,
- response_headers TEXT,
- fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
- expires_at DATETIME,
- etag TEXT,
- last_modified TEXT,
- UNIQUE(source_host, source_path, source_query),
- FOREIGN KEY (content_hash) REFERENCES source_content(content_hash)
-);
-CREATE INDEX IF NOT EXISTS idx_source_meta_host ON source_metadata(source_host);
-CREATE INDEX IF NOT EXISTS idx_source_meta_path_hash ON source_metadata(path_hash);
-CREATE INDEX IF NOT EXISTS idx_source_meta_expires ON source_metadata(expires_at);
-CREATE INDEX IF NOT EXISTS idx_source_meta_content_hash ON source_metadata(content_hash);
-
--- Processed variant blobs
--- Files stored at: cache/variants/// (plus a .meta
--- sidecar with the content type). Tracked here (like source content
--- blobs above) so total cache usage can be computed with a SUM query,
--- never a directory scan, and so LRU eviction has a timestamp to order
--- on.
-CREATE TABLE IF NOT EXISTS variant_content (
- cache_key TEXT PRIMARY KEY,
- size_bytes INTEGER NOT NULL,
- content_type TEXT NOT NULL,
- created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
- last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
-);
-CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
- ON variant_content(last_accessed_at);
-
--- Output/transformed content blobs
--- Files stored at: cache/dst-content///
-CREATE TABLE IF NOT EXISTS output_content (
- content_hash TEXT PRIMARY KEY,
- content_type TEXT NOT NULL,
- size_bytes INTEGER NOT NULL,
- fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP
-);
-
--- Request cache - maps full request params to output content
-CREATE TABLE IF NOT EXISTS request_cache (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- cache_key TEXT NOT NULL UNIQUE,
- source_metadata_id INTEGER NOT NULL,
- output_hash TEXT NOT NULL,
- width INTEGER NOT NULL,
- height INTEGER NOT NULL,
- format TEXT NOT NULL,
- quality INTEGER NOT NULL DEFAULT 85,
- fit_mode TEXT NOT NULL DEFAULT 'cover',
- fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
- access_count INTEGER NOT NULL DEFAULT 1,
- FOREIGN KEY (source_metadata_id) REFERENCES source_metadata(id),
- FOREIGN KEY (output_hash) REFERENCES output_content(content_hash)
-);
-CREATE INDEX IF NOT EXISTS idx_request_cache_key ON request_cache(cache_key);
-CREATE INDEX IF NOT EXISTS idx_request_cache_source ON request_cache(source_metadata_id);
-CREATE INDEX IF NOT EXISTS idx_request_cache_output ON request_cache(output_hash);
-CREATE INDEX IF NOT EXISTS idx_request_cache_fetched ON request_cache(fetched_at);
-
--- Negative cache for failed fetches (404s, timeouts, etc.)
-CREATE TABLE IF NOT EXISTS negative_cache (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- source_host TEXT NOT NULL,
- source_path TEXT NOT NULL,
- source_query TEXT NOT NULL DEFAULT '',
- status_code INTEGER NOT NULL,
- error_message TEXT,
- fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
- expires_at DATETIME NOT NULL,
- UNIQUE(source_host, source_path, source_query)
-);
-CREATE INDEX IF NOT EXISTS idx_negative_cache_expires ON negative_cache(expires_at);
-
--- Cache statistics for monitoring
-CREATE TABLE IF NOT EXISTS cache_stats (
- id INTEGER PRIMARY KEY CHECK (id = 1),
- hit_count INTEGER NOT NULL DEFAULT 0,
- miss_count INTEGER NOT NULL DEFAULT 0,
- upstream_fetch_count INTEGER NOT NULL DEFAULT 0,
- upstream_fetch_bytes INTEGER NOT NULL DEFAULT 0,
- transform_count INTEGER NOT NULL DEFAULT 0,
- last_updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
-);
-INSERT OR IGNORE INTO cache_stats (id) VALUES (1);
diff --git a/internal/database/schema/000.sql b/internal/db/migrations/000_migration.sql
similarity index 100%
rename from internal/database/schema/000.sql
rename to internal/db/migrations/000_migration.sql
diff --git a/internal/db/migrations/001_schema.sql b/internal/db/migrations/001_schema.sql
new file mode 100644
index 0000000..7819d9f
--- /dev/null
+++ b/internal/db/migrations/001_schema.sql
@@ -0,0 +1,137 @@
+-- Migration 001: Initial schema
+-- Creates all tables for the pixa caching image proxy
+
+-- Source images, one row per source URL
+-- Files stored at: cache/sources///, where source_key
+-- is the SHA-256 of the source's host, path and query.
+-- fetched_at is when the cached bytes were fetched. checked_at is when
+-- upstream last sent them or answered that they had not changed.
+-- upstream_etag and upstream_last_modified are the ETag and Last-Modified
+-- headers upstream sent with the bytes, or '' when it sent none.
+CREATE TABLE IF NOT EXISTS source_content (
+ source_key TEXT PRIMARY KEY,
+ size_bytes INTEGER NOT NULL,
+ last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ upstream_etag TEXT NOT NULL DEFAULT '',
+ upstream_last_modified TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS idx_source_content_last_accessed
+ ON source_content(last_accessed_at);
+CREATE INDEX IF NOT EXISTS idx_source_content_checked
+ ON source_content(checked_at);
+
+-- Processed variant blobs
+-- Files stored at: cache/variants///. Tracked here
+-- (like source content blobs above) so total cache usage is known
+-- without a directory scan, and so LRU eviction has a timestamp to order
+-- on. A cache hit does not read this table: it takes the content type
+-- from the file's first bytes, and its last_accessed_at is written
+-- later, in a batch. checked_at is the checked_at of the source the
+-- variant was made from, when it was made or last renewed; the variant
+-- file's modification time holds the same time for cache hits.
+-- upstream_etag and upstream_last_modified are those of the source bytes
+-- the variant was made from, so the variant can still be checked with
+-- upstream after its source is evicted.
+CREATE TABLE IF NOT EXISTS variant_content (
+ cache_key TEXT PRIMARY KEY,
+ size_bytes INTEGER NOT NULL,
+ last_accessed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ checked_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ upstream_etag TEXT NOT NULL DEFAULT '',
+ upstream_last_modified TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS idx_variant_content_last_accessed
+ ON variant_content(last_accessed_at);
+CREATE INDEX IF NOT EXISTS idx_variant_content_checked
+ ON variant_content(checked_at);
+
+-- Total cache usage: the sum of size_bytes over source_content and
+-- variant_content, kept by the triggers below in the same statement
+-- that adds, removes or resizes a row, so eviction reads this one row
+-- instead of summing both tables. Each trigger also adds one to
+-- change_count; the reconciliation pass, which sums both tables a page
+-- at a time, corrects total_size_bytes only if change_count did not
+-- move while it summed.
+CREATE TABLE IF NOT EXISTS cache_usage (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ total_size_bytes INTEGER NOT NULL DEFAULT 0,
+ change_count INTEGER NOT NULL DEFAULT 0
+);
+INSERT OR IGNORE INTO cache_usage (id) VALUES (1);
+
+CREATE TRIGGER IF NOT EXISTS source_content_usage_insert
+AFTER INSERT ON source_content
+BEGIN
+ UPDATE cache_usage
+ SET total_size_bytes = total_size_bytes + NEW.size_bytes,
+ change_count = change_count + 1
+ WHERE id = 1;
+END;
+
+CREATE TRIGGER IF NOT EXISTS source_content_usage_delete
+AFTER DELETE ON source_content
+BEGIN
+ UPDATE cache_usage
+ SET total_size_bytes = total_size_bytes - OLD.size_bytes,
+ change_count = change_count + 1
+ WHERE id = 1;
+END;
+
+CREATE TRIGGER IF NOT EXISTS source_content_usage_update
+AFTER UPDATE OF size_bytes ON source_content
+BEGIN
+ UPDATE cache_usage
+ SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
+ change_count = change_count + 1
+ WHERE id = 1;
+END;
+
+CREATE TRIGGER IF NOT EXISTS variant_content_usage_insert
+AFTER INSERT ON variant_content
+BEGIN
+ UPDATE cache_usage
+ SET total_size_bytes = total_size_bytes + NEW.size_bytes,
+ change_count = change_count + 1
+ WHERE id = 1;
+END;
+
+CREATE TRIGGER IF NOT EXISTS variant_content_usage_delete
+AFTER DELETE ON variant_content
+BEGIN
+ UPDATE cache_usage
+ SET total_size_bytes = total_size_bytes - OLD.size_bytes,
+ change_count = change_count + 1
+ WHERE id = 1;
+END;
+
+CREATE TRIGGER IF NOT EXISTS variant_content_usage_update
+AFTER UPDATE OF size_bytes ON variant_content
+BEGIN
+ UPDATE cache_usage
+ SET total_size_bytes = total_size_bytes - OLD.size_bytes + NEW.size_bytes,
+ change_count = change_count + 1
+ WHERE id = 1;
+END;
+
+-- Negative cache: source URLs whose fetch failed with an upstream error
+-- status, or whose first bytes did not match their Content-Type. pixa does
+-- not fetch such a URL again until its row expires.
+CREATE TABLE IF NOT EXISTS negative_cache (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ source_host TEXT NOT NULL,
+ source_path TEXT NOT NULL,
+ source_query TEXT NOT NULL DEFAULT '',
+ expires_at DATETIME NOT NULL,
+ UNIQUE(source_host, source_path, source_query)
+);
+CREATE INDEX IF NOT EXISTS idx_negative_cache_expires ON negative_cache(expires_at);
+
+-- A random value that a login session carries from its login. pixa makes it
+-- when the table has no row, and each logout replaces it; a login session
+-- carrying another value is refused, so logging out ends every login session.
+CREATE TABLE IF NOT EXISTS logout (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ logout_value TEXT NOT NULL
+);
diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go
new file mode 100644
index 0000000..62402ad
--- /dev/null
+++ b/internal/db/migrations/migrations.go
@@ -0,0 +1,15 @@
+// Package migrations provides the embedded SQL migration files.
+package migrations
+
+import (
+ "embed"
+ "io/fs"
+)
+
+//go:embed *.sql
+var files embed.FS
+
+// FS returns the embedded filesystem containing the migration files.
+func FS() fs.FS {
+ return files
+}
diff --git a/internal/encurl/encurl.go b/internal/encurl/encurl.go
index 5c8819c..9d67d35 100644
--- a/internal/encurl/encurl.go
+++ b/internal/encurl/encurl.go
@@ -7,18 +7,15 @@ import (
"github.com/fxamacker/cbor/v2"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/seal"
)
// Default values for optional fields.
const (
DefaultQuality = 85
- DefaultFormat = imgcache.FormatOriginal
- DefaultFitMode = imgcache.FitCover
-
- // HKDF salt for URL encryption key derivation
- urlKeySalt = "pixa-urlenc-v1"
+ DefaultFormat = imagerequest.FormatJXL
+ DefaultFitMode = imagerequest.FitCover
)
// Errors returned by encurl operations.
@@ -28,19 +25,19 @@ var (
ErrDecryptFailed = errors.New("failed to decrypt URL")
)
-// Payload contains all image request parameters in a compact format.
+// payload contains all image request parameters in a compact format.
// Field names are short to minimize encoded size.
// Fields with omitempty use sane defaults when absent.
-type Payload struct {
- SourceHost string `cbor:"h"` // required
- SourcePath string `cbor:"p"` // required
- SourceQuery string `cbor:"q,omitempty"` // optional
- Width int `cbor:"w,omitempty"` // 0 = original
- Height int `cbor:"ht,omitempty"` // 0 = original
- Format imgcache.ImageFormat `cbor:"f,omitempty"` // default: orig
- Quality int `cbor:"ql,omitempty"` // default: 85
- FitMode imgcache.FitMode `cbor:"fm,omitempty"` // default: cover
- ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
+type payload struct {
+ SourceHost string `cbor:"h"` // required
+ SourcePath string `cbor:"p"` // required
+ SourceQuery string `cbor:"q,omitempty"` // optional
+ Width int `cbor:"w,omitempty"` // 0 = original
+ Height int `cbor:"ht,omitempty"` // 0 = original
+ Format imagerequest.Format `cbor:"f,omitempty"` // default: jxl
+ Quality int `cbor:"ql,omitempty"` // default: 85
+ FitMode imagerequest.Fit `cbor:"fm,omitempty"` // default: cover
+ ExpiresAt int64 `cbor:"e,omitempty"` // 0 = never expires
}
// Generator creates and parses encrypted URL tokens.
@@ -48,20 +45,35 @@ type Generator struct {
key [seal.KeySize]byte
}
-// NewGenerator creates an encrypted URL generator with a key derived
-// from the signing key.
-func NewGenerator(signingKey string) (*Generator, error) {
- key, err := seal.DeriveKey([]byte(signingKey), urlKeySalt)
- if err != nil {
- return nil, err
- }
-
- return &Generator{key: key}, nil
+// NewGenerator creates an encrypted URL generator that encrypts its tokens
+// with key.
+func NewGenerator(key [seal.KeySize]byte) *Generator {
+ return &Generator{key: key}
}
-// Generate creates an encrypted URL token from the payload.
-// The token is CBOR-encoded, encrypted with NaCl secretbox, and base64url-encoded.
-func (g *Generator) Generate(p *Payload) (string, error) {
+// Generate creates an encrypted URL token from the request's source and
+// transform, each as the request holds it, and from expires; an empty format,
+// a quality of 0 or an empty fit mode is left out of the token, and a zero
+// expires makes a token that never expires. The token is CBOR-encoded,
+// encrypted with NaCl secretbox, and base64url-encoded.
+func (g *Generator) Generate(
+ req *imagerequest.Request, expires time.Time,
+) (string, error) {
+ p := &payload{
+ SourceHost: req.Source.Host,
+ SourcePath: req.Source.Path,
+ SourceQuery: req.Source.Query,
+ Width: req.Transform.Size.Width,
+ Height: req.Transform.Size.Height,
+ Format: req.Transform.Format,
+ Quality: req.Transform.Quality,
+ FitMode: req.Transform.FitMode,
+ }
+
+ if !expires.IsZero() {
+ p.ExpiresAt = expires.Unix()
+ }
+
// CBOR encode the payload
data, err := cbor.Marshal(p)
if err != nil {
@@ -72,39 +84,35 @@ func (g *Generator) Generate(p *Payload) (string, error) {
return seal.Encrypt(g.key, data)
}
-// Parse decrypts and validates an encrypted URL token.
+// Parse decrypts and validates an encrypted URL token and returns the
+// request it holds, with default values for omitted optional fields, and its
+// expiry. A token that never expires gives the zero time.
// Returns ErrExpired if the token has expired, or other errors for invalid tokens.
-func (g *Generator) Parse(token string) (*Payload, error) {
+func (g *Generator) Parse(token string) (*imagerequest.Request, time.Time, error) {
// Decrypt
data, err := seal.Decrypt(g.key, token)
if err != nil {
if errors.Is(err, seal.ErrDecryptionFailed) ||
errors.Is(err, seal.ErrInvalidPayload) {
- return nil, ErrDecryptFailed
+ return nil, time.Time{}, ErrDecryptFailed
}
- return nil, err
+ return nil, time.Time{}, err
}
// CBOR decode
- var p Payload
+ var p payload
err = cbor.Unmarshal(data, &p)
if err != nil {
- return nil, ErrInvalidFormat
+ return nil, time.Time{}, ErrInvalidFormat
}
// Check expiration (0 = never expires)
if p.ExpiresAt != 0 && time.Now().Unix() > p.ExpiresAt {
- return nil, ErrExpired
+ return nil, time.Time{}, ErrExpired
}
- return &p, nil
-}
-
-// ToImageRequest converts the payload to an ImageRequest.
-// Applies default values for omitted optional fields.
-func (p *Payload) ToImageRequest() *imgcache.ImageRequest {
format := p.Format
if format == "" {
format = DefaultFormat
@@ -120,43 +128,22 @@ func (p *Payload) ToImageRequest() *imgcache.ImageRequest {
fitMode = DefaultFitMode
}
- return &imgcache.ImageRequest{
- SourceHost: p.SourceHost,
- SourcePath: p.SourcePath,
- SourceQuery: p.SourceQuery,
- Size: imgcache.Size{
- Width: p.Width,
- Height: p.Height,
+ var expires time.Time
+ if p.ExpiresAt != 0 {
+ expires = time.Unix(p.ExpiresAt, 0)
+ }
+
+ return &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: p.SourceHost,
+ Path: p.SourcePath,
+ Query: p.SourceQuery,
},
- Format: format,
- Quality: quality,
- FitMode: fitMode,
- }
-}
-
-// FromImageRequest creates a Payload from an ImageRequest with the given expiration.
-func FromImageRequest(req *imgcache.ImageRequest, expiresAt time.Time) *Payload {
- p := &Payload{
- SourceHost: req.SourceHost,
- SourcePath: req.SourcePath,
- SourceQuery: req.SourceQuery,
- Width: req.Size.Width,
- Height: req.Size.Height,
- ExpiresAt: expiresAt.Unix(),
- }
-
- // Only set non-default values to benefit from omitempty
- if req.Format != "" && req.Format != DefaultFormat {
- p.Format = req.Format
- }
-
- if req.Quality != 0 && req.Quality != DefaultQuality {
- p.Quality = req.Quality
- }
-
- if req.FitMode != "" && req.FitMode != DefaultFitMode {
- p.FitMode = req.FitMode
- }
-
- return p
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: p.Width, Height: p.Height},
+ Format: format,
+ Quality: quality,
+ FitMode: fitMode,
+ },
+ }, expires, nil
}
diff --git a/internal/encurl/encurl_test.go b/internal/encurl/encurl_test.go
index 48a1dd8..7b19068 100644
--- a/internal/encurl/encurl_test.go
+++ b/internal/encurl/encurl_test.go
@@ -6,7 +6,8 @@ import (
"time"
"sneak.berlin/go/pixa/internal/encurl"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/seal"
)
// Shared test fixture strings.
@@ -19,24 +20,29 @@ const (
func TestGenerator_GenerateAndParse(t *testing.T) {
t.Parallel()
- gen, err := encurl.NewGenerator("test-signing-key-12345")
+ keys, err := seal.DeriveKeys("test-signing-key-12345")
if err != nil {
- t.Fatalf("NewGenerator() error = %v", err)
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
}
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- SourceQuery: testSourceQuery,
- Width: 800,
- Height: 600,
- Format: imgcache.FormatWebP,
- Quality: 90,
- FitMode: imgcache.FitContain,
- ExpiresAt: time.Now().Add(time.Hour).Unix(),
- }
+ gen := encurl.NewGenerator(keys.EncryptedURLKey)
- token, err := gen.Generate(payload)
+ req := &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: testSourceHost,
+ Path: testSourcePath,
+ Query: testSourceQuery,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 800, Height: 600},
+ Format: imagerequest.FormatWebP,
+ Quality: 90,
+ FitMode: imagerequest.FitContain,
+ },
+ }
+ expires := time.Now().Add(time.Hour)
+
+ token, err := gen.Generate(req, expires)
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
@@ -45,66 +51,73 @@ func TestGenerator_GenerateAndParse(t *testing.T) {
t.Fatal("Generate() returned empty token")
}
- parsed, err := gen.Parse(token)
+ parsed, parsedExpires, err := gen.Parse(token)
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
// Verify all fields
- if parsed.SourceHost != payload.SourceHost {
- t.Errorf("SourceHost = %q, want %q", parsed.SourceHost, payload.SourceHost)
+ if parsed.Source.Host != req.Source.Host {
+ t.Errorf("SourceHost = %q, want %q", parsed.Source.Host, req.Source.Host)
}
- if parsed.SourcePath != payload.SourcePath {
- t.Errorf("SourcePath = %q, want %q", parsed.SourcePath, payload.SourcePath)
+ if parsed.Source.Path != req.Source.Path {
+ t.Errorf("SourcePath = %q, want %q", parsed.Source.Path, req.Source.Path)
}
- if parsed.SourceQuery != payload.SourceQuery {
- t.Errorf("SourceQuery = %q, want %q", parsed.SourceQuery, payload.SourceQuery)
+ if parsed.Source.Query != req.Source.Query {
+ t.Errorf("SourceQuery = %q, want %q", parsed.Source.Query, req.Source.Query)
}
- if parsed.Width != payload.Width {
- t.Errorf("Width = %d, want %d", parsed.Width, payload.Width)
+ if parsed.Transform.Size.Width != req.Transform.Size.Width {
+ t.Errorf("Width = %d, want %d",
+ parsed.Transform.Size.Width, req.Transform.Size.Width)
}
- if parsed.Height != payload.Height {
- t.Errorf("Height = %d, want %d", parsed.Height, payload.Height)
+ if parsed.Transform.Size.Height != req.Transform.Size.Height {
+ t.Errorf("Height = %d, want %d",
+ parsed.Transform.Size.Height, req.Transform.Size.Height)
}
- if parsed.Format != payload.Format {
- t.Errorf("Format = %q, want %q", parsed.Format, payload.Format)
+ if parsed.Transform.Format != req.Transform.Format {
+ t.Errorf("Format = %q, want %q",
+ parsed.Transform.Format, req.Transform.Format)
}
- if parsed.Quality != payload.Quality {
- t.Errorf("Quality = %d, want %d", parsed.Quality, payload.Quality)
+ if parsed.Transform.Quality != req.Transform.Quality {
+ t.Errorf("Quality = %d, want %d",
+ parsed.Transform.Quality, req.Transform.Quality)
}
- if parsed.FitMode != payload.FitMode {
- t.Errorf("FitMode = %q, want %q", parsed.FitMode, payload.FitMode)
+ if parsed.Transform.FitMode != req.Transform.FitMode {
+ t.Errorf("FitMode = %q, want %q",
+ parsed.Transform.FitMode, req.Transform.FitMode)
}
- if parsed.ExpiresAt != payload.ExpiresAt {
- t.Errorf("ExpiresAt = %d, want %d", parsed.ExpiresAt, payload.ExpiresAt)
+ if parsedExpires.Unix() != expires.Unix() {
+ t.Errorf("ExpiresAt = %d, want %d", parsedExpires.Unix(), expires.Unix())
}
}
func TestGenerator_Parse_Expired(t *testing.T) {
t.Parallel()
- gen, _ := encurl.NewGenerator("test-signing-key-12345")
+ keys, _ := seal.DeriveKeys("test-signing-key-12345")
+ gen := encurl.NewGenerator(keys.EncryptedURLKey)
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- ExpiresAt: time.Now().Add(-time.Hour).Unix(), // Already expired
+ req := &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: testSourceHost,
+ Path: testSourcePath,
+ },
}
- token, err := gen.Generate(payload)
+ token, err := gen.Generate(req, time.Now().Add(-time.Hour)) // Already expired
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
- _, err = gen.Parse(token)
+ _, _, err = gen.Parse(token)
if err == nil {
t.Error("Parse() should fail for expired token")
}
@@ -117,9 +130,10 @@ func TestGenerator_Parse_Expired(t *testing.T) {
func TestGenerator_Parse_InvalidToken(t *testing.T) {
t.Parallel()
- gen, _ := encurl.NewGenerator("test-signing-key-12345")
+ keys, _ := seal.DeriveKeys("test-signing-key-12345")
+ gen := encurl.NewGenerator(keys.EncryptedURLKey)
- _, err := gen.Parse("not-a-valid-token")
+ _, _, err := gen.Parse("not-a-valid-token")
if err == nil {
t.Error("Parse() should fail for invalid token")
}
@@ -128,15 +142,17 @@ func TestGenerator_Parse_InvalidToken(t *testing.T) {
func TestGenerator_Parse_TamperedToken(t *testing.T) {
t.Parallel()
- gen, _ := encurl.NewGenerator("test-signing-key-12345")
+ keys, _ := seal.DeriveKeys("test-signing-key-12345")
+ gen := encurl.NewGenerator(keys.EncryptedURLKey)
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- ExpiresAt: time.Now().Add(time.Hour).Unix(),
+ req := &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: testSourceHost,
+ Path: testSourcePath,
+ },
}
- token, _ := gen.Generate(payload)
+ token, _ := gen.Generate(req, time.Now().Add(time.Hour))
// Tamper with the token
tampered := []byte(token)
@@ -144,7 +160,7 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) {
tampered[10] ^= 0x01
}
- _, err := gen.Parse(string(tampered))
+ _, _, err := gen.Parse(string(tampered))
if err == nil {
t.Error("Parse() should fail for tampered token")
}
@@ -153,172 +169,78 @@ func TestGenerator_Parse_TamperedToken(t *testing.T) {
func TestGenerator_Parse_WrongKey(t *testing.T) {
t.Parallel()
- gen1, _ := encurl.NewGenerator("signing-key-1")
- gen2, _ := encurl.NewGenerator("signing-key-2")
+ keys1, _ := seal.DeriveKeys("signing-key-1")
+ keys2, _ := seal.DeriveKeys("signing-key-2")
+ gen1 := encurl.NewGenerator(keys1.EncryptedURLKey)
+ gen2 := encurl.NewGenerator(keys2.EncryptedURLKey)
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- ExpiresAt: time.Now().Add(time.Hour).Unix(),
+ req := &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: testSourceHost,
+ Path: testSourcePath,
+ },
}
- token, _ := gen1.Generate(payload)
+ token, _ := gen1.Generate(req, time.Now().Add(time.Hour))
- _, err := gen2.Parse(token)
+ _, _, err := gen2.Parse(token)
if err == nil {
t.Error("Parse() should fail with different signing key")
}
}
-func TestPayload_ToImageRequest(t *testing.T) {
+func TestGenerator_Parse_Defaults(t *testing.T) {
t.Parallel()
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- SourceQuery: testSourceQuery,
- Width: 800,
- Height: 600,
- Format: imgcache.FormatWebP,
- Quality: 90,
- FitMode: imgcache.FitContain,
- ExpiresAt: time.Now().Add(time.Hour).Unix(),
+ keys, _ := seal.DeriveKeys("test-signing-key-12345")
+ gen := encurl.NewGenerator(keys.EncryptedURLKey)
+
+ // Request with only required fields - should get defaults
+ token, err := gen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: testSourceHost,
+ Path: testSourcePath,
+ },
+ }, time.Now().Add(time.Hour))
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
}
- req := payload.ToImageRequest()
-
- if req.SourceHost != payload.SourceHost {
- t.Errorf("SourceHost = %q, want %q", req.SourceHost, payload.SourceHost)
+ req, _, err := gen.Parse(token)
+ if err != nil {
+ t.Fatalf("Parse() error = %v", err)
}
- if req.SourcePath != payload.SourcePath {
- t.Errorf("SourcePath = %q, want %q", req.SourcePath, payload.SourcePath)
+ if req.Transform.Format != encurl.DefaultFormat {
+ t.Errorf("Format = %q, want default %q",
+ req.Transform.Format, encurl.DefaultFormat)
}
- if req.SourceQuery != payload.SourceQuery {
- t.Errorf("SourceQuery = %q, want %q", req.SourceQuery, payload.SourceQuery)
+ if req.Transform.Quality != encurl.DefaultQuality {
+ t.Errorf("Quality = %d, want default %d",
+ req.Transform.Quality, encurl.DefaultQuality)
}
- if req.Size.Width != payload.Width {
- t.Errorf("Width = %d, want %d", req.Size.Width, payload.Width)
- }
-
- if req.Size.Height != payload.Height {
- t.Errorf("Height = %d, want %d", req.Size.Height, payload.Height)
- }
-
- if req.Format != payload.Format {
- t.Errorf("Format = %q, want %q", req.Format, payload.Format)
- }
-
- if req.Quality != payload.Quality {
- t.Errorf("Quality = %d, want %d", req.Quality, payload.Quality)
- }
-
- if req.FitMode != payload.FitMode {
- t.Errorf("FitMode = %q, want %q", req.FitMode, payload.FitMode)
- }
-}
-
-func TestPayload_ToImageRequest_Defaults(t *testing.T) {
- t.Parallel()
-
- // Payload with only required fields - should get defaults
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- ExpiresAt: time.Now().Add(time.Hour).Unix(),
- }
-
- req := payload.ToImageRequest()
-
- if req.Format != encurl.DefaultFormat {
- t.Errorf("Format = %q, want default %q", req.Format, encurl.DefaultFormat)
- }
-
- if req.Quality != encurl.DefaultQuality {
- t.Errorf("Quality = %d, want default %d", req.Quality, encurl.DefaultQuality)
- }
-
- if req.FitMode != encurl.DefaultFitMode {
- t.Errorf("FitMode = %q, want default %q", req.FitMode, encurl.DefaultFitMode)
- }
-}
-
-func TestFromImageRequest(t *testing.T) {
- t.Parallel()
-
- req := &imgcache.ImageRequest{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- SourceQuery: testSourceQuery,
- Size: imgcache.Size{Width: 800, Height: 600},
- Format: imgcache.FormatWebP,
- Quality: 90,
- FitMode: imgcache.FitContain,
- }
-
- expiresAt := time.Now().Add(time.Hour)
- payload := encurl.FromImageRequest(req, expiresAt)
-
- if payload.SourceHost != req.SourceHost {
- t.Errorf("SourceHost = %q, want %q", payload.SourceHost, req.SourceHost)
- }
-
- if payload.SourcePath != req.SourcePath {
- t.Errorf("SourcePath = %q, want %q", payload.SourcePath, req.SourcePath)
- }
-
- if payload.Width != req.Size.Width {
- t.Errorf("Width = %d, want %d", payload.Width, req.Size.Width)
- }
-
- if payload.ExpiresAt != expiresAt.Unix() {
- t.Errorf("ExpiresAt = %d, want %d", payload.ExpiresAt, expiresAt.Unix())
- }
-}
-
-func TestFromImageRequest_OmitsDefaults(t *testing.T) {
- t.Parallel()
-
- // Request with default values - payload should omit them for
- // smaller encoding
- req := &imgcache.ImageRequest{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- Format: encurl.DefaultFormat,
- Quality: encurl.DefaultQuality,
- FitMode: encurl.DefaultFitMode,
- }
-
- payload := encurl.FromImageRequest(req, time.Now().Add(time.Hour))
-
- // These should be zero/empty because they match defaults
- if payload.Format != "" {
- t.Errorf("Format should be empty for default, got %q", payload.Format)
- }
-
- if payload.Quality != 0 {
- t.Errorf("Quality should be 0 for default, got %d", payload.Quality)
- }
-
- if payload.FitMode != "" {
- t.Errorf("FitMode should be empty for default, got %q", payload.FitMode)
+ if req.Transform.FitMode != encurl.DefaultFitMode {
+ t.Errorf("FitMode = %q, want default %q",
+ req.Transform.FitMode, encurl.DefaultFitMode)
}
}
func TestGenerator_TokenIsURLSafe(t *testing.T) {
t.Parallel()
- gen, _ := encurl.NewGenerator("test-signing-key-12345")
+ keys, _ := seal.DeriveKeys("test-signing-key-12345")
+ gen := encurl.NewGenerator(keys.EncryptedURLKey)
- payload := &encurl.Payload{
- SourceHost: testSourceHost,
- SourcePath: testSourcePath,
- ExpiresAt: time.Now().Add(time.Hour).Unix(),
+ req := &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: testSourceHost,
+ Path: testSourcePath,
+ },
}
- token, _ := gen.Generate(payload)
+ token, _ := gen.Generate(req, time.Now().Add(time.Hour))
// Check that token only contains URL-safe characters
for _, c := range token {
diff --git a/internal/globals/globals.go b/internal/globals/globals.go
index ac00731..7a3d2bb 100644
--- a/internal/globals/globals.go
+++ b/internal/globals/globals.go
@@ -7,8 +7,8 @@ import (
const appname = "pixad"
-// Version is populated from main() via ldflags.
-var Version string //nolint:gochecknoglobals // set from main
+// Version is set by app.Run to the version main was built with.
+var Version string //nolint:gochecknoglobals // set by app.Run
// Globals holds application-wide constants.
type Globals struct {
diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go
index a0ad493..1dda36d 100644
--- a/internal/handlers/auth.go
+++ b/internal/handlers/auth.go
@@ -9,12 +9,14 @@ import (
"math"
"net/http"
"net/url"
+ "slices"
"strconv"
+ "strings"
"time"
"sneak.berlin/go/pixa/internal/clientip"
"sneak.berlin/go/pixa/internal/encurl"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/templates"
)
@@ -89,10 +91,18 @@ func (s *Handlers) handleLoginPost(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/", http.StatusSeeOther)
}
-// HandleLogout clears the session and redirects to login.
+// HandleLogout logs out, which ends every login session (see
+// session.Manager.Logout), and redirects to login.
func (s *Handlers) HandleLogout() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
- s.sessMgr.ClearSession(w)
+ err := s.sessMgr.Logout(w, r)
+ if err != nil {
+ s.log.Error("failed to log out", "error", err)
+ http.Error(w, "Failed to log out", http.StatusInternalServerError)
+
+ return
+ }
+
http.Redirect(w, r, "/", http.StatusSeeOther)
}
}
@@ -117,7 +127,7 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
sourceURL := r.FormValue("url")
// Validate source URL
- parsed, err := url.Parse(sourceURL)
+ parsed, err := parseSourceURL(sourceURL)
if err != nil || parsed.Host == "" {
s.renderGeneratorWithForm(w, r, "Invalid source URL", r.Form,
http.StatusBadRequest)
@@ -125,7 +135,17 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
return
}
- payload, expiresAt, ttl, err := buildGeneratePayload(parsed, r.Form)
+ // /v1/image/ refuses a ".." segment in the source path, and so does
+ // the generator. The path is decoded, so %2e%2e counts too
+ if slices.Contains(strings.Split(parsed.Path, "/"), "..") {
+ s.renderGeneratorWithForm(w, r,
+ "Source URL path cannot hold a .. segment", r.Form,
+ http.StatusBadRequest)
+
+ return
+ }
+
+ req, expiresAt, ttl, err := buildGenerateRequest(parsed, r.Form)
if err != nil {
s.renderGeneratorWithForm(w, r, err.Error(), r.Form,
http.StatusBadRequest)
@@ -133,8 +153,20 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
return
}
+ // The image URL must fetch the source URL as it was written. The
+ // fetcher fetches over http only when allow_http is set.
+ fetchURL := req.Source.URL(s.config.AllowHTTP)
+
+ if !fetchedAsWritten(parsed, fetchURL) {
+ s.renderGeneratorWithForm(w, r, "Source URL cannot be fetched as "+
+ "written: pixa would fetch "+fetchURL.String(),
+ r.Form, http.StatusBadRequest)
+
+ return
+ }
+
// Generate encrypted token
- token, err := s.encGen.Generate(payload)
+ token, err := s.encGen.Generate(req, expiresAt)
if err != nil {
s.log.Error("failed to generate encrypted URL", "error", err)
s.renderGeneratorWithForm(w, r, "Failed to generate URL", r.Form,
@@ -165,14 +197,55 @@ func (s *Handlers) HandleGenerateURL() http.HandlerFunc {
}
}
-// buildGeneratePayload parses the numeric form fields and assembles the
-// encrypted URL payload. ttl=0 means never expires (ExpiresAt stays 0). A
-// non-numeric or out-of-range width, height, quality or ttl, or an
-// unrecognized fit mode, is a client error naming the offending field. The
-// format field is passed through unchecked.
-func buildGeneratePayload(
+// parseSourceURL parses the source URL entered on the generator page with its
+// host lower-cased and an empty path written as "/", as /v1/image/ reads a
+// source URL, so that each spelling of one source URL makes one cache key and
+// one upstream request.
+func parseSourceURL(sourceURL string) (*url.URL, error) {
+ parsed, err := url.Parse(sourceURL)
+ if err != nil {
+ return nil, err
+ }
+
+ parsed.Host = strings.ToLower(parsed.Host)
+
+ if parsed.Path == "" {
+ parsed.Path = "/"
+ }
+
+ return parsed, nil
+}
+
+// fetchedAsWritten reports whether fetchURL, the URL pixa fetches for an
+// image URL made from the source URL source, is source as it was written,
+// apart from the scheme, which the token does not keep. It is not when source
+// holds a user name and password, which the token does not keep either, or a
+// "?" with no query after it. The token keeps the source path decoded and
+// pixa fetches it percent-encoded again, so a path written another way, such
+// as with %2F, is fetched as another path. url.URL keeps the path as written
+// in RawPath when it is not the decoded path percent-encoded again.
+// EscapedPath ignores RawPath when it also holds a character net/url encodes,
+// such as a space, so RawPath is compared itself.
+func fetchedAsWritten(source, fetchURL *url.URL) bool {
+ writtenPath := source.RawPath
+ if writtenPath == "" {
+ writtenPath = source.EscapedPath()
+ }
+
+ return source.User == nil && !source.ForceQuery &&
+ fetchURL.EscapedPath() == writtenPath &&
+ fetchURL.RawQuery == source.RawQuery
+}
+
+// buildGenerateRequest parses the numeric form fields and assembles the
+// image request the encrypted URL's token is made from, and its expiry.
+// ttl=0 means never expires (the expiry is the zero time). A non-numeric or
+// out-of-range width, height, quality or ttl, or an unrecognized fit mode, is
+// a client error naming the offending field. The format field is passed
+// through unchecked.
+func buildGenerateRequest(
parsed *url.URL, form url.Values,
-) (*encurl.Payload, time.Time, int, error) {
+) (*imagerequest.Request, time.Time, int, error) {
width, err := parseFormDimension(form, "width")
if err != nil {
return nil, time.Time{}, 0, err
@@ -194,42 +267,39 @@ func buildGeneratePayload(
return nil, time.Time{}, 0, err
}
- fitMode := imgcache.FitMode(form.Get("fit"))
+ fitMode := imagerequest.Fit(form.Get("fit"))
- err = imgcache.ValidateFitMode(fitMode)
+ err = imagerequest.ValidateFitMode(fitMode)
if err != nil {
return nil, time.Time{}, 0,
- fmt.Errorf("%w: %s", imgcache.ErrInvalidFitMode, form.Get("fit"))
+ fmt.Errorf("%w: %s", imagerequest.ErrInvalidFitMode, form.Get("fit"))
}
- var (
- expiresAt time.Time
- expiresAtUnix int64
- )
-
+ var expiresAt time.Time
if ttl > 0 {
expiresAt = time.Now().Add(time.Duration(ttl) * time.Second)
- expiresAtUnix = expiresAt.Unix()
}
- payload := &encurl.Payload{
- SourceHost: parsed.Host,
- SourcePath: parsed.Path,
- SourceQuery: parsed.RawQuery,
- Width: width,
- Height: height,
- Format: imgcache.ImageFormat(form.Get("format")),
- Quality: quality,
- FitMode: fitMode,
- ExpiresAt: expiresAtUnix,
+ req := &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: parsed.Host,
+ Path: parsed.Path,
+ Query: parsed.RawQuery,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: width, Height: height},
+ Format: imagerequest.Format(form.Get("format")),
+ Quality: quality,
+ FitMode: fitMode,
+ },
}
- return payload, expiresAt, ttl, nil
+ return req, expiresAt, ttl, nil
}
// parseFormDimension reads an optional width or height form field. An empty
// value means "original size" (0). A non-numeric value, or one
-// imgcache.ValidateDimension rejects, is an error naming the field.
+// imagerequest.ValidateDimension rejects, is an error naming the field.
func parseFormDimension(form url.Values, field string) (int, error) {
raw := form.Get(field)
if raw == "" {
@@ -241,7 +311,7 @@ func parseFormDimension(form url.Values, field string) (int, error) {
return 0, fmt.Errorf("%w %s: not a number", errInvalidFormField, field)
}
- err = imgcache.ValidateDimension(field, value)
+ err = imagerequest.ValidateDimension(field, value)
if err != nil {
return 0, err
}
@@ -369,10 +439,15 @@ func (s *Handlers) buildGeneratedURL(r *http.Request, token, format string) stri
scheme = "http"
}
- // Determine file extension for the trailing filename
+ // Determine file extension for the trailing filename. A form with no
+ // format makes a token with none, which is served as encurl.DefaultFormat.
ext := format
- if ext == "" || ext == "orig" {
- ext = "jpg" // Default extension
+
+ switch format {
+ case "":
+ ext = string(encurl.DefaultFormat)
+ case "orig", "auto":
+ ext = "jpg"
}
return scheme + "://" + r.Host + "/v1/e/" + url.PathEscape(token) + "/img." + ext
diff --git a/internal/handlers/auth_csrf_internal_test.go b/internal/handlers/auth_csrf_internal_test.go
index 161253b..65b6d82 100644
--- a/internal/handlers/auth_csrf_internal_test.go
+++ b/internal/handlers/auth_csrf_internal_test.go
@@ -14,6 +14,7 @@ import (
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session"
)
@@ -44,27 +45,17 @@ func newCSRFTestRouter(t *testing.T) (*Handlers, http.Handler) {
cfg := &config.Config{SigningKey: testSigningKey, Debug: true}
- sessMgr, err := session.NewManager(testSigningKey)
+ keys, err := seal.DeriveKeys(testSigningKey)
if err != nil {
- t.Fatalf("session.NewManager() error = %v", err)
- }
-
- encGen, err := encurl.NewGenerator(testSigningKey)
- if err != nil {
- t.Fatalf("encurl.NewGenerator() error = %v", err)
- }
-
- protect, err := newCSRFProtect(testSigningKey, cfg.Debug)
- if err != nil {
- t.Fatalf("newCSRFProtect() error = %v", err)
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
}
h := &Handlers{
log: slog.New(slog.DiscardHandler),
config: cfg,
- sessMgr: sessMgr,
- encGen: encGen,
- csrfProtect: protect,
+ sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
+ encGen: encurl.NewGenerator(keys.EncryptedURLKey),
+ csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug, slog.New(slog.DiscardHandler)),
}
r := chi.NewRouter()
diff --git a/internal/handlers/auth_generate_internal_test.go b/internal/handlers/auth_generate_internal_test.go
index bc26990..1da7c1b 100644
--- a/internal/handlers/auth_generate_internal_test.go
+++ b/internal/handlers/auth_generate_internal_test.go
@@ -11,7 +11,7 @@ import (
"time"
"sneak.berlin/go/pixa/internal/encurl"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
)
// Generator form field names, and a value that is not a number.
@@ -125,10 +125,10 @@ func TestGeneratePostRejectsBadField(t *testing.T) {
}
}
-// TestBuildGeneratePayloadDefaultAndLimits verifies that an empty quality
+// TestBuildGenerateRequestDefaultAndLimits verifies that an empty quality
// takes the default, and that the largest accepted width, height, quality and
// ttl are accepted with an expiry still in the future.
-func TestBuildGeneratePayloadDefaultAndLimits(t *testing.T) {
+func TestBuildGenerateRequestDefaultAndLimits(t *testing.T) {
t.Parallel()
parsed, err := url.Parse(testSourceURL)
@@ -136,19 +136,19 @@ func TestBuildGeneratePayloadDefaultAndLimits(t *testing.T) {
t.Fatalf("url.Parse() error = %v", err)
}
- payload, _, _, err := buildGeneratePayload(parsed, url.Values{})
+ req, _, _, err := buildGenerateRequest(parsed, url.Values{})
if err != nil {
t.Fatalf("empty form: error = %v", err)
}
- if payload.Quality != encurl.DefaultQuality {
+ if req.Transform.Quality != encurl.DefaultQuality {
t.Errorf("empty quality gave %d, want %d",
- payload.Quality, encurl.DefaultQuality)
+ req.Transform.Quality, encurl.DefaultQuality)
}
- _, expiresAt, _, err := buildGeneratePayload(parsed, url.Values{
- widthField: {strconv.Itoa(imgcache.MaxDimension)},
- heightField: {strconv.Itoa(imgcache.MaxDimension)},
+ _, expiresAt, _, err := buildGenerateRequest(parsed, url.Values{
+ widthField: {strconv.Itoa(imagerequest.MaxDimension)},
+ heightField: {strconv.Itoa(imagerequest.MaxDimension)},
qualityField: {strconv.Itoa(maxQuality)},
ttlField: {strconv.Itoa(maxTTL)},
})
diff --git a/internal/handlers/auth_login_log_internal_test.go b/internal/handlers/auth_login_log_internal_test.go
new file mode 100644
index 0000000..71943a8
--- /dev/null
+++ b/internal/handlers/auth_login_log_internal_test.go
@@ -0,0 +1,60 @@
+package handlers
+
+import (
+ "bytes"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+
+ "sneak.berlin/go/pixa/internal/config"
+ "sneak.berlin/go/pixa/internal/seal"
+ "sneak.berlin/go/pixa/internal/session"
+)
+
+// TestLoginLogLeavesOutSubmittedKey verifies that the log lines for a
+// failed and for a successful login do not contain the submitted key.
+func TestLoginLogLeavesOutSubmittedKey(t *testing.T) {
+ t.Parallel()
+
+ const wrongKey = "wrong-signing-key-fedcba9876543210"
+
+ var buf bytes.Buffer
+
+ keys, err := seal.DeriveKeys(testSigningKey)
+ if err != nil {
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ h := &Handlers{
+ log: slog.New(slog.NewJSONHandler(&buf, nil)),
+ config: &config.Config{SigningKey: testSigningKey},
+ sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
+ }
+
+ submittedKeys := []string{wrongKey, testSigningKey}
+
+ for _, key := range submittedKeys {
+ form := url.Values{loginKeyField: {key}}
+ req := httptest.NewRequestWithContext(
+ t.Context(), http.MethodPost, "/",
+ strings.NewReader(form.Encode()))
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+ h.handleLoginPost(httptest.NewRecorder(), req)
+ }
+
+ for _, msg := range []string{"failed login attempt", "successful login"} {
+ if !strings.Contains(buf.String(), msg) {
+ t.Fatalf("log missing %q; got %q", msg, buf.String())
+ }
+ }
+
+ for _, key := range submittedKeys {
+ if strings.Contains(buf.String(), key) {
+ t.Errorf("log contains submitted key %q; got %q", key, buf.String())
+ }
+ }
+}
diff --git a/internal/handlers/auth_session_internal_test.go b/internal/handlers/auth_session_internal_test.go
new file mode 100644
index 0000000..8920ae5
--- /dev/null
+++ b/internal/handlers/auth_session_internal_test.go
@@ -0,0 +1,390 @@
+package handlers
+
+import (
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "regexp"
+ "strings"
+ "testing"
+ "time"
+
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/seal"
+ "sneak.berlin/go/pixa/internal/session"
+)
+
+// formatField is the generator form's format field name.
+const formatField = "format"
+
+// Markers telling the login page from the generator page.
+const (
+ loginForm = `action="/"`
+ loginKeyInput = `name="key"`
+ generatorForm = `action="/generate"`
+)
+
+// generatedURLPattern extracts the path of the URL the generator page shows.
+// The test router runs with debug on, so the URL starts with http, and its
+// host is httptest's default request host.
+var generatedURLPattern = regexp.MustCompile(
+ `value="http://example\.com(/v1/e/[^"]+)"`)
+
+// findSessionCookie returns the session cookie rec sets, or nil if it sets
+// none.
+func findSessionCookie(rec *httptest.ResponseRecorder) *http.Cookie {
+ for _, c := range rec.Result().Cookies() {
+ if c.Name == session.CookieName {
+ return c
+ }
+ }
+
+ return nil
+}
+
+// TestHandleRoot_NoSession_ShowsLoginForm verifies that GET / without a
+// login session shows the login form.
+func TestHandleRoot_NoSession_ShowsLoginForm(t *testing.T) {
+ t.Parallel()
+
+ _, srv := newCSRFTestRouter(t)
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, "/", nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
+ }
+
+ body := rec.Body.String()
+ if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
+ t.Errorf("page is not the login form: %s", body)
+ }
+}
+
+// TestLoginPost_WrongKey_ShowsErrorWithoutSession verifies that a wrong key
+// shows the login form again with an error, and sets no session cookie.
+func TestLoginPost_WrongKey_ShowsErrorWithoutSession(t *testing.T) {
+ t.Parallel()
+
+ _, srv := newCSRFTestRouter(t)
+
+ cookies, token := csrfCredentials(t, srv, nil)
+
+ rec := postForm(srv, "/", cookies, url.Values{
+ loginKeyField: {"wrong-signing-key-fedcba9876543210"},
+ csrfTokenField: {token},
+ })
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
+ }
+
+ body := rec.Body.String()
+ if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
+ t.Errorf("page is not the login form: %s", body)
+ }
+
+ if !strings.Contains(body, "Invalid signing key") {
+ t.Error("login form does not show the error")
+ }
+
+ if c := findSessionCookie(rec); c != nil {
+ t.Errorf("wrong key set a session cookie: %s", c)
+ }
+}
+
+// TestLoginPost_RightKey_SetsSessionCookie verifies that the right key answers
+// 303 to / with a session cookie marked Secure, HttpOnly and SameSite=Strict,
+// and that GET / with that cookie shows the generator page.
+func TestLoginPost_RightKey_SetsSessionCookie(t *testing.T) {
+ t.Parallel()
+
+ _, srv := newCSRFTestRouter(t)
+
+ cookies, token := csrfCredentials(t, srv, nil)
+
+ rec := postForm(srv, "/", cookies, url.Values{
+ loginKeyField: {testSigningKey},
+ csrfTokenField: {token},
+ })
+
+ if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
+ t.Fatalf("status = %d, Location = %q, want %d to /",
+ rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
+ }
+
+ sessionCookie := findSessionCookie(rec)
+ if sessionCookie == nil {
+ t.Fatal("right key set no session cookie")
+ }
+
+ t.Logf("Set-Cookie: %s", sessionCookie)
+
+ if !sessionCookie.Secure {
+ t.Error("session cookie is not Secure")
+ }
+
+ if !sessionCookie.HttpOnly {
+ t.Error("session cookie is not HttpOnly")
+ }
+
+ if sessionCookie.SameSite != http.SameSiteStrictMode {
+ t.Errorf("session cookie SameSite = %v, want Strict", sessionCookie.SameSite)
+ }
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.AddCookie(sessionCookie)
+
+ rec = httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK ||
+ !strings.Contains(rec.Body.String(), generatorForm) {
+ t.Errorf("GET / with the session cookie: status = %d, "+
+ "want %d and the generator page", rec.Code, http.StatusOK)
+ }
+}
+
+// TestHandleLogout_ClearsSessionCookie verifies that GET /logout answers 303
+// to / and replaces the session cookie with an empty one sent with
+// Max-Age=0, which makes the browser delete it.
+func TestHandleLogout_ClearsSessionCookie(t *testing.T) {
+ t.Parallel()
+
+ h, _ := newCSRFTestRouter(t)
+
+ req := httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, "/logout", nil)
+ req.AddCookie(newSessionCookie(t, h))
+
+ rec := httptest.NewRecorder()
+ h.HandleLogout().ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
+ t.Fatalf("status = %d, Location = %q, want %d to /",
+ rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
+ }
+
+ t.Logf("Set-Cookie: %s", rec.Header().Get("Set-Cookie"))
+
+ sessionCookie := findSessionCookie(rec)
+ if sessionCookie == nil {
+ t.Fatal("logout did not set the session cookie")
+ }
+
+ if sessionCookie.Value != "" {
+ t.Errorf("session cookie value = %q, want empty", sessionCookie.Value)
+ }
+
+ // net/http reads a Max-Age=0 attribute back as MaxAge -1.
+ if sessionCookie.MaxAge != -1 {
+ t.Errorf("session cookie MaxAge = %d, want -1 (Max-Age=0)",
+ sessionCookie.MaxAge)
+ }
+}
+
+// TestHandleLogout_EndsCopiedSession verifies that after POST /logout a copy
+// of the session cookie taken before it shows the login page, not the
+// generator page.
+func TestHandleLogout_EndsCopiedSession(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newCSRFTestRouter(t)
+
+ copied := newSessionCookie(t, h)
+
+ req := httptest.NewRequestWithContext(
+ t.Context(), http.MethodPost, "/logout", nil)
+ req.AddCookie(copied)
+
+ rec := httptest.NewRecorder()
+ h.HandleLogout().ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusSeeOther {
+ t.Fatalf("POST /logout status = %d, want %d",
+ rec.Code, http.StatusSeeOther)
+ }
+
+ req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.AddCookie(copied)
+
+ rec = httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+
+ body := rec.Body.String()
+ t.Logf("GET / with the copied cookie: status %d, login page %v",
+ rec.Code, strings.Contains(body, loginForm))
+
+ if !strings.Contains(body, loginForm) || strings.Contains(body, generatorForm) {
+ t.Error("a cookie copied before /logout still opens the generator page")
+ }
+}
+
+// TestHandleLogout_SaveFails_Answers500 verifies that when the logout cannot
+// be saved in the database, POST /logout answers 500, leaves the session
+// cookie in place and ends no session.
+func TestHandleLogout_SaveFails_Answers500(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newCSRFTestRouter(t)
+
+ db := setupTestDB(t)
+
+ keys, err := seal.DeriveKeys(testSigningKey)
+ if err != nil {
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ sessMgr, err := session.NewManagerWithDatabase(
+ t.Context(), keys.SessionHashKey, keys.SessionBlockKey, db)
+ if err != nil {
+ t.Fatalf("session.NewManagerWithDatabase() error = %v", err)
+ }
+
+ h.sessMgr = sessMgr
+ cookie := newSessionCookie(t, h)
+
+ // Saving the logout fails on a closed database.
+ err = db.Close()
+ if err != nil {
+ t.Fatalf("closing the database: %v", err)
+ }
+
+ req := httptest.NewRequestWithContext(
+ t.Context(), http.MethodPost, "/logout", nil)
+ req.AddCookie(cookie)
+
+ rec := httptest.NewRecorder()
+ h.HandleLogout().ServeHTTP(rec, req)
+
+ t.Logf("POST /logout: status %d, Set-Cookie %q",
+ rec.Code, rec.Header().Get("Set-Cookie"))
+
+ if rec.Code != http.StatusInternalServerError {
+ t.Errorf("POST /logout status = %d, want %d",
+ rec.Code, http.StatusInternalServerError)
+ }
+
+ if c := findSessionCookie(rec); c != nil {
+ t.Errorf("a logout that failed set the session cookie: %s", c)
+ }
+
+ req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.AddCookie(cookie)
+
+ rec = httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+
+ if !strings.Contains(rec.Body.String(), generatorForm) {
+ t.Error("a logout that failed ended the session")
+ }
+}
+
+// TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate
+// with a valid CSRF token but no login session answers 303 to / and makes no
+// URL.
+func TestGeneratePost_NoSession_RedirectsToLogin(t *testing.T) {
+ t.Parallel()
+
+ _, srv := newCSRFTestRouter(t)
+
+ cookies, token := csrfCredentials(t, srv, nil)
+
+ rec := postForm(srv, "/generate", cookies, url.Values{
+ sourceURLField: {testSourceURL},
+ csrfTokenField: {token},
+ })
+
+ if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
+ t.Fatalf("status = %d, Location = %q, want %d to /",
+ rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
+ }
+
+ if strings.Contains(rec.Body.String(), "/v1/e/") {
+ t.Error("a URL was made without a login session")
+ }
+}
+
+// TestGeneratePost_URLServesImage verifies that the URL the generator page
+// makes is served by /v1/e/. The image route runs on handlers of its own,
+// made with the same signing key.
+func TestGeneratePost_URLServesImage(t *testing.T) {
+ t.Parallel()
+
+ _, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ rec := generatePost(t, url.Values{
+ sourceURLField: {"https://" + signedHost + photoPath},
+ widthField: {"50"},
+ heightField: {"50"},
+ formatField: {string(imagerequest.FormatJPEG)},
+ })
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
+ }
+
+ match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
+ if match == nil {
+ t.Fatalf("generator page shows no URL: %s", rec.Body.String())
+ }
+
+ t.Logf("generated URL path: %s", match[1])
+
+ imageRec := httptest.NewRecorder()
+ imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, match[1], nil))
+
+ requireServedPhoto(t, imageRec)
+}
+
+// TestGeneratePost_URLWithTTLExpires verifies that a URL the generator page
+// makes with a ttl of one second is served by /v1/e/ at once and answers 410
+// once the ttl has passed.
+func TestGeneratePost_URLWithTTLExpires(t *testing.T) {
+ t.Parallel()
+
+ _, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ rec := generatePost(t, url.Values{
+ sourceURLField: {"https://" + signedHost + photoPath},
+ widthField: {"50"},
+ heightField: {"50"},
+ formatField: {string(imagerequest.FormatJPEG)},
+ ttlField: {"1"},
+ })
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
+ }
+
+ match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
+ if match == nil {
+ t.Fatalf("generator page shows no URL: %s", rec.Body.String())
+ }
+
+ imageRec := httptest.NewRecorder()
+ imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, match[1], nil))
+
+ requireServedPhoto(t, imageRec)
+
+ // The URL keeps the time it expires in whole seconds and is served
+ // through the whole of that second, so a ttl of one second has passed
+ // for certain two seconds after the URL was made.
+ time.Sleep(2 * time.Second)
+
+ imageRec = httptest.NewRecorder()
+ imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, match[1], nil))
+
+ t.Logf("GET %s after the ttl: %d %q", match[1], imageRec.Code, imageRec.Body)
+
+ if imageRec.Code != http.StatusGone {
+ t.Errorf("status after the ttl = %d, want %d",
+ imageRec.Code, http.StatusGone)
+ }
+}
diff --git a/internal/handlers/avif_too_large_internal_test.go b/internal/handlers/avif_too_large_internal_test.go
new file mode 100644
index 0000000..2febf57
--- /dev/null
+++ b/internal/handlers/avif_too_large_internal_test.go
@@ -0,0 +1,39 @@
+package handlers
+
+import (
+ "fmt"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// TestAVIFOutputTooLargeAnswers400 checks that respondImageError, which
+// answers the errors of both image routes, answers 400 naming the AVIF pixel
+// limit when the image processor refuses an AVIF output above it, wrapped as
+// the service wraps it. The request's source only goes into a log line, so it
+// is left empty.
+func TestAVIFOutputTooLargeAnswers400(t *testing.T) {
+ t.Parallel()
+
+ h := &Handlers{log: slog.New(slog.DiscardHandler)}
+ err := fmt.Errorf("image processing failed: %w: 4097x4096",
+ imageprocessor.ErrAVIFOutputTooLarge)
+ wantBody := fmt.Sprintf("above %d pixels", imageprocessor.MaxAVIFOutputPixels)
+
+ rec := httptest.NewRecorder()
+ h.respondImageError(rec, &imagerequest.Request{}, err)
+ t.Logf("status = %d, body = %q", rec.Code, rec.Body.String())
+
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest)
+ }
+
+ if !strings.Contains(rec.Body.String(), wantBody) {
+ t.Errorf("body = %q, want it to say %q", rec.Body.String(), wantBody)
+ }
+}
diff --git a/internal/handlers/bodylimit_internal_test.go b/internal/handlers/bodylimit_internal_test.go
index 59e5fd7..895a866 100644
--- a/internal/handlers/bodylimit_internal_test.go
+++ b/internal/handlers/bodylimit_internal_test.go
@@ -11,6 +11,7 @@ import (
"sneak.berlin/go/pixa/internal/config"
"sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session"
)
@@ -32,27 +33,17 @@ func newBodyLimitTestRouter(
cfg := &config.Config{SigningKey: testSigningKey, Debug: true}
- sessMgr, err := session.NewManager(testSigningKey)
+ keys, err := seal.DeriveKeys(testSigningKey)
if err != nil {
- t.Fatalf("session.NewManager() error = %v", err)
- }
-
- encGen, err := encurl.NewGenerator(testSigningKey)
- if err != nil {
- t.Fatalf("encurl.NewGenerator() error = %v", err)
- }
-
- protect, err := newCSRFProtect(testSigningKey, cfg.Debug)
- if err != nil {
- t.Fatalf("newCSRFProtect() error = %v", err)
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
}
h := &Handlers{
log: slog.New(slog.DiscardHandler),
config: cfg,
- sessMgr: sessMgr,
- encGen: encGen,
- csrfProtect: protect,
+ sessMgr: session.NewManager(keys.SessionHashKey, keys.SessionBlockKey),
+ encGen: encurl.NewGenerator(keys.EncryptedURLKey),
+ csrfProtect: newCSRFProtect(keys.CSRFKey, cfg.Debug, slog.New(slog.DiscardHandler)),
}
r := chi.NewRouter()
diff --git a/internal/handlers/cache_hit_internal_test.go b/internal/handlers/cache_hit_internal_test.go
new file mode 100644
index 0000000..6295146
--- /dev/null
+++ b/internal/handlers/cache_hit_internal_test.go
@@ -0,0 +1,278 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "image"
+ "image/jpeg"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+ "testing/fstest"
+ "time"
+
+ "github.com/go-chi/chi/v5"
+ "github.com/prometheus/client_golang/prometheus"
+
+ "sneak.berlin/go/pixa/internal/allowlist"
+ "sneak.berlin/go/pixa/internal/database"
+ "sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imageproxy"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/seal"
+ "sneak.berlin/go/pixa/internal/signature"
+)
+
+// countingHost is the allowlisted host whose photoPath the server of
+// newCountingDatabaseServer serves.
+const countingHost = "counting.example.com"
+
+// countingConnector opens connections to the SQLite database file name with
+// sqliteDriver, each one counting in statements the statements run on it.
+type countingConnector struct {
+ sqliteDriver driver.Driver
+ name string
+ statements *atomic.Int64
+}
+
+func (c countingConnector) Connect(context.Context) (driver.Conn, error) {
+ conn, err := c.sqliteDriver.Open(c.name)
+ if err != nil {
+ return nil, err
+ }
+
+ return countingConn{Conn: conn, statements: c.statements}, nil
+}
+
+func (c countingConnector) Driver() driver.Driver {
+ return c.sqliteDriver
+}
+
+// countingConn counts the statements run on its connection. It has only the
+// methods of driver.Conn, so database/sql prepares every query and every
+// other statement with Prepare, which counts it.
+type countingConn struct {
+ driver.Conn
+
+ statements *atomic.Int64
+}
+
+func (c countingConn) Prepare(query string) (driver.Stmt, error) {
+ c.statements.Add(1)
+
+ return c.Conn.Prepare(query)
+}
+
+// newCountingDatabase returns a database with pixa's schema, in a file of
+// its own and with one connection, and the count of the statements run on
+// it once the schema is in place.
+func newCountingDatabase(tb testing.TB) (*sql.DB, *atomic.Int64) {
+ tb.Helper()
+
+ path := filepath.Join(tb.TempDir(), "state.sqlite3")
+
+ schemaDB, err := sql.Open("sqlite", path)
+ if err != nil {
+ tb.Fatalf("opening the database: %v", err)
+ }
+
+ defer func() { _ = schemaDB.Close() }()
+
+ err = database.ApplyMigrations(tb.Context(), schemaDB, nil)
+ if err != nil {
+ tb.Fatalf("applying the migrations: %v", err)
+ }
+
+ statements := new(atomic.Int64)
+
+ db := sql.OpenDB(countingConnector{
+ sqliteDriver: schemaDB.Driver(),
+ name: path,
+ statements: statements,
+ })
+ db.SetMaxOpenConns(1)
+
+ tb.Cleanup(func() { _ = db.Close() })
+
+ return db, statements
+}
+
+// newCountingDatabaseServer returns a router for both image routes whose
+// image cache uses newCountingDatabase, the count of the statements run on
+// that database, and a URL on each route for a different image of
+// photoPath on countingHost. Nothing writes the cache's access times in the
+// background, so every statement counted comes from a request.
+func newCountingDatabaseServer(
+ tb testing.TB,
+) (http.Handler, *atomic.Int64, []string) {
+ tb.Helper()
+
+ db, statements := newCountingDatabase(tb)
+
+ var photo bytes.Buffer
+
+ err := jpeg.Encode(&photo, image.NewRGBA(image.Rect(0, 0, 100, 100)), nil)
+ if err != nil {
+ tb.Fatalf("encoding the photo: %v", err)
+ }
+
+ cache, err := imgcache.NewCache(db, imgcache.Config{
+ StateDir: tb.TempDir(),
+ NegativeTTL: 5 * time.Minute,
+ MaxBytes: 1 << 30,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ tb.Fatalf("imgcache.NewCache() error = %v", err)
+ }
+
+ svc, err := imageproxy.New(&imageproxy.Config{
+ Cache: cache,
+ Fetcher: newMockFetcher(fstest.MapFS{
+ countingHost + photoPath: &fstest.MapFile{Data: photo.Bytes()},
+ }),
+ Processor: imageprocessor.New(imageprocessor.Params{
+ MaxInputBytes: 50 << 20,
+ MaxConcurrentProcessing: 2,
+ }),
+ MaxResponseSize: 50 << 20,
+ Logger: slog.New(slog.DiscardHandler),
+ }, prometheus.NewRegistry())
+ if err != nil {
+ tb.Fatalf("imageproxy.New() error = %v", err)
+ }
+
+ keys, err := seal.DeriveKeys(testSigningKey)
+ if err != nil {
+ tb.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ encGen := encurl.NewGenerator(keys.EncryptedURLKey)
+
+ token, err := encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{Host: countingHost, Path: photoPath},
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 40, Height: 40},
+ Format: imagerequest.FormatJPEG,
+ Quality: 85,
+ FitMode: imagerequest.FitCover,
+ },
+ }, time.Time{})
+ if err != nil {
+ tb.Fatalf("encGen.Generate() error = %v", err)
+ }
+
+ h := &Handlers{
+ log: slog.New(slog.DiscardHandler),
+ imgSvc: svc,
+ encGen: encGen,
+ signer: signature.New(testSigningKey),
+ allowlist: allowlist.New([]string{countingHost}),
+ }
+
+ r := chi.NewRouter()
+ r.Get("/v1/image/*", h.HandleImage())
+ r.Get("/v1/e/{token}/*", h.HandleImageEnc())
+
+ return r, statements, []string{
+ "/v1/image/" + countingHost + photoPath + "/50x50.jpeg",
+ "/v1/e/" + token + "/img.jpg",
+ }
+}
+
+// serveImage sends srv a GET for target and returns the response, failing
+// the test unless it is a 200 with an image.
+func serveImage(
+ tb testing.TB, srv http.Handler, target string,
+) *httptest.ResponseRecorder {
+ tb.Helper()
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec,
+ httptest.NewRequestWithContext(tb.Context(), http.MethodGet, target, nil))
+
+ if rec.Code != http.StatusOK || rec.Body.Len() == 0 {
+ tb.Errorf("GET %s: %d with %d bytes, want 200 with an image",
+ target, rec.Code, rec.Body.Len())
+ }
+
+ return rec
+}
+
+// TestImageRoutes_CacheHitsMakeNoDatabaseCall serves an image on each image
+// route once, which makes and caches it, then serves it again from the cache.
+// Serving those cache hits must run no statement on the database.
+func TestImageRoutes_CacheHitsMakeNoDatabaseCall(t *testing.T) {
+ t.Parallel()
+
+ srv, statements, targets := newCountingDatabaseServer(t)
+
+ for _, target := range targets {
+ before := statements.Load()
+
+ serveImage(t, srv, target)
+
+ afterMiss := statements.Load()
+ t.Logf("GET %s, a miss: %d statements", target, afterMiss-before)
+
+ if afterMiss == before {
+ t.Fatalf("GET %s, a miss: no statement counted, want some", target)
+ }
+
+ for range 3 {
+ rec := serveImage(t, srv, target)
+
+ cacheStatus := rec.Header().Get("X-Pixa-Cache")
+ if cacheStatus != string(imageproxy.CacheHit) {
+ t.Errorf("GET %s: X-Pixa-Cache = %q, want %s",
+ target, cacheStatus, imageproxy.CacheHit)
+ }
+ }
+
+ if hitStatements := statements.Load() - afterMiss; hitStatements != 0 {
+ t.Errorf("GET %s: 3 cache hits ran %d statements, want 0",
+ target, hitStatements)
+ }
+ }
+}
+
+// BenchmarkHandleImage_CacheHits serves one cached image through the
+// /v1/image/ route from b.RunParallel's goroutines, and reports the cache
+// hits served per second. It fails if serving them ran any statement on the
+// database. script/bench runs it.
+func BenchmarkHandleImage_CacheHits(b *testing.B) {
+ srv, statements, targets := newCountingDatabaseServer(b)
+ target := targets[0]
+
+ serveImage(b, srv, target)
+
+ before := statements.Load()
+
+ b.ResetTimer()
+
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ rec := serveImage(b, srv, target)
+
+ cacheStatus := rec.Header().Get("X-Pixa-Cache")
+ if cacheStatus != string(imageproxy.CacheHit) {
+ b.Errorf("GET %s: X-Pixa-Cache = %q, want %s",
+ target, cacheStatus, imageproxy.CacheHit)
+
+ return
+ }
+ }
+ })
+
+ b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "hits/s")
+
+ if hitStatements := statements.Load() - before; hitStatements != 0 {
+ b.Fatalf("%d cache hits ran %d statements, want 0", b.N, hitStatements)
+ }
+}
diff --git a/internal/handlers/client_gone_internal_test.go b/internal/handlers/client_gone_internal_test.go
new file mode 100644
index 0000000..df91306
--- /dev/null
+++ b/internal/handlers/client_gone_internal_test.go
@@ -0,0 +1,177 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/go-chi/chi/v5/middleware"
+ "sneak.berlin/go/pixa/internal/httpfetcher"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// heldFetcher sends on started when a fetch begins, and holds the fetch until
+// release is closed. It then fails the fetch as an upstream error.
+type heldFetcher struct {
+ started chan struct{}
+ release chan struct{}
+}
+
+// Fetch sends on f.started, waits for f.release to be closed, and returns
+// httpfetcher.ErrUpstreamError.
+func (f heldFetcher) Fetch(
+ context.Context, imagerequest.Source, string, string,
+) (*httpfetcher.FetchResult, error) {
+ f.started <- struct{}{}
+
+ <-f.release
+
+ return nil, httpfetcher.ErrUpstreamError
+}
+
+// readLogLines decodes the JSON log lines in logs.
+func readLogLines(t *testing.T, logs *bytes.Buffer) []map[string]any {
+ t.Helper()
+
+ t.Logf("logged:\n%s", logs.String())
+
+ var lines []map[string]any
+
+ dec := json.NewDecoder(logs)
+ for dec.More() {
+ var line map[string]any
+
+ err := dec.Decode(&line)
+ if err != nil {
+ t.Fatalf("decoding log line: %v", err)
+ }
+
+ lines = append(lines, line)
+ }
+
+ return lines
+}
+
+// TestImageRoutes_ClientGoneWhileWaitingAnswers499 checks that both image
+// routes answer 499 to a request whose client goes away while it waits for an
+// image another request is making. A first request's fetch is held, so the
+// image is being made, and a second request for the same image waits for it
+// until its client goes away. The second request must be logged once at info
+// level with its request ID and the source host and path. Nothing may be
+// logged at error level.
+func TestImageRoutes_ClientGoneWhileWaitingAnswers499(t *testing.T) {
+ t.Parallel()
+
+ const requestID = "test-request-id"
+
+ for route, imageURL := range photoURLsByRoute() {
+ t.Run(route, func(t *testing.T) {
+ t.Parallel()
+
+ var logs bytes.Buffer
+
+ fetcher := heldFetcher{started: make(chan struct{}), release: make(chan struct{})}
+ h, routes := newRoutesFetchingThrough(t, fetcher,
+ slog.New(slog.NewJSONHandler(&logs, nil)))
+ target := imageURL(t, h)
+
+ firstDone := make(chan struct{})
+
+ go func() {
+ defer close(firstDone)
+
+ sendGet(t, routes, target)
+ }()
+
+ <-fetcher.started
+
+ ctx, cancel := context.WithCancel(context.WithValue(t.Context(),
+ middleware.RequestIDKey, requestID))
+ rec := httptest.NewRecorder()
+ secondDone := make(chan struct{})
+
+ go func() {
+ defer close(secondDone)
+
+ routes.ServeHTTP(rec, httptest.NewRequestWithContext(
+ ctx, http.MethodGet, target, nil))
+ }()
+
+ // Give the second request time to start waiting for the first
+ // request's image. The fetch is still held, so the second request
+ // can end only because its client went away
+ time.Sleep(100 * time.Millisecond)
+ cancel()
+ <-secondDone
+
+ close(fetcher.release)
+ <-firstDone
+
+ if rec.Code != 499 {
+ t.Errorf("status = %d, want 499", rec.Code)
+ }
+
+ var clientGoneLines int
+
+ for _, line := range readLogLines(t, &logs) {
+ if line["level"] == slog.LevelError.String() {
+ t.Errorf("logged at error level: %v", line["msg"])
+ }
+
+ if line["level"] == slog.LevelInfo.String() &&
+ line["msg"] == "client went away before the image was ready" &&
+ line["request_id"] == requestID &&
+ line["host"] == signedHost &&
+ line["path"] == photoPath {
+ clientGoneLines++
+ }
+ }
+
+ if clientGoneLines != 1 {
+ t.Errorf("info line naming the request logged %d times, want 1",
+ clientGoneLines)
+ }
+ })
+ }
+}
+
+// TestImageRoutes_CanceledWithLiveRequestAnswers500 checks that both image
+// routes answer 500, logged as "failed to get image" at error level, for an
+// image service error that wraps context.Canceled while the request's own
+// context is live. Its client has not gone away, so the cancellation is
+// pixa's own failure.
+func TestImageRoutes_CanceledWithLiveRequestAnswers500(t *testing.T) {
+ t.Parallel()
+
+ for route, imageURL := range photoURLsByRoute() {
+ t.Run(route, func(t *testing.T) {
+ t.Parallel()
+
+ var logs bytes.Buffer
+
+ h, routes := newRoutesFetchingThrough(t,
+ failingFetcher{err: context.Canceled},
+ slog.New(slog.NewJSONHandler(&logs, nil)))
+
+ rec := sendGet(t, routes, imageURL(t, h))
+ if rec.Code != http.StatusInternalServerError {
+ t.Errorf("status = %d, want %d",
+ rec.Code, http.StatusInternalServerError)
+ }
+
+ if !slices.ContainsFunc(readLogLines(t, &logs),
+ func(line map[string]any) bool {
+ return line["level"] == slog.LevelError.String() &&
+ line["msg"] == "failed to get image"
+ }) {
+ t.Error(`no "failed to get image" line logged at error level`)
+ }
+ })
+ }
+}
diff --git a/internal/handlers/csrf.go b/internal/handlers/csrf.go
index 166abdc..1139361 100644
--- a/internal/handlers/csrf.go
+++ b/internal/handlers/csrf.go
@@ -1,23 +1,43 @@
package handlers
import (
+ "bytes"
+ "errors"
"html/template"
+ "log/slog"
"net/http"
+ "net/url"
+ "github.com/go-chi/chi/v5/middleware"
"github.com/gorilla/csrf"
"sneak.berlin/go/pixa/internal/seal"
+ "sneak.berlin/go/pixa/internal/templates"
)
-// csrfKeySalt provides domain separation for the CSRF authentication key,
-// derived from the signing key so tokens survive restarts without extra
-// configuration and never reuse the session or encrypted-URL key material.
-const csrfKeySalt = "pixa-csrf-v1"
+// The causes the form-refused page names, each with what to do about it.
+const (
+ causePlainHTTP = "This page was opened over plain HTTP while debug is " +
+ "off. Open it over HTTPS, or turn debug on to use plain HTTP on " +
+ "your own machine."
+ causeHTTPSWithDebug = "This page was opened over HTTPS while debug is " +
+ "on. Turn debug off, or open the page over plain HTTP."
+ causeOtherHost = "The form was sent from another host than the one in " +
+ "the request's Host header. A reverse proxy in front of pixa must " +
+ "pass the Host header on unchanged."
+ causeNoReferer = "The browser sent neither an Origin nor a Referer " +
+ "header, and over HTTPS pixa needs one to check where the form " +
+ "came from. Let the browser send the Referer header to this site."
+ causeStaleToken = "The form token is missing or out of date. Reload " +
+ "the page and submit the form again."
+)
// newCSRFProtect builds the CSRF-protection middleware for the
// state-mutating HTML form routes. The token lives in its own cookie,
// independent of the session cookie, so it also protects the login POST
-// where no session exists yet (login CSRF).
+// where no session exists yet (login CSRF). key signs that cookie. A
+// refused form is answered by the handler that handleFormRefused returns.
+// log gets a warning with the library's reason, such as "origin invalid".
//
// When plaintext is true (local HTTP development), requests are marked
// plaintext so the library neither demands an https Referer nor sets a
@@ -25,22 +45,18 @@ const csrfKeySalt = "pixa-csrf-v1"
// service runs behind a TLS-terminating proxy, so plaintext is false and
// the library enforces its https Referer origin check.
func newCSRFProtect(
- signingKey string, plaintext bool,
-) (func(http.Handler) http.Handler, error) {
- key, err := seal.DeriveKey([]byte(signingKey), csrfKeySalt)
- if err != nil {
- return nil, err
- }
-
+ key [seal.KeySize]byte, plaintext bool, log *slog.Logger,
+) func(http.Handler) http.Handler {
protect := csrf.Protect(
key[:],
csrf.Path("/"),
csrf.Secure(!plaintext),
csrf.SameSite(csrf.SameSiteStrictMode),
+ csrf.ErrorHandler(handleFormRefused(plaintext, log)),
)
if !plaintext {
- return protect, nil
+ return protect
}
return func(next http.Handler) http.Handler {
@@ -49,7 +65,85 @@ func newCSRFProtect(
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
protected.ServeHTTP(w, csrf.PlaintextHTTPRequest(r))
})
- }, nil
+ }
+}
+
+// handleFormRefused returns the handler the CSRF middleware calls when it
+// refuses a request. It logs the library's reason with the request ID and
+// answers 403 with a page naming the cause and what to do. A reason that is
+// not a refusal, such as a failure to make a token, answers 500.
+func handleFormRefused(plaintext bool, log *slog.Logger) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reason := csrf.FailureReason(r)
+ requestID := middleware.GetReqID(r.Context())
+
+ cause := formRefusedCause(r, reason, plaintext)
+ if cause == "" {
+ log.Error("CSRF middleware failed",
+ "request_id", requestID, "error", reason)
+ http.Error(w, "Internal server error",
+ http.StatusInternalServerError)
+
+ return
+ }
+
+ log.Warn("form refused", "request_id", requestID, "reason", reason)
+
+ var page bytes.Buffer
+
+ err := templates.Render(&page, "form_refused.html",
+ struct{ Cause string }{Cause: cause})
+ if err != nil {
+ log.Error("failed to render form refused template", "error", err)
+ http.Error(w, "Internal server error",
+ http.StatusInternalServerError)
+
+ return
+ }
+
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ w.WriteHeader(http.StatusForbidden)
+
+ _, err = page.WriteTo(w)
+ if err != nil {
+ log.Error("failed to write form refused page", "error", err)
+ }
+ })
+}
+
+// formRefusedCause returns the cause the form-refused page names for the
+// CSRF middleware's reason, or "" when the reason is not a refusal.
+func formRefusedCause(r *http.Request, reason error, plaintext bool) string {
+ switch {
+ case errors.Is(reason, csrf.ErrNoToken), errors.Is(reason, csrf.ErrBadToken):
+ return causeStaleToken
+ case errors.Is(reason, csrf.ErrNoReferer):
+ return causeNoReferer
+ case errors.Is(reason, csrf.ErrBadOrigin):
+ return pageAddressCause(r.Header.Get("Origin"), r.Host, plaintext)
+ case errors.Is(reason, csrf.ErrBadReferer):
+ return pageAddressCause(r.Referer(), r.Host, plaintext)
+ default:
+ return ""
+ }
+}
+
+// pageAddressCause returns the cause for a form whose page address, the
+// Origin or Referer header, the CSRF middleware refused. The middleware
+// expects the request's own host, with https when plaintext is false and
+// http when it is true. So on the request's own host, only the scheme was
+// wrong.
+func pageAddressCause(pageAddress, host string, plaintext bool) string {
+ page, err := url.Parse(pageAddress)
+ if err != nil || page.Host != host {
+ return causeOtherHost
+ }
+
+ if plaintext {
+ return causeHTTPSWithDebug
+ }
+
+ return causePlainHTTP
}
// CSRF returns the CSRF-protection middleware for the login and generator
diff --git a/internal/handlers/csrf_refused_internal_test.go b/internal/handlers/csrf_refused_internal_test.go
new file mode 100644
index 0000000..98777c7
--- /dev/null
+++ b/internal/handlers/csrf_refused_internal_test.go
@@ -0,0 +1,205 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "html/template"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+
+ "github.com/go-chi/chi/v5/middleware"
+ "github.com/gorilla/csrf"
+
+ "sneak.berlin/go/pixa/internal/seal"
+)
+
+// refusedFormRequestID is the request ID of the requests postRefusedForm
+// sends, which the warning for a refused form must carry.
+const refusedFormRequestID = "test-request-id"
+
+// TestFormRefusedNamesCause verifies that a form the CSRF middleware refuses
+// answers 403 with a page naming the cause, and logs a warning with the
+// request ID and the middleware's reason.
+func TestFormRefusedNamesCause(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ debug bool
+ origin string
+ referer string
+ wantCause string
+ wantReason error
+ }{
+ {
+ name: "plain HTTP origin with debug off",
+ origin: "http://example.com",
+ wantCause: causePlainHTTP,
+ wantReason: csrf.ErrBadOrigin,
+ },
+ {
+ name: "plain HTTP referer with debug off",
+ referer: "http://example.com/",
+ wantCause: causePlainHTTP,
+ wantReason: csrf.ErrBadReferer,
+ },
+ {
+ name: "origin on another host",
+ origin: "https://proxy.example.net",
+ wantCause: causeOtherHost,
+ wantReason: csrf.ErrBadOrigin,
+ },
+ {
+ name: "referer on another host",
+ referer: "https://proxy.example.net/",
+ wantCause: causeOtherHost,
+ wantReason: csrf.ErrBadReferer,
+ },
+ {
+ name: "HTTPS origin with debug on",
+ debug: true,
+ origin: "https://example.com",
+ wantCause: causeHTTPSWithDebug,
+ wantReason: csrf.ErrBadOrigin,
+ },
+ {
+ name: "no origin or referer with debug off",
+ wantCause: causeNoReferer,
+ wantReason: csrf.ErrNoReferer,
+ },
+ {
+ name: "missing token",
+ debug: true,
+ origin: "http://example.com",
+ wantCause: causeStaleToken,
+ wantReason: csrf.ErrNoToken,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ rec, logLine := postRefusedForm(t, tt.debug, tt.origin, tt.referer,
+ nil, "")
+ checkFormRefused(t, rec, logLine, tt.wantCause, tt.wantReason)
+ })
+ }
+}
+
+// TestFormRefusedNamesCauseForForeignToken verifies that a form whose token
+// does not match its CSRF cookie answers 403 with a page naming
+// causeStaleToken, and logs a warning with the request ID and the
+// middleware's reason.
+func TestFormRefusedNamesCauseForForeignToken(t *testing.T) {
+ t.Parallel()
+
+ // The router and postRefusedForm derive their CSRF key from
+ // testSigningKey, so cookiesA is valid there and only tokenB is wrong.
+ _, router := newCSRFTestRouter(t)
+ cookiesA, _ := csrfCredentials(t, router, nil)
+ _, tokenB := csrfCredentials(t, router, nil)
+
+ rec, logLine := postRefusedForm(t, true, "http://example.com", "",
+ cookiesA, tokenB)
+ checkFormRefused(t, rec, logLine, causeStaleToken, csrf.ErrBadToken)
+}
+
+// postRefusedForm sends a POST with the given cookies to the CSRF
+// middleware built for debug. It adds the form token and the Origin and
+// Referer headers when they are not empty. It returns the response and the
+// one JSON log line the middleware wrote.
+func postRefusedForm(
+ t *testing.T, debug bool, origin, referer string,
+ cookies []*http.Cookie, token string,
+) (*httptest.ResponseRecorder, map[string]any) {
+ t.Helper()
+
+ keys, err := seal.DeriveKeys(testSigningKey)
+ if err != nil {
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ var logs bytes.Buffer
+
+ log := slog.New(slog.NewJSONHandler(&logs, nil))
+ reached := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+ t.Error("the refused form reached the handler behind the middleware")
+ })
+ srv := newCSRFProtect(keys.CSRFKey, debug, log)(reached)
+
+ form := url.Values{loginKeyField: {"x"}}
+ if token != "" {
+ form.Set(csrfTokenField, token)
+ }
+
+ ctx := context.WithValue(context.Background(),
+ middleware.RequestIDKey, refusedFormRequestID)
+ req := httptest.NewRequestWithContext(ctx, http.MethodPost,
+ "http://example.com/", strings.NewReader(form.Encode()))
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+ for _, c := range cookies {
+ req.AddCookie(c)
+ }
+
+ if origin != "" {
+ req.Header.Set("Origin", origin)
+ }
+
+ if referer != "" {
+ req.Header.Set("Referer", referer)
+ }
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+
+ var logLine map[string]any
+
+ err = json.Unmarshal(logs.Bytes(), &logLine)
+ if err != nil {
+ t.Fatalf("decoding log line %q: %v", logs.String(), err)
+ }
+
+ return rec, logLine
+}
+
+// checkFormRefused checks that rec is a 403 page naming wantCause and that
+// logLine is a warning carrying the request ID and wantReason.
+func checkFormRefused(
+ t *testing.T, rec *httptest.ResponseRecorder, logLine map[string]any,
+ wantCause string, wantReason error,
+) {
+ t.Helper()
+
+ if rec.Code != http.StatusForbidden {
+ t.Errorf("status = %d, want %d", rec.Code, http.StatusForbidden)
+ }
+
+ body := rec.Body.String()
+ t.Logf("body: %s", body)
+
+ if !strings.Contains(body, template.HTMLEscapeString(wantCause)) {
+ t.Errorf("page does not name the cause %q", wantCause)
+ }
+
+ t.Logf("log line: %v", logLine)
+
+ if logLine["level"] != "WARN" || logLine["msg"] != "form refused" {
+ t.Errorf("log line = %v, want a WARN \"form refused\"", logLine)
+ }
+
+ if logLine["request_id"] != refusedFormRequestID {
+ t.Errorf("logged request_id = %v, want %q",
+ logLine["request_id"], refusedFormRequestID)
+ }
+
+ if logLine["reason"] != wantReason.Error() {
+ t.Errorf("logged reason = %v, want %q", logLine["reason"], wantReason)
+ }
+}
diff --git a/internal/handlers/first_bytes_internal_test.go b/internal/handlers/first_bytes_internal_test.go
new file mode 100644
index 0000000..3125edc
--- /dev/null
+++ b/internal/handlers/first_bytes_internal_test.go
@@ -0,0 +1,203 @@
+package handlers
+
+import (
+ "bytes"
+ "encoding/json"
+ "image/color"
+ "log/slog"
+ "net/http"
+ "slices"
+ "testing"
+ "testing/fstest"
+ "time"
+
+ "github.com/go-chi/chi/v5"
+ "github.com/prometheus/client_golang/prometheus"
+ "sneak.berlin/go/pixa/internal/allowlist"
+ "sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/httpfetcher"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imageproxy"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/seal"
+ "sneak.berlin/go/pixa/internal/signature"
+)
+
+// mismatchedSourcePath is the path of the source that
+// TestFirstBytesNotMatchingContentType serves on allowlistedHost. The mock
+// fetcher names a file's Content-Type from its extension, so the source is
+// sent as image/png.
+const mismatchedSourcePath = "/images/source.png"
+
+// newRoutesFetchingThrough returns both image routes, and the Handlers behind
+// them, whose image service fetches through fetcher, and which have
+// allowlistedHost on their allowlist. The handlers and the image service log
+// to log.
+func newRoutesFetchingThrough(
+ t *testing.T, fetcher httpfetcher.Fetcher, log *slog.Logger,
+) (*Handlers, http.Handler) {
+ t.Helper()
+
+ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
+ StateDir: t.TempDir(),
+ NegativeTTL: 5 * time.Minute,
+ MaxBytes: 1 << 30,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("imgcache.NewCache() error = %v", err)
+ }
+
+ svc, err := imageproxy.New(&imageproxy.Config{
+ Cache: cache,
+ Fetcher: fetcher,
+ Processor: imageprocessor.New(imageprocessor.Params{
+ MaxInputBytes: 50 << 20,
+ MaxConcurrentProcessing: 2,
+ }),
+ MaxResponseSize: 50 << 20,
+ Logger: log,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("imageproxy.New() error = %v", err)
+ }
+
+ keys, err := seal.DeriveKeys(testSigningKey)
+ if err != nil {
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ encGen := encurl.NewGenerator(keys.EncryptedURLKey)
+
+ h := &Handlers{
+ log: log,
+ imgSvc: svc,
+ encGen: encGen,
+ signer: signature.New(testSigningKey),
+ allowlist: allowlist.New([]string{allowlistedHost}),
+ }
+
+ r := chi.NewRouter()
+ r.Get("/v1/image/*", h.HandleImage())
+ r.Get("/v1/e/{token}/*", h.HandleImageEnc())
+
+ return h, r
+}
+
+// mismatchedSourcePlainURL returns the /v1/image/ URL of mismatchedSourcePath
+// on allowlistedHost at 50x50 as a JPEG.
+func mismatchedSourcePlainURL(*testing.T, *Handlers) string {
+ return "/v1/image/" + allowlistedHost + mismatchedSourcePath + "/50x50.jpeg"
+}
+
+// mismatchedSourceEncURL returns an encrypted /v1/e/ URL, made with h's
+// generator, of mismatchedSourcePath on allowlistedHost at 50x50 as a JPEG.
+func mismatchedSourceEncURL(t *testing.T, h *Handlers) string {
+ t.Helper()
+
+ token, err := h.encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: allowlistedHost,
+ Path: mismatchedSourcePath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ },
+ }, time.Time{})
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ return "/v1/e/" + token + "/img.jpg"
+}
+
+// loggedLevels reads the JSON log lines in logs and returns the messages
+// logged at each level.
+func loggedLevels(t *testing.T, logs *bytes.Buffer) map[string][]string {
+ t.Helper()
+
+ t.Logf("logged:\n%s", logs.String())
+
+ messages := make(map[string][]string)
+
+ dec := json.NewDecoder(logs)
+ for dec.More() {
+ var line struct {
+ Level string `json:"level"`
+ Msg string `json:"msg"`
+ }
+
+ err := dec.Decode(&line)
+ if err != nil {
+ t.Fatalf("decoding log line: %v", err)
+ }
+
+ messages[line.Level] = append(messages[line.Level], line.Msg)
+ }
+
+ return messages
+}
+
+// TestFirstBytesNotMatchingContentType checks that both image routes answer
+// 415 for an upstream body whose first bytes are not those of the format its
+// Content-Type names, and log it as a refused source, not as an error of
+// pixa's. The next request for that source must answer 502 without fetching
+// it again, as after an upstream error status.
+func TestFirstBytesNotMatchingContentType(t *testing.T) {
+ t.Parallel()
+
+ bodies := map[string][]byte{
+ "JPEG sent as image/png": generateTestJPEG(t, 10, 10, color.Black),
+ "text sent as image/png": []byte("This is just text, not a PNG"),
+ "body too short to name a format": []byte("short"),
+ }
+
+ // The source's URL on each image route.
+ sourceURLs := map[string]func(*testing.T, *Handlers) string{
+ "plain URL": mismatchedSourcePlainURL,
+ "encrypted URL": mismatchedSourceEncURL,
+ }
+
+ for bodyName, body := range bodies {
+ for urlName, sourceURL := range sourceURLs {
+ t.Run(urlName+", "+bodyName, func(t *testing.T) {
+ t.Parallel()
+
+ fetcher := &countingFetcher{
+ Fetcher: httpfetcher.NewMock(fstest.MapFS{
+ allowlistedHost + mismatchedSourcePath: &fstest.MapFile{
+ Data: body,
+ },
+ }),
+ }
+
+ var logs bytes.Buffer
+
+ h, routes := newRoutesFetchingThrough(t, fetcher,
+ slog.New(slog.NewJSONHandler(&logs, nil)))
+ target := sourceURL(t, h)
+
+ checkErrorBody(t, sendGet(t, routes, target),
+ http.StatusUnsupportedMediaType, "unsupported source format")
+
+ levels := loggedLevels(t, &logs)
+ if len(levels[slog.LevelError.String()]) != 0 {
+ t.Errorf("logged at error level: %q",
+ levels[slog.LevelError.String()])
+ }
+
+ if !slices.Contains(levels[slog.LevelWarn.String()], "source refused") {
+ t.Error(`no "source refused" line logged at warning level`)
+ }
+
+ checkErrorBody(t, sendGet(t, routes, target),
+ http.StatusBadGateway, "upstream error")
+
+ if n := fetcher.fetches.Load(); n != 1 {
+ t.Errorf("upstream fetched %d times, want 1", n)
+ }
+ })
+ }
+ }
+}
diff --git a/internal/handlers/format_auto.go b/internal/handlers/format_auto.go
new file mode 100644
index 0000000..c54bebd
--- /dev/null
+++ b/internal/handlers/format_auto.go
@@ -0,0 +1,134 @@
+package handlers
+
+import (
+ "errors"
+ "fmt"
+ "mime"
+ "net/http"
+ "strconv"
+ "strings"
+
+ "sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// Errors for an Accept header that an auto URL cannot be served for.
+var (
+ errInvalidAccept = errors.New("invalid Accept header")
+ errNotAcceptable = errors.New("not acceptable: auto serves " +
+ "image/jxl, image/avif, image/webp or image/jpeg")
+)
+
+// chooseAutoFormat returns the format to serve for the requested format: for
+// the format auto, the format formatForAccept chooses from r's Accept header,
+// and any other format as it is. For auto it adds Vary: Accept to the
+// response, which then depends on that header. It answers 400 for an Accept
+// header that is not valid and 406 for one that allows none of the formats,
+// and reports whether the image can be served.
+func (s *Handlers) chooseAutoFormat(
+ w http.ResponseWriter, r *http.Request, requested imagerequest.Format,
+) (imagerequest.Format, bool) {
+ if requested != imagerequest.FormatAuto {
+ return requested, true
+ }
+
+ w.Header().Add("Vary", "Accept")
+
+ format, err := formatForAccept(strings.Join(r.Header.Values("Accept"), ","))
+ if errors.Is(err, errNotAcceptable) {
+ s.respondError(w, err.Error(), http.StatusNotAcceptable)
+
+ return "", false
+ }
+
+ if err != nil {
+ s.respondError(w, err.Error(), http.StatusBadRequest)
+
+ return "", false
+ }
+
+ return format, true
+}
+
+// formatForAccept returns the format an auto URL is served in for the Accept
+// header accept: JPEG XL when it names image/jxl, else AVIF when it names
+// image/avif, else WebP when it names image/webp, else JPEG when its most
+// specific entry of image/jpeg, image/* and */* allows it, or when it names
+// nothing. A q of 0 refuses a format. JPEG XL, AVIF and WebP must be named, as
+// clients that cannot show them send image/* and */* too.
+func formatForAccept(accept string) (imagerequest.Format, error) {
+ qualities, err := parseAccept(accept)
+ if err != nil {
+ return "", err
+ }
+
+ if len(qualities) == 0 {
+ return imagerequest.FormatJPEG, nil
+ }
+
+ if qualities[imagerequest.FormatJXL.MIMEType()] > 0 {
+ return imagerequest.FormatJXL, nil
+ }
+
+ if qualities[imagerequest.FormatAVIF.MIMEType()] > 0 {
+ return imagerequest.FormatAVIF, nil
+ }
+
+ if qualities[imagerequest.FormatWebP.MIMEType()] > 0 {
+ return imagerequest.FormatWebP, nil
+ }
+
+ // For JPEG, the most specific entry the header has decides
+ quality, named := qualities[imagerequest.FormatJPEG.MIMEType()]
+ if !named {
+ quality, named = qualities["image/*"]
+ }
+
+ if !named {
+ quality = qualities["*/*"]
+ }
+
+ if quality > 0 {
+ return imagerequest.FormatJPEG, nil
+ }
+
+ return "", errNotAcceptable
+}
+
+// parseAccept returns the q of each media range the Accept header accept
+// names, 1 where it gives none. A media range named more than once keeps its
+// lowest q, so a refusal is never overridden. A media range that does not
+// parse, or a q that is not a number from 0 to 1, is an error.
+func parseAccept(accept string) (map[string]float64, error) {
+ qualities := make(map[string]float64)
+
+ for entry := range strings.SplitSeq(accept, ",") {
+ // A header field list may hold empty entries
+ if strings.TrimSpace(entry) == "" {
+ continue
+ }
+
+ mediaRange, params, err := mime.ParseMediaType(entry)
+ if err != nil {
+ return nil, fmt.Errorf("%w: %q: %w", errInvalidAccept, entry, err)
+ }
+
+ quality := 1.0
+
+ if qParam, given := params["q"]; given {
+ quality, err = strconv.ParseFloat(qParam, 64)
+ inRange := quality >= 0 && quality <= 1
+
+ if err != nil || !inRange {
+ return nil, fmt.Errorf("%w: %q: q is not a number from 0 to 1",
+ errInvalidAccept, entry)
+ }
+ }
+
+ previous, named := qualities[mediaRange]
+ if !named || quality < previous {
+ qualities[mediaRange] = quality
+ }
+ }
+
+ return qualities, nil
+}
diff --git a/internal/handlers/format_auto_internal_test.go b/internal/handlers/format_auto_internal_test.go
new file mode 100644
index 0000000..5a99f1e
--- /dev/null
+++ b/internal/handlers/format_auto_internal_test.go
@@ -0,0 +1,315 @@
+package handlers
+
+import (
+ "errors"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/go-chi/chi/v5"
+
+ "sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// The content types the tests below expect.
+const (
+ avifType = "image/avif"
+ webpType = "image/webp"
+ jpegType = "image/jpeg"
+ jsonType = "application/json"
+)
+
+// TestFormatForAccept verifies the format chosen for the format auto from each
+// Accept header below, and the error for one that allows none of AVIF, WebP
+// and JPEG or is not valid.
+func TestFormatForAccept(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ accept string
+ want imagerequest.Format
+ wantErr error
+ }{
+ {"AVIF-capable browser",
+ "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
+ imagerequest.FormatAVIF, nil},
+ {"WebP-capable browser",
+ "image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5",
+ imagerequest.FormatWebP, nil},
+ {"WebP only", webpType, imagerequest.FormatWebP, nil},
+ {"neither", "image/png,image/*;q=0.8,*/*;q=0.5", imagerequest.FormatJPEG, nil},
+ {"wildcard only", "*/*", imagerequest.FormatJPEG, nil},
+ {"image wildcard only", "image/*", imagerequest.FormatJPEG, nil},
+ {"absent", "", imagerequest.FormatJPEG, nil},
+ {"q=0 on AVIF", "image/avif;q=0,image/webp,*/*", imagerequest.FormatWebP, nil},
+ {"AVIF named twice, once with q=0", "image/avif,image/avif;q=0.0,*/*",
+ imagerequest.FormatJPEG, nil},
+ {"upper case and spaces", " Image/AVIF ; Q=0.5 ", imagerequest.FormatAVIF, nil},
+ {"q=0 on JPEG", "image/jpeg;q=0,image/*", "", errNotAcceptable},
+ {"q=0 on everything", "*/*;q=0", "", errNotAcceptable},
+ {"PNG only", "image/png", "", errNotAcceptable},
+ {"malformed media range", "image/", "", errInvalidAccept},
+ {"q not a number", "image/avif;q=high", "", errInvalidAccept},
+ {"q above 1", "image/avif;q=2", "", errInvalidAccept},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ got, err := formatForAccept(tt.accept)
+ t.Logf("Accept %q: %q, %v", tt.accept, got, err)
+
+ if got != tt.want || !errors.Is(err, tt.wantErr) {
+ t.Errorf("formatForAccept(%q) = %q, %v, want %q, %v",
+ tt.accept, got, err, tt.want, tt.wantErr)
+ }
+ })
+ }
+}
+
+// autoPhotoURLs returns a signed /v1/image/ URL and an encrypted /v1/e/ URL,
+// both valid for a minute, for the JPEG at photoPath on signedHost at 50x50 in
+// the format auto. The first is signed with testSigningKey, the key the
+// handlers that newSignedHostServer builds check signatures with. The second
+// is made with h's generator.
+func autoPhotoURLs(t *testing.T, h *Handlers) (string, string) {
+ t.Helper()
+
+ signedURL := generateSignedURL(testSigningKey, &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatAuto,
+ },
+ }, time.Minute)
+
+ token, err := h.encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatAuto,
+ },
+ }, time.Now().Add(time.Minute))
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ return signedURL, "/v1/e/" + token + "/img.jpg"
+}
+
+// requestImage sends method for target to srv with an Accept header line for
+// each of accept, and returns the response.
+func requestImage(
+ t *testing.T, srv http.Handler, method, target string, accept ...string,
+) *httptest.ResponseRecorder {
+ t.Helper()
+
+ req := httptest.NewRequestWithContext(t.Context(), method, target, nil)
+
+ for _, value := range accept {
+ req.Header.Add("Accept", value)
+ }
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+ t.Logf("%s %s with Accept %q: %d, Content-Type %s, Vary %v, X-Pixa-Cache %s",
+ method, target, accept, rec.Code, rec.Header().Get("Content-Type"),
+ rec.Header().Values("Vary"), rec.Header().Get("X-Pixa-Cache"))
+
+ return rec
+}
+
+// TestFormatAuto_ChosenFromAccept requests an auto URL on each image route
+// with each Accept below, and checks the answer and that it carries
+// Vary: Accept. The signed URL is signed for auto, so it is valid whatever
+// Accept chooses.
+func TestFormatAuto_ChosenFromAccept(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+ signedURL, encryptedURL := autoPhotoURLs(t, h)
+
+ tests := []struct {
+ name string
+ accept []string
+ wantStatus int
+ wantType string
+ }{
+ {"AVIF accepted", []string{"image/avif,image/webp,*/*;q=0.8"},
+ http.StatusOK, avifType},
+ {"WebP accepted", []string{"image/webp,*/*;q=0.8"}, http.StatusOK, webpType},
+ {"no Accept", nil, http.StatusOK, jpegType},
+ {"two Accept lines", []string{"image/png", webpType}, http.StatusOK, webpType},
+ {"none of the three", []string{"image/gif"},
+ http.StatusNotAcceptable, jsonType},
+ {"not valid", []string{"image/avif;q=high"}, http.StatusBadRequest, jsonType},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ for _, target := range []string{signedURL, encryptedURL} {
+ rec := requestImage(t, srv, http.MethodGet, target, tt.accept...)
+ gotType := rec.Header().Get("Content-Type")
+
+ if rec.Code != tt.wantStatus || gotType != tt.wantType {
+ t.Errorf("%s: %d %s, want %d %s; body %s", target,
+ rec.Code, gotType, tt.wantStatus, tt.wantType, rec.Body)
+ }
+
+ if !slices.Contains(rec.Header().Values("Vary"), "Accept") {
+ t.Errorf("%s: Vary = %v, want Accept in it",
+ target, rec.Header().Values("Vary"))
+ }
+ }
+ })
+ }
+}
+
+// TestFormatAuto_SignatureCoversAuto verifies that a /v1/image/ URL with the
+// format auto is checked against a signature for auto, not for the format
+// chosen: a URL signed for avif, with auto put in its path, is refused for a
+// client whose Accept chooses AVIF.
+func TestFormatAuto_SignatureCoversAuto(t *testing.T) {
+ t.Parallel()
+
+ _, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ signedForAVIF := generateSignedURL(testSigningKey, &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatAVIF,
+ },
+ }, time.Minute)
+
+ target := strings.Replace(signedForAVIF, "/50x50.avif?", "/50x50.auto?", 1)
+ if target == signedForAVIF {
+ t.Fatalf("no /50x50.avif? in %s", signedForAVIF)
+ }
+
+ rec := requestImage(t, srv, http.MethodGet, target, avifType)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("status = %d, want %d", rec.Code, http.StatusUnauthorized)
+ }
+}
+
+// TestFormatAuto_CachesEachFormatApart requests an auto URL for AVIF, then
+// JPEG, then both again. Each format is processed once and then served from
+// the cache, with an ETag of its own, so a client never gets the other format
+// from the cache.
+func TestFormatAuto_CachesEachFormatApart(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+ signedURL, _ := autoPhotoURLs(t, h)
+
+ steps := []struct {
+ wantType string
+ wantCache string
+ }{
+ {avifType, "MISS"},
+ {jpegType, "MISS"},
+ {avifType, "HIT"},
+ {jpegType, "HIT"},
+ }
+
+ etags := make(map[string]string)
+
+ for _, step := range steps {
+ rec := requestImage(t, srv, http.MethodGet, signedURL, step.wantType)
+ gotType := rec.Header().Get("Content-Type")
+ gotCache := rec.Header().Get("X-Pixa-Cache")
+
+ if rec.Code != http.StatusOK || gotType != step.wantType ||
+ gotCache != step.wantCache {
+ t.Fatalf("Accept %s: %d %s %s, want 200 %s %s", step.wantType,
+ rec.Code, gotType, gotCache, step.wantType, step.wantCache)
+ }
+
+ etag := rec.Header().Get("ETag")
+ if previous, seen := etags[gotType]; seen && previous != etag {
+ t.Errorf("%s ETag changed from %s to %s", gotType, previous, etag)
+ }
+
+ etags[gotType] = etag
+ }
+
+ if etags[avifType] == etags[jpegType] {
+ t.Errorf("AVIF and JPEG have the same ETag %s", etags[avifType])
+ }
+}
+
+// TestFormatAuto_Vary verifies that on each image route a HEAD answer and a
+// 304 for an auto URL carry Vary: Accept, and that the answer for a URL with
+// a fixed format does not.
+func TestFormatAuto_Vary(t *testing.T) {
+ t.Parallel()
+
+ h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ srv := chi.NewRouter()
+ srv.Get("/v1/image/*", h.HandleImage())
+ srv.Head("/v1/image/*", h.HandleImage())
+ srv.Get("/v1/e/{token}/*", h.HandleImageEnc())
+ srv.Head("/v1/e/{token}/*", h.HandleImageEnc())
+
+ signedURL, encryptedURL := autoPhotoURLs(t, h)
+
+ for _, urls := range [][2]string{
+ {signedURL, signedPhotoURL(t, h)},
+ {encryptedURL, encPhotoURL(t, h)},
+ } {
+ autoURL, fixedURL := urls[0], urls[1]
+
+ head := requestImage(t, srv, http.MethodHead, autoURL, webpType)
+ checkVaryAccept(t, head, http.StatusOK, true)
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
+ autoURL, nil)
+ req.Header.Set("Accept", webpType)
+ req.Header.Set("If-None-Match", head.Header().Get("ETag"))
+
+ notModified := httptest.NewRecorder()
+ srv.ServeHTTP(notModified, req)
+ checkVaryAccept(t, notModified, http.StatusNotModified, true)
+
+ fixed := requestImage(t, srv, http.MethodGet, fixedURL)
+ checkVaryAccept(t, fixed, http.StatusOK, false)
+ }
+}
+
+// checkVaryAccept fails the test unless rec answered wantStatus and, as
+// wantVary says, has or has not Accept in its Vary header.
+func checkVaryAccept(
+ t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantVary bool,
+) {
+ t.Helper()
+
+ vary := rec.Header().Values("Vary")
+ t.Logf("status %d, Vary %v", rec.Code, vary)
+
+ if rec.Code != wantStatus {
+ t.Errorf("status = %d, want %d", rec.Code, wantStatus)
+ }
+
+ if slices.Contains(vary, "Accept") != wantVary {
+ t.Errorf("Vary = %v, want Accept in it: %v", vary, wantVary)
+ }
+}
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go
index 09f3ffc..6b68345 100644
--- a/internal/handlers/handlers.go
+++ b/internal/handlers/handlers.go
@@ -9,14 +9,15 @@ import (
"time"
"go.uber.org/fx"
+ "sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/config"
- "sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/healthcheck"
- "sneak.berlin/go/pixa/internal/httpfetcher"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imageproxy"
"sneak.berlin/go/pixa/internal/logger"
+ "sneak.berlin/go/pixa/internal/seal"
"sneak.berlin/go/pixa/internal/session"
+ "sneak.berlin/go/pixa/internal/signature"
)
// Params defines dependencies for Handlers.
@@ -25,129 +26,56 @@ type Params struct {
Logger *logger.Logger
Healthcheck *healthcheck.Healthcheck
- Database *database.Database
Config *config.Config
+ Keys seal.Keys
+
+ ImageService *imageproxy.Proxy
+ SessionManager *session.Manager
+ EncryptedURLGenerator *encurl.Generator
}
// Handlers provides HTTP request handlers.
type Handlers struct {
log *slog.Logger
hc *healthcheck.Healthcheck
- db *database.Database
config *config.Config
- imgSvc *imgcache.Service
- imgCache *imgcache.Cache
+ imgSvc *imageproxy.Proxy
sessMgr *session.Manager
encGen *encurl.Generator
csrfProtect func(http.Handler) http.Handler
+
+ // signer checks the signature of a /v1/image/ URL with signing_key.
+ signer *signature.Signer
+ // allowlist matches the hosts of allowlist_hosts, which a /v1/image/
+ // URL may name without a signature.
+ allowlist *allowlist.HostAllowList
+ // refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
+ // reports whether a URL's host is on that list.
+ refererBlocklist *allowlist.HostAllowList
}
// New creates a new Handlers instance.
-func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
- csrfProtect, err := newCSRFProtect(params.Config.SigningKey, params.Config.Debug)
- if err != nil {
- return nil, err
+func New(_ fx.Lifecycle, params Params) *Handlers {
+ log := params.Logger.Get()
+
+ return &Handlers{
+ log: log,
+ hc: params.Healthcheck,
+ config: params.Config,
+ imgSvc: params.ImageService,
+ sessMgr: params.SessionManager,
+ encGen: params.EncryptedURLGenerator,
+ csrfProtect: newCSRFProtect(params.Keys.CSRFKey, params.Config.Debug, log),
+ signer: signature.New(params.Config.SigningKey),
+ allowlist: allowlist.New(params.Config.AllowlistHosts),
+ refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
}
-
- s := &Handlers{
- log: params.Logger.Get(),
- hc: params.Healthcheck,
- db: params.Database,
- config: params.Config,
- csrfProtect: csrfProtect,
- }
-
- lc.Append(fx.Hook{
- // The eviction goroutine must outlive OnStart, so it cannot
- // inherit this hook's context. It makes its own instead, which
- // leaves it uncancellable: an in-flight pass runs to completion
- // during OnStop regardless of the shutdown deadline. Making the
- // loop cancellable changes shutdown semantics and is tracked
- // separately in issue #102, rather than being folded into the
- // lint-conformance change that surfaced it.
- //nolint:contextcheck // see issue #102
- OnStart: func(_ context.Context) error {
- return s.initImageService()
- },
- OnStop: func(_ context.Context) error {
- if s.imgCache != nil {
- s.imgCache.StopEviction()
- }
-
- return nil
- },
- })
-
- return s, nil
}
-// initImageService initializes the image cache and service.
-func (s *Handlers) initImageService() error {
- // Create the cache. cache_max_bytes: 0 disables the disk cache
- // entirely; any other value is the eviction limit in bytes.
- cache, err := imgcache.NewCache(s.db.DB(), imgcache.CacheConfig{
- StateDir: s.config.StateDir,
- CacheTTL: imgcache.DefaultCacheTTL,
- NegativeTTL: imgcache.DefaultNegativeTTL,
- MaxBytes: s.config.CacheMaxBytes,
- DisableDiskCache: s.config.CacheMaxBytes == 0,
- Logger: s.log,
- })
- if err != nil {
- return err
- }
-
- s.imgCache = cache
-
- // Background eviction: startup reconciliation, then periodic and
- // write-pressure passes. No-op when the disk cache is disabled.
- cache.StartEviction(imgcache.DefaultEvictionInterval)
-
- // Create the fetcher config
- fetcherCfg := httpfetcher.DefaultConfig()
- fetcherCfg.AllowHTTP = s.config.AllowHTTP
-
- if s.config.UpstreamConnectionsPerHost > 0 {
- fetcherCfg.MaxConnectionsPerHost = s.config.UpstreamConnectionsPerHost
- }
-
- fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
-
- // Create the service
- svc, err := imgcache.NewService(&imgcache.ServiceConfig{
- Cache: cache,
- FetcherConfig: fetcherCfg,
- SigningKey: s.config.SigningKey,
- Allowlist: s.config.AllowlistHosts,
- Logger: s.log,
- })
- if err != nil {
- return err
- }
-
- s.imgSvc = svc
- s.log.Info("image service initialized")
-
- // Initialize session manager (signing key is validated at config load
- // time). Session cookies are always Secure/HttpOnly/SameSite=Strict.
- sessMgr, err := session.NewManager(s.config.SigningKey)
- if err != nil {
- return err
- }
-
- s.sessMgr = sessMgr
-
- // Initialize encrypted URL generator
- encGen, err := encurl.NewGenerator(s.config.SigningKey)
- if err != nil {
- return err
- }
-
- s.encGen = encGen
-
- s.log.Info("session manager and URL generator initialized")
-
- return nil
+// WaitForProcessing waits until no image is being processed, or until ctx
+// ends, and returns how many images were still being processed then.
+func (s *Handlers) WaitForProcessing(ctx context.Context) int {
+ return s.imgSvc.WaitForProcessing(ctx)
}
func (s *Handlers) respondJSON(w http.ResponseWriter, data any, status int) {
diff --git a/internal/handlers/handlers_internal_test.go b/internal/handlers/handlers_internal_test.go
index 35c1522..0be15a7 100644
--- a/internal/handlers/handlers_internal_test.go
+++ b/internal/handlers/handlers_internal_test.go
@@ -18,15 +18,20 @@ import (
"time"
"github.com/go-chi/chi/v5"
+ "github.com/prometheus/client_golang/prometheus"
+ "sneak.berlin/go/pixa/internal/allowlist"
"sneak.berlin/go/pixa/internal/database"
"sneak.berlin/go/pixa/internal/httpfetcher"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imageproxy"
+ "sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/signature"
)
// testFixtures contains test data.
type testFixtures struct {
handler *Handlers
- service *imgcache.Service
goodHost string
}
@@ -45,33 +50,37 @@ func setupTestHandler(t *testing.T) *testFixtures {
tmpDir := t.TempDir()
db := setupTestDB(t)
- cache, err := imgcache.NewCache(db, imgcache.CacheConfig{
+ cache, err := imgcache.NewCache(db, imgcache.Config{
StateDir: tmpDir,
- CacheTTL: time.Hour,
NegativeTTL: 5 * time.Minute,
- })
+ MaxBytes: 1 << 30,
+ }, prometheus.NewRegistry())
if err != nil {
t.Fatalf("failed to create cache: %v", err)
}
- svc, err := imgcache.NewService(&imgcache.ServiceConfig{
- Cache: cache,
- Fetcher: newMockFetcher(mockFS),
- SigningKey: "test-signing-key-must-be-32-chars",
- Allowlist: []string{goodHost},
- })
+ svc, err := imageproxy.New(&imageproxy.Config{
+ Cache: cache,
+ Fetcher: newMockFetcher(mockFS),
+ Processor: imageprocessor.New(imageprocessor.Params{
+ MaxInputBytes: 50 << 20,
+ MaxConcurrentProcessing: 2,
+ }),
+ MaxResponseSize: 50 << 20,
+ }, prometheus.NewRegistry())
if err != nil {
- t.Fatalf("failed to create service: %v", err)
+ t.Fatalf("failed to create proxy: %v", err)
}
h := &Handlers{
- imgSvc: svc,
- log: slog.Default(),
+ imgSvc: svc,
+ log: slog.Default(),
+ signer: signature.New("test-signing-key-must-be-32-chars"),
+ allowlist: allowlist.New([]string{goodHost}),
}
return &testFixtures{
handler: h,
- service: svc,
goodHost: goodHost,
}
}
@@ -122,10 +131,9 @@ func newMockFetcher(fs fs.FS) *mockFetcher {
}
func (f *mockFetcher) Fetch(
- _ context.Context, url string,
+ _ context.Context, source imagerequest.Source, _, _ string,
) (*httpfetcher.FetchResult, error) {
- // Remove https:// prefix
- path := url[8:] // Remove "https://"
+ path := source.Host + source.Path
data, err := fs.ReadFile(f.fs, path)
if err != nil {
@@ -249,6 +257,90 @@ func TestHandleImage_ConditionalRequest_IfNoneMatch_DifferentETag(t *testing.T)
}
}
+// TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage serves an
+// image, changes it upstream, and requests it again with the ETag first
+// served, once the cache TTL has passed. The changed image must be served,
+// not 304 Not Modified.
+func TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage(t *testing.T) {
+ t.Parallel()
+
+ const host = "goodhost.example.com"
+
+ upstream := fstest.MapFS{
+ host + photoPath: &fstest.MapFile{
+ Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
+ },
+ }
+
+ // A cache TTL of one nanosecond has passed by the second request
+ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
+ StateDir: t.TempDir(),
+ NegativeTTL: 5 * time.Minute,
+ CacheTTL: time.Nanosecond,
+ MaxBytes: 1 << 30,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("imgcache.NewCache() error = %v", err)
+ }
+
+ svc, err := imageproxy.New(&imageproxy.Config{
+ Cache: cache,
+ Fetcher: newMockFetcher(upstream),
+ Processor: imageprocessor.New(imageprocessor.Params{
+ MaxInputBytes: 50 << 20,
+ MaxConcurrentProcessing: 2,
+ }),
+ MaxResponseSize: 50 << 20,
+ Logger: slog.New(slog.DiscardHandler),
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("imageproxy.New() error = %v", err)
+ }
+
+ h := &Handlers{
+ imgSvc: svc,
+ log: slog.New(slog.DiscardHandler),
+ signer: signature.New(testSigningKey),
+ allowlist: allowlist.New([]string{host}),
+ }
+
+ r := chi.NewRouter()
+ r.Get("/v1/image/*", h.HandleImage())
+
+ target := "/v1/image/" + host + photoPath + "/50x50.jpeg"
+
+ first := httptest.NewRecorder()
+ r.ServeHTTP(first,
+ httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil))
+
+ etag := first.Header().Get("ETag")
+ if first.Code != http.StatusOK || etag == "" {
+ t.Fatalf("first request: status = %d, ETag = %q, want %d and an ETag",
+ first.Code, etag, http.StatusOK)
+ }
+
+ upstream[host+photoPath] = &fstest.MapFile{
+ Data: generateTestJPEG(t, 100, 100, color.RGBA{0, 0, 255, 255}),
+ }
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+ req.Header.Set("If-None-Match", etag)
+
+ second := httptest.NewRecorder()
+ r.ServeHTTP(second, req)
+ t.Logf("first ETag %s; with it after the change: status %d, ETag %s, %d bytes",
+ etag, second.Code, second.Header().Get("ETag"), second.Body.Len())
+
+ if second.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d with the changed image",
+ second.Code, http.StatusOK)
+ }
+
+ if bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
+ t.Error("the image served is the one made before the change")
+ }
+}
+
func TestHandleImage_ETagHeader(t *testing.T) {
t.Parallel()
diff --git a/internal/handlers/image.go b/internal/handlers/image.go
index 00b271e..792fb14 100644
--- a/internal/handlers/image.go
+++ b/internal/handlers/image.go
@@ -1,35 +1,51 @@
package handlers
import (
+ "context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
+ "strings"
"time"
- "github.com/go-chi/chi/v5"
+ "github.com/go-chi/chi/v5/middleware"
"sneak.berlin/go/pixa/internal/encurl"
"sneak.berlin/go/pixa/internal/httpfetcher"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imageproxy"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/magic"
)
+// statusClientClosedRequest is the status nginx calls "client closed
+// request". The image routes answer it when the client goes away before its
+// image is ready, unless its request is the one making the image. Nobody reads
+// it; it is for logs and metrics.
+const statusClientClosedRequest = 499
+
// HandleImage handles the main image proxy route:
-// /v1/image///x.
+// /v1/image///x., or with no format
+// /v1/image///x
func (s *Handlers) HandleImage() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
- req, ok := s.parseImageRequest(w, r)
+ if s.refuseBlockedReferer(w, r) {
+ return
+ }
+
+ req, sig, expires, ok := s.parseImageRequest(w, r)
if !ok {
return
}
// Validate signature if required
- err := s.imgSvc.ValidateRequest(req)
+ err := s.validateSignature(req, sig, expires)
if err != nil {
s.log.Warn("signature validation failed",
- "host", req.SourceHost,
- "path", req.SourcePath,
+ "host", req.Source.Host,
+ "path", req.Source.Path,
"error", err,
)
s.respondError(w, "unauthorized", http.StatusUnauthorized)
@@ -37,22 +53,7 @@ func (s *Handlers) HandleImage() http.HandlerFunc {
return
}
- // Get cache key for logging
- cacheKey := imgcache.CacheKey(req)
-
- // Get the image (from cache or fetch/process)
- startTime := time.Now()
-
- resp, err := s.imgSvc.Get(r.Context(), req)
- if err != nil {
- s.respondImageError(w, req, err)
-
- return
- }
-
- defer func() { _ = resp.Content.Close() }()
-
- s.writeImageResponse(w, r, req, resp, cacheKey, startTime)
+ s.serveImage(w, r, req, expires)
}
}
@@ -68,30 +69,28 @@ func (s *Handlers) HandleRobotsTxt() http.HandlerFunc {
}
}
-// parseImageRequest parses the wildcard path and query parameters into
-// an ImageRequest. On invalid input it writes an error response and
-// returns false.
+// parseImageRequest parses the image URL's path and query parameters into
+// an image request, and returns it with the URL's signature and expiry. On
+// invalid input it writes an error response and returns false.
func (s *Handlers) parseImageRequest(
w http.ResponseWriter, r *http.Request,
-) (*imgcache.ImageRequest, bool) {
- // Get the wildcard path from chi
- pathParam := chi.URLParam(r, "*")
-
- // Parse the URL path
- parsed, err := imgcache.ParseImagePath(pathParam)
+) (*imagerequest.Request, string, time.Time, bool) {
+ // The path is parsed decoded once, as r.URL.Path holds it, and not from
+ // chi's wildcard: chi routes on the path as the client wrote it whenever
+ // that is not Go's own encoding of it, so its wildcard would still hold
+ // the %3F before a source query when, say, a "/" in the query is written
+ // as %2F
+ req, err := imagerequest.ParseImageURL(r.URL.Path)
if err != nil {
s.log.Warn("failed to parse image URL",
- "path", pathParam,
+ "path", r.URL.Path,
"error", err,
)
s.respondError(w, "invalid image URL: "+err.Error(), http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
- // Convert to ImageRequest
- req := parsed.ToImageRequest()
-
// Parse signature params from query string. r.URL.Query() would silently
// drop a pair it cannot decode, such as q=80%, so that q would be served
// at 85; a query string that cannot be decoded is refused instead. A
@@ -102,7 +101,7 @@ func (s *Handlers) parseImageRequest(
s.respondError(w, fmt.Sprintf("invalid query string %q: %v",
r.URL.RawQuery, err), http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
for name, values := range query {
@@ -110,17 +109,15 @@ func (s *Handlers) parseImageRequest(
s.respondError(w, fmt.Sprintf("invalid %s: given more than once",
name), http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
}
- req.Signature = query.Get("sig")
-
- req.Expires, err = parseExpires(query)
+ expires, err := parseExpires(query)
if err != nil {
s.respondError(w, err.Error(), http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
// Parse optional quality and fit params. Only a q missing from the URL is
@@ -132,51 +129,76 @@ func (s *Handlers) parseImageRequest(
s.respondError(w, `invalid q: not a number, got ""`,
http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
- req.Quality, err = parseFormInt(query, "q",
+ req.Transform.Quality, err = parseFormInt(query, "q",
encurl.DefaultQuality, minQuality, maxQuality)
if err != nil {
s.respondError(w, fmt.Sprintf("%v, got %q", err, qStr),
http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
// Only a fit missing from the URL is cover. A fit in the URL that is not a
// fit mode is refused by the fit-mode check below; that check would take an
// empty fit as missing, so an empty one is refused here.
- req.FitMode = imgcache.FitMode(query.Get("fit"))
- if query.Has("fit") && req.FitMode == "" {
+ req.Transform.FitMode = imagerequest.Fit(query.Get("fit"))
+ if query.Has("fit") && req.Transform.FitMode == "" {
s.respondError(w, `invalid fit: not a fit mode, got ""`, http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
// Default fit mode if not set
- if req.FitMode == "" {
- req.FitMode = imgcache.FitCover
+ if req.Transform.FitMode == "" {
+ req.Transform.FitMode = imagerequest.FitCover
}
// Enforce dimension and fit-mode bounds, shared with the encrypted-URL
// route. Dimensions are already bounded by the path parser above; this
// also rejects an unrecognized fit mode with 400 instead of letting it
// reach the processor as a 500.
- err = imgcache.ValidateImageRequest(req)
+ err = imagerequest.ValidateImageRequest(req)
if err != nil {
s.respondError(w, "invalid image request: "+err.Error(),
http.StatusBadRequest)
- return nil, false
+ return nil, "", time.Time{}, false
}
- return req, true
+ return req, query.Get("sig"), expires, true
+}
+
+// validateSignature returns nil when the source host of req is on
+// allowlist_hosts and names no port. Otherwise it verifies sig, the URL's
+// signature of req, and expires, the URL's expiry.
+func (s *Handlers) validateSignature(
+ req *imagerequest.Request, sig string, expires time.Time,
+) error {
+ // An allowlist entry names no port, so a host followed by ":", with or
+ // without a port, needs a signature. Go's URL type reads the port as the
+ // fetcher does: in an IPv6 address written without brackets, such as
+ // 2001:db8::1:8443, it is what follows the last ":". This is checked here
+ // and not in allowlist.IsAllowed, which also matches referer_blocklist,
+ // where a Referer must be blocked whatever its port. Port and IsAllowed
+ // read only the host of a URL.
+ sourceURL := &url.URL{Host: req.Source.Host}
+ hostHasPort := sourceURL.Port() != "" ||
+ strings.HasSuffix(sourceURL.Host, ":")
+
+ if !hostHasPort && s.allowlist.IsAllowed(sourceURL) {
+ return nil
+ }
+
+ // Signature required for non-allowed hosts
+ return s.signer.Verify(req, sig, expires)
}
// parseExpires reads the exp query parameter, a Unix time in seconds. An exp
-// missing from the URL gives the zero time, which the signature check takes
-// as no expiration. An exp in the URL that is not a whole number, an empty
+// missing from the URL gives the zero time, which the signature check refuses
+// as a missing exp. An exp in the URL that is not a whole number, an empty
// one included, is an error naming exp and the value.
func parseExpires(query url.Values) (time.Time, error) {
if !query.Has("exp") {
@@ -194,61 +216,245 @@ func parseExpires(query url.Values) (time.Time, error) {
return time.Unix(exp, 0), nil
}
-// respondImageError maps image retrieval errors to HTTP responses.
-func (s *Handlers) respondImageError(
- w http.ResponseWriter, req *imgcache.ImageRequest, err error,
-) {
- s.log.Error("failed to get image",
- "host", req.SourceHost,
- "path", req.SourcePath,
- "error", err,
- )
-
- // Check for specific error types
- if errors.Is(err, httpfetcher.ErrSSRFBlocked) {
- s.respondError(w, "forbidden", http.StatusForbidden)
-
- return
+// HandleTooManyRequests answers 429 Too Many Requests with the JSON error
+// body the image routes send for their other errors. The image routes' rate
+// limit calls it for a request over the limit.
+func (s *Handlers) HandleTooManyRequests() http.HandlerFunc {
+ return func(w http.ResponseWriter, _ *http.Request) {
+ s.respondError(w, "too many requests, try again later",
+ http.StatusTooManyRequests)
}
-
- if errors.Is(err, httpfetcher.ErrUpstreamError) {
- s.respondError(w, "upstream error", http.StatusBadGateway)
-
- return
- }
-
- s.respondError(w, "internal error", http.StatusInternalServerError)
}
-// writeImageResponse writes headers and streams the image content,
-// handling conditional and HEAD requests.
-func (s *Handlers) writeImageResponse(
- w http.ResponseWriter, r *http.Request,
- req *imgcache.ImageRequest, resp *imgcache.ImageResponse,
- cacheKey imgcache.VariantKey, startTime time.Time,
+// respondImageError answers err, the image service's error for req, on both
+// image routes. It logs a 500 at error level, as pixa's own failure. It logs
+// the other answers at warn level, since the client, the upstream or a full
+// limit caused them. The 415 for a source refused for its first bytes is not
+// logged here, because the image service logs it.
+func (s *Handlers) respondImageError(
+ w http.ResponseWriter, req *imagerequest.Request, err error,
) {
+ // The image service has logged this as a refused source
+ if isFirstBytesMismatch(err) {
+ s.respondError(w, "unsupported source format",
+ http.StatusUnsupportedMediaType)
+
+ return
+ }
+
+ var (
+ status int
+ message string
+ )
+
+ switch {
+ case errors.Is(err, httpfetcher.ErrSSRFBlocked):
+ status, message = http.StatusForbidden, "forbidden"
+ case errors.Is(err, httpfetcher.ErrUpstreamError),
+ errors.Is(err, httpfetcher.ErrInvalidHost),
+ errors.Is(err, httpfetcher.ErrUpstreamRequestFailed):
+ status, message = http.StatusBadGateway, "upstream error"
+ case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
+ status, message = http.StatusGatewayTimeout, "upstream timeout"
+ case errors.Is(err, httpfetcher.ErrTooManyConnections),
+ errors.Is(err, imageprocessor.ErrTooManyImages),
+ errors.Is(err, imageproxy.ErrProcessingTimedOut):
+ status = http.StatusServiceUnavailable
+ message = "server busy, try again later"
+ case errors.Is(err, httpfetcher.ErrInvalidContentType),
+ errors.Is(err, imageprocessor.ErrUnsupportedInputFormat):
+ status = http.StatusUnsupportedMediaType
+ message = "unsupported source format"
+ case errors.Is(err, imageprocessor.ErrSVGTooThin):
+ status = http.StatusUnsupportedMediaType
+ message = fmt.Sprintf("SVG source too thin to render: one side is "+
+ "more than %d times the other", imageprocessor.MaxInputDimension)
+ case errors.Is(err, imageprocessor.ErrSVGRenderFailed):
+ status = http.StatusUnsupportedMediaType
+ message = fmt.Sprintf("SVG source could not be rendered: it is not "+
+ "valid, or it takes more than %d seconds or %d GiB of memory to "+
+ "render", int(imageprocessor.SVGRenderTimeout.Seconds()),
+ imageprocessor.MaxSVGRenderMemory/bytesPerGiB)
+ case errors.Is(err, imageprocessor.ErrOutputTooLarge):
+ status = http.StatusBadRequest
+ message = fmt.Sprintf("size too large for the source: the side "+
+ "computed from its proportions is above %d",
+ imageprocessor.MaxOutputDimension)
+ case errors.Is(err, imageprocessor.ErrSVGNotOriginalSize):
+ status = http.StatusBadRequest
+ message = "an SVG source is returned as SVG only at the orig size; " +
+ "ask for another format to resize it"
+ case errors.Is(err, imageprocessor.ErrSVGTooLargeForOrigSize):
+ status = http.StatusBadRequest
+ message = fmt.Sprintf("SVG source above %d on a side: give a size "+
+ "to render it at", imageprocessor.MaxOutputDimension)
+ case errors.Is(err, imageprocessor.ErrAVIFOutputTooLarge):
+ status = http.StatusBadRequest
+ message = fmt.Sprintf("size too large for AVIF: the output "+
+ "would be above %d pixels", imageprocessor.MaxAVIFOutputPixels)
+ default:
+ s.log.Error("failed to get image",
+ "host", req.Source.Host,
+ "path", req.Source.Path,
+ "error", err,
+ )
+ s.respondError(w, "internal error", http.StatusInternalServerError)
+
+ return
+ }
+
+ s.log.Warn("failed to get image",
+ "host", req.Source.Host,
+ "path", req.Source.Path,
+ "error", err,
+ )
+ s.respondError(w, message, status)
+}
+
+// bytesPerGiB is the number of bytes in a GiB.
+const bytesPerGiB = 1 << 30
+
+// isFirstBytesMismatch reports whether err refuses an upstream body whose
+// first bytes are not those of the format its Content-Type names. That is the
+// upstream's error, not pixa's.
+func isFirstBytesMismatch(err error) bool {
+ return errors.Is(err, magic.ErrMagicByteMismatch) ||
+ errors.Is(err, magic.ErrUnknownFormat) ||
+ errors.Is(err, magic.ErrNotEnoughData)
+}
+
+// cacheControl returns the Cache-Control header for an image served through a
+// URL that expires at expires, or never when expires is the zero time. A cache
+// may keep the image for a year, but not past the URL's expiry, after which
+// pixa refuses the URL. The seconds left are rounded down and never negative.
+// immutable only stops revalidation while the image is fresh, so it also ends
+// at the expiry.
+func cacheControl(expires time.Time) string {
+ const oneYear = 365 * 24 * time.Hour
+
+ maxAge := oneYear
+
+ if !expires.IsZero() {
+ maxAge = min(maxAge, max(time.Until(expires), 0))
+ }
+
+ return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
+}
+
+// svgContentSecurityPolicyValue is the Content-Security-Policy header value
+// of an SVG response, in place of the middleware's. A browser that opens the
+// SVG applies its styles and draws its embedded data: images, but runs none
+// of its scripts and loads nothing else. The styles need unsafe-inline, as an
+// SVG keeps them inline, in style elements and attributes.
+const svgContentSecurityPolicyValue = "default-src 'none'; " +
+ "style-src 'unsafe-inline'; " +
+ "img-src data:; " +
+ "frame-ancestors 'none'; " +
+ "sandbox"
+
+// setSVGContentSecurityPolicy sets the Content-Security-Policy in header to
+// svgContentSecurityPolicyValue when contentType is that of SVG. Any other
+// response keeps the middleware's policy.
+func setSVGContentSecurityPolicy(header http.Header, contentType string) {
+ if contentType == imagerequest.FormatSVG.MIMEType() {
+ header.Set("Content-Security-Policy", svgContentSecurityPolicyValue)
+ }
+}
+
+// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
+// names a host on referer_blocklist, and reports whether it answered. A request
+// with no Referer, or one that does not parse as a URL with a host, is not
+// refused.
+func (s *Handlers) refuseBlockedReferer(
+ w http.ResponseWriter, r *http.Request,
+) bool {
+ referer, err := url.Parse(r.Referer())
+ if err != nil || !s.refererBlocklist.IsAllowed(referer) {
+ return false
+ }
+
+ s.respondError(w, "referer blocked", http.StatusForbidden)
+
+ return true
+}
+
+// notModified sets the ETag header to etag and, when the request's
+// If-None-Match is that ETag, answers 304 Not Modified. It reports whether it
+// answered. An empty etag sets no header and never answers.
+func notModified(w http.ResponseWriter, r *http.Request, etag string) bool {
+ if etag == "" {
+ return false
+ }
+
+ w.Header().Set("ETag", etag)
+
+ if r.Header.Get("If-None-Match") != etag {
+ return false
+ }
+
+ w.WriteHeader(http.StatusNotModified)
+
+ return true
+}
+
+// serveImage serves the image that req asks for. Both image routes call it
+// once they have the request and its URL's expiry, which ends the
+// Cache-Control max-age. It chooses the format for the format auto, answers
+// If-None-Match with 304 and HEAD with the headers only. It answers 499 when
+// the image service stopped because the client went away, and any other error
+// of the image service with respondImageError.
+func (s *Handlers) serveImage(
+ w http.ResponseWriter, r *http.Request,
+ req *imagerequest.Request, expires time.Time,
+) {
+ // The signature or the token covers the format auto, not the format chosen
+ format, ok := s.chooseAutoFormat(w, r, req.Transform.Format)
+ if !ok {
+ return
+ }
+
+ req.Transform.Format = format
+
+ startTime := time.Now()
+
+ resp, err := s.imgSvc.Get(r.Context(), req)
+
+ // The client went away before its image was ready, which is not pixa's
+ // failure. A cancellation while the request's context is live is pixa's,
+ // and respondImageError answers it 500.
+ if errors.Is(err, context.Canceled) && r.Context().Err() != nil {
+ s.log.Info("client went away before the image was ready",
+ "request_id", middleware.GetReqID(r.Context()),
+ "host", req.Source.Host,
+ "path", req.Source.Path,
+ )
+ w.WriteHeader(statusClientClosedRequest)
+
+ return
+ }
+
+ if err != nil {
+ s.respondImageError(w, req, err)
+
+ return
+ }
+
+ defer func() { _ = resp.Content.Close() }()
+
// Set response headers
w.Header().Set("Content-Type", resp.ContentType)
+ setSVGContentSecurityPolicy(w.Header(), resp.ContentType)
if resp.ContentLength > 0 {
w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
}
// Cache control headers
- w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
+ w.Header().Set("Cache-Control", cacheControl(expires))
w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
- if resp.ETag != "" {
- w.Header().Set("ETag", resp.ETag)
-
- // Check for conditional request (If-None-Match)
- if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
- if ifNoneMatch == resp.ETag {
- w.WriteHeader(http.StatusNotModified)
-
- return
- }
- }
+ if notModified(w, r, resp.ETag) {
+ return
}
// Handle HEAD request - return headers only
@@ -261,20 +467,31 @@ func (s *Handlers) writeImageResponse(
// Stream the response
w.WriteHeader(http.StatusOK)
+ // A failed write is most often a client that has gone away, not pixa's
+ // failure, so it is logged at warn level
servedBytes, err := io.Copy(w, resp.Content)
if err != nil {
- s.log.Error("failed to write response",
+ s.log.Warn("failed to write response",
+ "request_id", middleware.GetReqID(r.Context()),
+ "host", req.Source.Host,
+ "path", req.Source.Path,
+ "served_bytes", servedBytes,
"error", err,
)
+
+ return
}
// Log cache status and timing after serving
duration := time.Since(startTime)
s.log.Info("image served",
- "cache_key", cacheKey,
+ "request_id", middleware.GetReqID(r.Context()),
+ "cache_key", imagerequest.CacheKey(req),
+ "host", req.Source.Host,
+ "path", req.Source.Path,
"cache_status", resp.CacheStatus,
"duration_ms", duration.Milliseconds(),
- "format", req.Format,
+ "format", req.Transform.Format,
"served_bytes", servedBytes,
"fetched_bytes", resp.FetchedBytes,
)
diff --git a/internal/handlers/image_cache_control_internal_test.go b/internal/handlers/image_cache_control_internal_test.go
new file mode 100644
index 0000000..c056719
--- /dev/null
+++ b/internal/handlers/image_cache_control_internal_test.go
@@ -0,0 +1,232 @@
+package handlers
+
+import (
+ "image/color"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "testing"
+ "testing/fstest"
+ "time"
+
+ "github.com/go-chi/chi/v5"
+ "github.com/prometheus/client_golang/prometheus"
+
+ "sneak.berlin/go/pixa/internal/allowlist"
+ "sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imageproxy"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/seal"
+ "sneak.berlin/go/pixa/internal/signature"
+)
+
+// photoPath is the path of the JPEG that newSignedHostServer serves.
+const photoPath = "/images/photo.jpg"
+
+// newSignedHostServer returns a router for both image routes, and the Handlers
+// behind it, whose fetcher serves a JPEG at photoPath on signedHost. signedHost
+// is not on the allowlist, so a /v1/image/ URL for it is served only with a
+// valid signature. The handlers and the image service log to log.
+func newSignedHostServer(
+ t *testing.T, log *slog.Logger,
+) (*Handlers, http.Handler) {
+ t.Helper()
+
+ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
+ StateDir: t.TempDir(),
+ NegativeTTL: 5 * time.Minute,
+ MaxBytes: 1 << 30,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("imgcache.NewCache() error = %v", err)
+ }
+
+ jpegData := generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255})
+
+ svc, err := imageproxy.New(&imageproxy.Config{
+ Cache: cache,
+ Fetcher: newMockFetcher(fstest.MapFS{
+ signedHost + photoPath: &fstest.MapFile{Data: jpegData},
+ }),
+ Processor: imageprocessor.New(imageprocessor.Params{
+ MaxInputBytes: 50 << 20,
+ MaxConcurrentProcessing: 2,
+ }),
+ MaxResponseSize: 50 << 20,
+ Logger: log,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("imageproxy.New() error = %v", err)
+ }
+
+ keys, err := seal.DeriveKeys(testSigningKey)
+ if err != nil {
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ encGen := encurl.NewGenerator(keys.EncryptedURLKey)
+
+ h := &Handlers{
+ log: log,
+ imgSvc: svc,
+ encGen: encGen,
+ signer: signature.New(testSigningKey),
+ allowlist: allowlist.New(nil),
+ }
+
+ r := chi.NewRouter()
+ r.Get("/v1/image/*", h.HandleImage())
+ r.Get("/v1/e/{token}/*", h.HandleImageEnc())
+
+ return h, r
+}
+
+// getMaxAge sends a GET for target to srv, requires a 200, and returns the
+// max-age of the response's Cache-Control header, which must read
+// "public, max-age=, immutable".
+func getMaxAge(t *testing.T, srv http.Handler, target string) int {
+ t.Helper()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+ rec := httptest.NewRecorder()
+
+ srv.ServeHTTP(rec, req)
+
+ header := rec.Header().Get("Cache-Control")
+ t.Logf("GET %s: %d, Cache-Control: %s", target, rec.Code, header)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
+ }
+
+ value, hasPrefix := strings.CutPrefix(header, "public, max-age=")
+ value, hasSuffix := strings.CutSuffix(value, ", immutable")
+
+ maxAge, err := strconv.Atoi(value)
+ if !hasPrefix || !hasSuffix || err != nil {
+ t.Fatalf("Cache-Control = %q, want public, max-age=, immutable",
+ header)
+ }
+
+ return maxAge
+}
+
+// TestHandleImage_SignedURL_MaxAgeEndsAtExp verifies that an image served
+// through a signed URL expiring in 60 seconds may be cached for at most those
+// 60 seconds. The lower bound of 50 shows the max-age is the time left, not 0.
+func TestHandleImage_SignedURL_MaxAgeEndsAtExp(t *testing.T) {
+ t.Parallel()
+
+ _, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ signedURL := generateSignedURL(testSigningKey, &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ },
+ }, time.Minute)
+
+ maxAge := getMaxAge(t, srv, signedURL)
+ if maxAge < 50 || maxAge > 60 {
+ t.Errorf("max-age = %d, want 50 to 60", maxAge)
+ }
+}
+
+// TestHandleImage_AllowlistedHost_MaxAge verifies the max-age of an image from
+// an allowlisted host, which is served without checking sig or exp. A URL with
+// no exp may be cached for a year. A URL whose exp has passed is the one request
+// that reaches the header after its expiry, and must get 0, never less.
+func TestHandleImage_AllowlistedHost_MaxAge(t *testing.T) {
+ t.Parallel()
+
+ pastExp := strconv.FormatInt(time.Now().Add(-time.Hour).Unix(), 10)
+
+ tests := []struct {
+ name string
+ query string
+ wantMaxAge int
+ }{
+ {"no exp", "", 31536000},
+ {"exp already past", "?exp=" + pastExp, 0},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ fix := setupTestHandler(t)
+
+ r := chi.NewRouter()
+ r.Get("/v1/image/*", fix.handler.HandleImage())
+
+ maxAge := getMaxAge(t, r,
+ "/v1/image/"+fix.goodHost+"/images/photo.jpg/50x50.jpeg"+tt.query)
+ if maxAge != tt.wantMaxAge {
+ t.Errorf("max-age = %d, want %d", maxAge, tt.wantMaxAge)
+ }
+ })
+ }
+}
+
+// TestHandleImageEnc_MaxAge verifies that an image served through an encrypted
+// URL with a 60 second TTL may be cached for at most those 60 seconds, that one
+// with a two-year TTL may be cached for a year, and that one made without a
+// TTL, which never expires, may be cached for a year.
+func TestHandleImageEnc_MaxAge(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ ttl time.Duration
+ wantAtLeast int
+ wantAtMost int
+ }{
+ {"60 second TTL", time.Minute, 50, 60},
+ {"two-year TTL", 2 * 365 * 24 * time.Hour, 31536000, 31536000},
+ {"no TTL", 0, 31536000, 31536000},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ // The expiry is set here, not in the table: a parallel subtest
+ // can wait behind other tests for more than the 10 seconds the
+ // 60 second case allows.
+ var expires time.Time
+ if tt.ttl != 0 {
+ expires = time.Now().Add(tt.ttl)
+ }
+
+ token, err := h.encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ },
+ }, expires)
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ maxAge := getMaxAge(t, srv, "/v1/e/"+token+"/img.jpg")
+ if maxAge < tt.wantAtLeast || maxAge > tt.wantAtMost {
+ t.Errorf("max-age = %d, want %d to %d",
+ maxAge, tt.wantAtLeast, tt.wantAtMost)
+ }
+ })
+ }
+}
diff --git a/internal/handlers/image_errors_internal_test.go b/internal/handlers/image_errors_internal_test.go
new file mode 100644
index 0000000..e398e15
--- /dev/null
+++ b/internal/handlers/image_errors_internal_test.go
@@ -0,0 +1,595 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "image/color"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "strings"
+ "syscall"
+ "testing"
+ "testing/fstest"
+ "time"
+
+ "github.com/go-chi/chi/v5"
+ "github.com/go-chi/chi/v5/middleware"
+ "github.com/prometheus/client_golang/prometheus"
+ "sneak.berlin/go/pixa/internal/allowlist"
+ "sneak.berlin/go/pixa/internal/httpfetcher"
+ "sneak.berlin/go/pixa/internal/imageprocessor"
+ "sneak.berlin/go/pixa/internal/imageproxy"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/signature"
+)
+
+// allowlistedHost is the only host on the allowlist of the image route
+// newImageRoute builds.
+const allowlistedHost = "allowed.example.com"
+
+// newImageRoute returns the image route of a Handlers whose service fetches
+// with fetcher, and which checks signatures with testSigningKey.
+func newImageRoute(t *testing.T, fetcher httpfetcher.Fetcher) http.Handler {
+ t.Helper()
+
+ cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
+ StateDir: t.TempDir(),
+ NegativeTTL: 5 * time.Minute,
+ MaxBytes: 1 << 30,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("failed to create cache: %v", err)
+ }
+
+ svc, err := imageproxy.New(&imageproxy.Config{
+ Cache: cache,
+ Fetcher: fetcher,
+ Processor: imageprocessor.New(imageprocessor.Params{
+ MaxInputBytes: 50 << 20,
+ MaxConcurrentProcessing: 2,
+ }),
+ MaxResponseSize: 50 << 20,
+ }, prometheus.NewRegistry())
+ if err != nil {
+ t.Fatalf("failed to create proxy: %v", err)
+ }
+
+ h := &Handlers{
+ imgSvc: svc,
+ log: slog.New(slog.DiscardHandler),
+ signer: signature.New(testSigningKey),
+ allowlist: allowlist.New([]string{allowlistedHost}),
+ }
+
+ r := chi.NewRouter()
+ r.Get("/v1/image/*", h.HandleImage())
+
+ return r
+}
+
+// newPhotoFetcher returns a mock fetcher that serves a JPEG at photoPath on
+// each of hosts, and answers any other source with an upstream error.
+func newPhotoFetcher(t *testing.T, hosts ...string) *httpfetcher.MockFetcher {
+ t.Helper()
+
+ photo := &fstest.MapFile{
+ Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
+ }
+
+ files := fstest.MapFS{}
+ for _, host := range hosts {
+ files[host+photoPath] = photo
+ }
+
+ return httpfetcher.NewMock(files)
+}
+
+// photoURL returns the image route URL of photoPath on host, as a 50x50 JPEG.
+func photoURL(host string) string {
+ return "/v1/image/" + host + photoPath + "/50x50.jpeg"
+}
+
+// photoURLWithSig returns photoURL(host) with sig and expires as its sig and
+// exp.
+func photoURLWithSig(host, sig string, expires time.Time) string {
+ return fmt.Sprintf("%s?sig=%s&exp=%d", photoURL(host), sig, expires.Unix())
+}
+
+// photoRequest returns the image request of photoURL(host) at the default
+// quality and fit.
+func photoRequest(host string) *imagerequest.Request {
+ return &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: host,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ Quality: 85,
+ FitMode: imagerequest.FitCover,
+ },
+ }
+}
+
+// photoSignature returns the signature of photoURL(host) at the default
+// quality and fit, made with key and expiring at expires.
+func photoSignature(key, host string, expires time.Time) string {
+ return signature.New(key).Sign(photoRequest(host), expires)
+}
+
+// sendGet sends a GET for target to route and returns the response.
+func sendGet(
+ t *testing.T, route http.Handler, target string,
+) *httptest.ResponseRecorder {
+ t.Helper()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+ rec := httptest.NewRecorder()
+
+ route.ServeHTTP(rec, req)
+ t.Logf("GET %s: %d", target, rec.Code)
+
+ return rec
+}
+
+// checkErrorBody checks that rec has status wantStatus and the JSON error body
+// the image route sends: wantError, wantStatus and the time in RFC 3339.
+func checkErrorBody(
+ t *testing.T, rec *httptest.ResponseRecorder, wantStatus int, wantError string,
+) {
+ t.Helper()
+
+ if rec.Code != wantStatus {
+ t.Errorf("status = %d, want %d", rec.Code, wantStatus)
+ }
+
+ if ct := rec.Header().Get("Content-Type"); ct != "application/json" {
+ t.Errorf("Content-Type = %q, want application/json", ct)
+ }
+
+ var body struct {
+ Error string `json:"error"`
+ Status int `json:"status"`
+ Timestamp string `json:"timestamp"`
+ }
+
+ err := json.NewDecoder(rec.Body).Decode(&body)
+ if err != nil {
+ t.Fatalf("decoding response body: %v", err)
+ }
+
+ if body.Error != wantError || body.Status != wantStatus {
+ t.Errorf("body error and status = %q %d, want %q %d",
+ body.Error, body.Status, wantError, wantStatus)
+ }
+
+ _, err = time.Parse(time.RFC3339, body.Timestamp)
+ if err != nil {
+ t.Errorf("body timestamp: %v", err)
+ }
+}
+
+// TestHandleImage_ErrorAnswers checks the status and the JSON error body the
+// image route answers each request below with. The JPEG at photoPath exists on
+// signedHost and on each host below that differs from it, so a request refused
+// with 401 would otherwise be served.
+func TestHandleImage_ErrorAnswers(t *testing.T) {
+ t.Parallel()
+
+ // A signature for signedHost must not verify for any of these.
+ parentHost := "example.com"
+ siblingHost := "other.example.com"
+ subdomainHost := "img." + signedHost
+ appendedHost := signedHost + ".example.net"
+
+ photos := newPhotoFetcher(t,
+ signedHost, parentHost, siblingHost, subdomainHost, appendedHost)
+ // The real fetcher refuses localhost before any lookup or connection.
+ realFetcher := httpfetcher.New(&httpfetcher.Config{
+ Timeout: 30 * time.Second,
+ MaxResponseSize: 50 << 20,
+ MaxConnectionsPerHost: 20,
+ MaxConnections: 64,
+ })
+
+ exp := time.Now().Add(time.Hour)
+ expired := time.Now().Add(-time.Hour)
+ sig := photoSignature(testSigningKey, signedHost, exp)
+ otherKeySig := photoSignature("another-signing-key", signedHost, exp)
+ expiredSig := photoSignature(testSigningKey, signedHost, expired)
+ localhostSig := photoSignature(testSigningKey, "localhost", exp)
+
+ // The error every request refused for its signature gets.
+ const unauthorized = "unauthorized"
+
+ tests := []struct {
+ name string
+ fetcher httpfetcher.Fetcher
+ target string
+ wantStatus int
+ wantError string
+ }{
+ {"no sig or exp", photos, photoURL(signedHost),
+ http.StatusUnauthorized, unauthorized},
+ {"exp but no sig", photos,
+ fmt.Sprintf("%s?exp=%d", photoURL(signedHost), exp.Unix()),
+ http.StatusUnauthorized, unauthorized},
+ {"sig made with another key", photos,
+ photoURLWithSig(signedHost, otherKeySig, exp),
+ http.StatusUnauthorized, unauthorized},
+ {"sig without its = padding", photos,
+ photoURLWithSig(signedHost, strings.TrimRight(sig, "="), exp),
+ http.StatusUnauthorized, unauthorized},
+ {"sig in upper case", photos,
+ photoURLWithSig(signedHost, strings.ToUpper(sig), exp),
+ http.StatusUnauthorized, unauthorized},
+ {"expired sig", photos, photoURLWithSig(signedHost, expiredSig, expired),
+ http.StatusUnauthorized, unauthorized},
+ {"sig sent for the parent domain", photos,
+ photoURLWithSig(parentHost, sig, exp),
+ http.StatusUnauthorized, unauthorized},
+ {"sig sent for a sibling host", photos,
+ photoURLWithSig(siblingHost, sig, exp),
+ http.StatusUnauthorized, unauthorized},
+ {"sig sent for a subdomain", photos,
+ photoURLWithSig(subdomainHost, sig, exp),
+ http.StatusUnauthorized, unauthorized},
+ {"sig sent with another domain appended", photos,
+ photoURLWithSig(appendedHost, sig, exp),
+ http.StatusUnauthorized, unauthorized},
+ {"unparseable path", photos,
+ "/v1/image/" + allowlistedHost + photoPath + "/big.jpeg",
+ http.StatusBadRequest, "invalid image URL: invalid size format"},
+ {"blocked upstream address", realFetcher,
+ photoURLWithSig("localhost", localhostSig, exp),
+ http.StatusForbidden, "forbidden"},
+ {"upstream error", photos,
+ "/v1/image/" + allowlistedHost + "/images/missing.jpg/50x50.jpeg",
+ http.StatusBadGateway, "upstream error"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ rec := sendGet(t, newImageRoute(t, tt.fetcher), tt.target)
+ checkErrorBody(t, rec, tt.wantStatus, tt.wantError)
+ })
+ }
+}
+
+// TestHandleImage_AllowlistOrSignature checks that the image route serves an
+// image without a signature for a host on the allowlist only, and for another
+// host only with a valid signature.
+func TestHandleImage_AllowlistOrSignature(t *testing.T) {
+ t.Parallel()
+
+ photos := newPhotoFetcher(t, allowlistedHost, signedHost)
+ exp := time.Now().Add(time.Hour)
+ sig := photoSignature(testSigningKey, signedHost, exp)
+
+ tests := []struct {
+ name string
+ target string
+ wantStatus int
+ }{
+ {"allowlisted host, no sig", photoURL(allowlistedHost), http.StatusOK},
+ {"other host, no sig", photoURL(signedHost), http.StatusUnauthorized},
+ {"other host, valid sig", photoURLWithSig(signedHost, sig, exp),
+ http.StatusOK},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ rec := sendGet(t, newImageRoute(t, photos), tt.target)
+ if rec.Code != tt.wantStatus {
+ t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
+ }
+ })
+ }
+}
+
+// TestHandleImage_InvalidHost_Returns400 sends image URLs whose host no URL
+// host can be, and checks that each is refused with 400 naming the host as
+// decoded: allowlistedHost with a character after it that a URL host cannot
+// hold, percent-encoded, an IPv6 address without brackets, and brackets
+// around something that is not an IPv6 address. Later steps put the host
+// unparsed into the allowlist check, the cache key and the upstream URL, so
+// such a host is refused first.
+func TestHandleImage_InvalidHost_Returns400(t *testing.T) {
+ t.Parallel()
+
+ photos := newPhotoFetcher(t, allowlistedHost)
+
+ tests := []struct {
+ name string
+ writtenHost string
+ decodedHost string
+ }{
+ {"? written as %3F", allowlistedHost + "%3Fx=1", allowlistedHost + "?x=1"},
+ {"# written as %23", allowlistedHost + "%23x", allowlistedHost + "#x"},
+ {"% written as %25", allowlistedHost + "%253F", allowlistedHost + "%3F"},
+ {`\ written as %5C`, allowlistedHost + "%5Cx", allowlistedHost + `\x`},
+ {"space written as %20", allowlistedHost + "%20", allowlistedHost + " "},
+ {"IPv6 address without brackets", "2001:db8::1", "2001:db8::1"},
+ {"IPv4 address in brackets", "[203.0.113.5]", "[203.0.113.5]"},
+ {"no address in brackets", "[abc]", "[abc]"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ rec := sendGet(t, newImageRoute(t, photos), photoURL(tt.writtenHost))
+ checkErrorBody(t, rec, http.StatusBadRequest,
+ fmt.Sprintf("invalid image URL: invalid source host: %q",
+ tt.decodedHost))
+ })
+ }
+}
+
+// TestHandleImage_HostForms_Served checks that the image route serves, with
+// a valid signature, the image at photoPath on a host name, an IPv4 address,
+// an IPv6 address in brackets, and a host followed by ":" with or without a
+// port.
+func TestHandleImage_HostForms_Served(t *testing.T) {
+ t.Parallel()
+
+ hosts := []string{
+ allowlistedHost,
+ "img_1.example.com",
+ "203.0.113.5",
+ "[2001:db8::1]",
+ allowlistedHost + ":8443",
+ "[2001:db8::1]:8443",
+ allowlistedHost + ":",
+ }
+
+ photos := newPhotoFetcher(t, hosts...)
+ exp := time.Now().Add(time.Hour)
+
+ for _, host := range hosts {
+ t.Run(host, func(t *testing.T) {
+ t.Parallel()
+
+ sig := photoSignature(testSigningKey, host, exp)
+
+ rec := sendGet(t, newImageRoute(t, photos),
+ photoURLWithSig(host, sig, exp))
+ if rec.Code != http.StatusOK {
+ t.Errorf("status = %d, want %d: %s",
+ rec.Code, http.StatusOK, rec.Body)
+ }
+ })
+ }
+}
+
+// TestHandleImage_HashInSourcePathOrQuery_Returns400 sends image URLs whose
+// source path or source query holds a "#", written as %23, and checks that
+// each is refused with 400, the one signed for that source path included.
+// A "#" starts a URL's fragment, so neither the source path nor the source
+// query may hold one.
+func TestHandleImage_HashInSourcePathOrQuery_Returns400(t *testing.T) {
+ t.Parallel()
+
+ photos := newPhotoFetcher(t, allowlistedHost, signedHost)
+ exp := time.Now().Add(time.Hour)
+
+ sig := signature.New(testSigningKey).Sign(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath + "#x",
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ Quality: 85,
+ FitMode: imagerequest.FitCover,
+ },
+ }, exp)
+
+ tests := []struct {
+ name string
+ target string
+ }{
+ {"# in the source path",
+ "/v1/image/" + allowlistedHost + photoPath + "%23x/50x50.jpeg"},
+ {"# in the source query",
+ "/v1/image/" + allowlistedHost + photoPath + "%3Fv=1%23x/50x50.jpeg"},
+ {"# in the source path, signed",
+ fmt.Sprintf("/v1/image/%s%s%%23x/50x50.jpeg?sig=%s&exp=%d",
+ signedHost, photoPath, sig, exp.Unix())},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ rec := sendGet(t, newImageRoute(t, photos), tt.target)
+ checkErrorBody(t, rec, http.StatusBadRequest,
+ `invalid image URL: invalid image path: `+
+ `the source path or query holds a "#"`)
+ })
+ }
+}
+
+// failingFetcher answers every fetch with its error.
+type failingFetcher struct {
+ err error
+}
+
+// Fetch returns f's error.
+func (f failingFetcher) Fetch(
+ context.Context, imagerequest.Source, string, string,
+) (*httpfetcher.FetchResult, error) {
+ return nil, f.err
+}
+
+// photoURLsByRoute returns, for each image route, the function that makes the
+// URL of the JPEG at photoPath on signedHost on that route.
+func photoURLsByRoute() map[string]func(*testing.T, *Handlers) string {
+ return map[string]func(*testing.T, *Handlers) string{
+ "/v1/image/": signedPhotoURL,
+ "/v1/e/": encPhotoURL,
+ }
+}
+
+// TestImageRoutes_SameErrorAnswers checks that both image routes answer each
+// upstream fetch error below with the same status. They log it as "failed to
+// get image" at the same level: error for the 500 and warn for the others.
+// Each request gets routes of its own, so no request finds another's failed
+// fetch in the negative cache.
+func TestImageRoutes_SameErrorAnswers(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ err error
+ wantStatus int
+ wantLevel slog.Level
+ }{
+ {"blocked upstream address", httpfetcher.ErrSSRFBlocked,
+ http.StatusForbidden, slog.LevelWarn},
+ {"upstream error status",
+ fmt.Errorf("%w: status 404", httpfetcher.ErrUpstreamError),
+ http.StatusBadGateway, slog.LevelWarn},
+ {"upstream host with no address",
+ fmt.Errorf("%w: cdn.example.com", httpfetcher.ErrInvalidHost),
+ http.StatusBadGateway, slog.LevelWarn},
+ {"connection refused",
+ fmt.Errorf("%w: %w", httpfetcher.ErrUpstreamRequestFailed,
+ syscall.ECONNREFUSED),
+ http.StatusBadGateway, slog.LevelWarn},
+ {"no response headers within the fetch timeout",
+ httpfetcher.ErrUpstreamTimeout, http.StatusGatewayTimeout,
+ slog.LevelWarn},
+ {"no free upstream connection", httpfetcher.ErrTooManyConnections,
+ http.StatusServiceUnavailable, slog.LevelWarn},
+ {"Content-Type not a source format",
+ fmt.Errorf("%w: image/tiff", httpfetcher.ErrInvalidContentType),
+ http.StatusUnsupportedMediaType, slog.LevelWarn},
+ {"any other failure", io.ErrUnexpectedEOF,
+ http.StatusInternalServerError, slog.LevelError},
+ }
+
+ for _, tt := range tests {
+ for route, imageURL := range photoURLsByRoute() {
+ t.Run(route+" "+tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ var logs bytes.Buffer
+
+ h, routes := newRoutesFetchingThrough(t,
+ failingFetcher{err: tt.err},
+ slog.New(slog.NewJSONHandler(&logs, nil)))
+
+ rec := sendGet(t, routes, imageURL(t, h))
+ if rec.Code != tt.wantStatus {
+ t.Errorf("status = %d, want %d: %s",
+ rec.Code, tt.wantStatus, rec.Body)
+ }
+
+ levels := loggedLevels(t, &logs)
+ if !slices.Contains(levels[tt.wantLevel.String()],
+ "failed to get image") {
+ t.Errorf(`no "failed to get image" line logged at level %s`,
+ tt.wantLevel)
+ }
+
+ if tt.wantLevel != slog.LevelError &&
+ len(levels[slog.LevelError.String()]) != 0 {
+ t.Errorf("logged at error level: %q",
+ levels[slog.LevelError.String()])
+ }
+ })
+ }
+ }
+}
+
+// failingResponseWriter is a response writer whose Write always fails, as when
+// the client has gone away.
+type failingResponseWriter struct {
+ *httptest.ResponseRecorder
+}
+
+// Write writes nothing and returns io.ErrClosedPipe.
+func (failingResponseWriter) Write([]byte) (int, error) {
+ return 0, io.ErrClosedPipe
+}
+
+// TestImageRoutes_FailedWriteLoggedOnce checks that both image routes log a
+// failed write of the image once, at warn level, with the request's ID, the
+// source host and path, the bytes served and the error. Neither may also log
+// the image as served.
+func TestImageRoutes_FailedWriteLoggedOnce(t *testing.T) {
+ t.Parallel()
+
+ const requestID = "test-request-id"
+
+ for route, imageURL := range photoURLsByRoute() {
+ t.Run(route, func(t *testing.T) {
+ t.Parallel()
+
+ var logs bytes.Buffer
+
+ h, srv := newSignedHostServer(t,
+ slog.New(slog.NewJSONHandler(&logs, nil)))
+
+ ctx := context.WithValue(t.Context(),
+ middleware.RequestIDKey, requestID)
+ srv.ServeHTTP(failingResponseWriter{httptest.NewRecorder()},
+ httptest.NewRequestWithContext(
+ ctx, http.MethodGet, imageURL(t, h), nil))
+
+ t.Logf("logged:\n%s", logs.String())
+
+ var failedWrites []map[string]any
+
+ dec := json.NewDecoder(&logs)
+ for dec.More() {
+ var line map[string]any
+
+ err := dec.Decode(&line)
+ if err != nil {
+ t.Fatalf("decoding log line: %v", err)
+ }
+
+ switch line["msg"] {
+ case "image served":
+ t.Error(`"image served" logged after a failed write`)
+ case "failed to write response":
+ failedWrites = append(failedWrites, line)
+ }
+ }
+
+ if len(failedWrites) != 1 {
+ t.Fatalf(`"failed to write response" logged %d times, want 1`,
+ len(failedWrites))
+ }
+
+ want := map[string]any{
+ "level": slog.LevelWarn.String(),
+ "request_id": requestID,
+ "host": signedHost,
+ "path": photoPath,
+ "served_bytes": float64(0),
+ "error": io.ErrClosedPipe.Error(),
+ }
+
+ for key, wantValue := range want {
+ if got := failedWrites[0][key]; got != wantValue {
+ t.Errorf("%s = %v, want %v", key, got, wantValue)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/handlers/image_signature_internal_test.go b/internal/handlers/image_signature_internal_test.go
index c435668..eaed6ee 100644
--- a/internal/handlers/image_signature_internal_test.go
+++ b/internal/handlers/image_signature_internal_test.go
@@ -5,18 +5,19 @@ import (
"fmt"
"net/http"
"net/http/httptest"
+ "net/url"
"testing"
"time"
"github.com/go-chi/chi/v5"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
"sneak.berlin/go/pixa/internal/signature"
)
// signedHost is not on the allowlist setupTestHandler builds, so a request
-// for it needs a valid signature. No image is served for it: a request that
-// passes the signature check gets 502 from the failed fetch, and one that
-// fails the check gets 401.
+// for it needs a valid signature. setupTestHandler serves no image for it: a
+// request that passes the signature check gets 502 from the failed fetch, and
+// one that fails the check gets 401.
const signedHost = "signed.example.com"
// getImage sends a GET for target to the image route of fix and returns the
@@ -36,6 +37,47 @@ func getImage(t *testing.T, fix *testFixtures, target string) int {
return rec.Code
}
+// generateSignedURL returns the /v1/image/ URL of req, signed with key and
+// expiring after ttl. It signs a copy of req, so req itself is not changed.
+// An unset quality or fit in the copy is set to 85 or cover, the values the
+// route uses when a URL has no q or fit. The URL carries q and fit next to sig
+// and exp, so the route verifies it for the quality and fit it was signed
+// with.
+func generateSignedURL(
+ key string, req *imagerequest.Request, ttl time.Duration,
+) string {
+ signed := *req
+
+ if signed.Transform.Quality == 0 {
+ signed.Transform.Quality = 85
+ }
+
+ if signed.Transform.FitMode == "" {
+ signed.Transform.FitMode = imagerequest.FitCover
+ }
+
+ expires := time.Now().Add(ttl)
+ sig := signature.New(key).Sign(&signed, expires)
+
+ // A source query follows the source path after a "?". The image route
+ // reads the path decoded once, so the whole path is written in Go's own
+ // percent-encoding of a URL path, which writes "?" as %3F and "%" as %25,
+ // and leaves "/", ";", ",", "=" and "&" as they are.
+ sourcePathAndQuery := signed.Source.Path
+ if signed.Source.Query != "" {
+ sourcePathAndQuery += "?" + signed.Source.Query
+ }
+
+ unescapedPath := fmt.Sprintf("/v1/image/%s%s/%dx%d.%s",
+ signed.Source.Host, sourcePathAndQuery,
+ signed.Transform.Size.Width, signed.Transform.Size.Height,
+ signed.Transform.Format)
+
+ return fmt.Sprintf("%s?sig=%s&exp=%d&q=%d&fit=%s",
+ (&url.URL{Path: unescapedPath}).EscapedPath(), sig, expires.Unix(),
+ signed.Transform.Quality, signed.Transform.FitMode)
+}
+
// TestHandleImage_SignatureCoversQualityAndFit signs a URL for quality 85
// and fit cover, the values the route uses when a URL has no q or fit, and
// sends that signature with each q and fit below.
@@ -44,16 +86,18 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) {
expires := time.Now().Add(time.Hour)
signer := signature.New("test-signing-key-must-be-32-chars")
- sig := signer.Sign(&signature.Request{
- SourceHost: signedHost,
- SourcePath: "/images/photo.jpg",
- Width: 50,
- Height: 50,
- Format: string(imgcache.FormatJPEG),
- Quality: 85,
- FitMode: string(imgcache.FitCover),
- Expires: expires,
- })
+ sig := signer.Sign(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: "/images/photo.jpg",
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ Quality: 85,
+ FitMode: imagerequest.FitCover,
+ },
+ }, expires)
signedURL := fmt.Sprintf("/v1/image/%s/images/photo.jpg/50x50.jpeg?sig=%s&exp=%d",
signedHost, sig, expires.Unix())
@@ -80,17 +124,17 @@ func TestHandleImage_SignatureCoversQualityAndFit(t *testing.T) {
}
}
-// TestHandleImage_GeneratedSignedURLVerifies sends URLs built by the
-// service's signed-URL generator to the route.
+// TestHandleImage_GeneratedSignedURLVerifies sends URLs built by
+// generateSignedURL to the route.
func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) {
t.Parallel()
tests := []struct {
name string
quality int
- fitMode imgcache.FitMode
+ fitMode imagerequest.Fit
}{
- {"quality 40 and fit contain", 40, imgcache.FitContain},
+ {"quality 40 and fit contain", 40, imagerequest.FitContain},
{"quality and fit unset", 0, ""},
}
@@ -98,21 +142,21 @@ func TestHandleImage_GeneratedSignedURLVerifies(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
- fix := setupTestHandler(t)
+ signedURL := generateSignedURL("test-signing-key-must-be-32-chars",
+ &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: "/images/photo.jpg",
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ Quality: tt.quality,
+ FitMode: tt.fitMode,
+ },
+ }, time.Hour)
- signedURL, err := fix.service.GenerateSignedURL("", &imgcache.ImageRequest{
- SourceHost: signedHost,
- SourcePath: "/images/photo.jpg",
- Size: imgcache.Size{Width: 50, Height: 50},
- Format: imgcache.FormatJPEG,
- Quality: tt.quality,
- FitMode: tt.fitMode,
- }, time.Hour)
- if err != nil {
- t.Fatalf("GenerateSignedURL() error = %v", err)
- }
-
- status := getImage(t, fix, signedURL)
+ status := getImage(t, setupTestHandler(t), signedURL)
if status != http.StatusBadGateway {
t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
}
@@ -169,3 +213,170 @@ func TestHandleImage_InvalidExp_Returns400(t *testing.T) {
})
}
}
+
+// TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies sends URLs built
+// by generateSignedURL for source URLs with a query to the route, through a
+// chi router: chi routes on the path as the client wrote it whenever that is
+// not Go's own encoding of it. A 502 means the signature passed, so the route
+// read the source path and query that were signed.
+func TestHandleImage_GeneratedSignedURLWithSourceQueryVerifies(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ query string
+ }{
+ {"slash and colon", "url=https://origin.example.com/cat.jpg"},
+ {"semicolon", "a=1;b=2"},
+ {"comma", "crop=0,0,50,50"},
+ {"equals and ampersand", "a=1&b=2"},
+ {"percent-escape", "name=my%20cat"},
+ {"question mark", "next=/a?b=1"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ signedURL := generateSignedURL("test-signing-key-must-be-32-chars",
+ &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ Query: tt.query,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ },
+ }, time.Hour)
+
+ status := getImage(t, setupTestHandler(t), signedURL)
+ if status != http.StatusBadGateway {
+ t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
+ }
+ })
+ }
+}
+
+// TestHandleImage_SourceQueryEncodedAnyWay signs one source URL with a query
+// and writes it into the image URL in ways the README allows: the "?" before
+// the query and each "%" in it percent-encoded, and every other character as
+// itself or percent-encoded. A 502 means the signature passed.
+func TestHandleImage_SourceQueryEncodedAnyWay(t *testing.T) {
+ t.Parallel()
+
+ expires := time.Now().Add(time.Hour)
+ signer := signature.New("test-signing-key-must-be-32-chars")
+ sig := signer.Sign(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ Query: "url=https://origin.example.com/my%20cat.jpg",
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ Quality: 85,
+ FitMode: imagerequest.FitCover,
+ },
+ }, expires)
+
+ tests := []struct {
+ name string
+ writtenQuery string
+ }{
+ {"only ? and % encoded",
+ "%3Furl=https://origin.example.com/my%2520cat.jpg"},
+ {"every reserved character encoded",
+ "%3Furl%3Dhttps%3A%2F%2Forigin.example.com%2Fmy%2520cat.jpg"},
+ {"lower-case hex digits",
+ "%3furl=https://origin.example.com/my%2520cat.jpg"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ target := fmt.Sprintf("/v1/image/%s%s%s/50x50.jpeg?sig=%s&exp=%d",
+ signedHost, photoPath, tt.writtenQuery, sig, expires.Unix())
+
+ status := getImage(t, setupTestHandler(t), target)
+ if status != http.StatusBadGateway {
+ t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
+ }
+ })
+ }
+}
+
+// TestHandleImage_SourceURLHoldingPercent25Verifies signs a source path and a
+// source query that each hold %25, the percent-escape of a literal "%", and
+// sends each to the route written as the README says, with that "%" written as
+// %25, and as generateSignedURL writes it. A 502 means the signature passed.
+func TestHandleImage_SourceURLHoldingPercent25Verifies(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ sourcePath string
+ sourceQuery string
+ // writtenPathAndQuery is the source path and query as the README says
+ // to write them into the image URL
+ writtenPathAndQuery string
+ }{
+ {"path", "/sale/50%25off.jpg", "", "/sale/50%2525off.jpg"},
+ {"query", photoPath, "discount=50%25", photoPath + "%3Fdiscount=50%2525"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name+" written by the README rule", func(t *testing.T) {
+ t.Parallel()
+
+ expires := time.Now().Add(time.Hour)
+ signer := signature.New("test-signing-key-must-be-32-chars")
+ sig := signer.Sign(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: tt.sourcePath,
+ Query: tt.sourceQuery,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ Quality: 85,
+ FitMode: imagerequest.FitCover,
+ },
+ }, expires)
+
+ target := fmt.Sprintf("/v1/image/%s%s/50x50.jpeg?sig=%s&exp=%d",
+ signedHost, tt.writtenPathAndQuery, sig, expires.Unix())
+
+ status := getImage(t, setupTestHandler(t), target)
+ if status != http.StatusBadGateway {
+ t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
+ }
+ })
+
+ t.Run(tt.name+" written by generateSignedURL", func(t *testing.T) {
+ t.Parallel()
+
+ signedURL := generateSignedURL("test-signing-key-must-be-32-chars",
+ &imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: tt.sourcePath,
+ Query: tt.sourceQuery,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJPEG,
+ },
+ }, time.Hour)
+
+ status := getImage(t, setupTestHandler(t), signedURL)
+ if status != http.StatusBadGateway {
+ t.Errorf("status = %d, want %d", status, http.StatusBadGateway)
+ }
+ })
+ }
+}
diff --git a/internal/handlers/imageenc.go b/internal/handlers/imageenc.go
index 8089f64..07c2772 100644
--- a/internal/handlers/imageenc.go
+++ b/internal/handlers/imageenc.go
@@ -2,17 +2,13 @@ package handlers
import (
"errors"
- "fmt"
- "io"
"net/http"
- "strconv"
"time"
"github.com/go-chi/chi/v5"
"sneak.berlin/go/pixa/internal/encurl"
- "sneak.berlin/go/pixa/internal/httpfetcher"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
)
// HandleImageEnc handles requests to /v1/e/{token}/* for encrypted
@@ -20,112 +16,63 @@ import (
// browsers identify the content type.
func (s *Handlers) HandleImageEnc() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
- ctx := r.Context()
- start := time.Now()
-
- // Extract token from URL
- token := chi.URLParam(r, "token")
- if token == "" {
- s.respondError(w, "missing token", http.StatusBadRequest)
-
+ if s.refuseBlockedReferer(w, r) {
return
}
- // Decrypt and validate the payload
- payload, err := s.encGen.Parse(token)
- if err != nil {
- if errors.Is(err, encurl.ErrExpired) {
- s.log.Debug("encrypted URL expired", "error", err)
- s.respondError(w, "URL has expired", http.StatusGone)
-
- return
- }
-
- s.log.Debug("failed to decrypt URL", "error", err)
- s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
-
+ req, expires, ok := s.parseImageEncRequest(w, r)
+ if !ok {
return
}
- // Convert payload to ImageRequest
- req := payload.ToImageRequest()
-
- // Apply the same dimension and fit-mode bounds as the plain image
- // route: a sealed payload is trusted for its origin, not for staying
- // within limits, so an over-limit size or unknown fit mode is a 400
- // here rather than an out-of-memory or a 500 from the processor.
- err = imgcache.ValidateImageRequest(req)
- if err != nil {
- s.log.Debug("encrypted URL failed validation", "error", err)
- s.respondError(w, "invalid encrypted URL: "+err.Error(),
- http.StatusBadRequest)
-
- return
- }
-
- // Log the request
- s.log.Debug("encrypted image request",
- "host", req.SourceHost,
- "path", req.SourcePath,
- "dimensions", fmt.Sprintf("%dx%d", req.Size.Width, req.Size.Height),
- "format", req.Format,
- )
-
- // Fetch and process the image (no signature validation
- // needed - encrypted URL is trusted)
- resp, err := s.imgSvc.Get(ctx, req)
- if err != nil {
- s.handleImageError(w, err)
-
- return
- }
-
- defer func() { _ = resp.Content.Close() }()
-
- // Set response headers
- w.Header().Set("Content-Type", resp.ContentType)
-
- if resp.ContentLength > 0 {
- w.Header().Set("Content-Length", strconv.FormatInt(resp.ContentLength, 10))
- }
-
- // Cache headers - encrypted URLs can be cached since they're immutable
- w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
- w.Header().Set("X-Pixa-Cache", string(resp.CacheStatus))
-
- // Stream the response
- written, err := io.Copy(w, resp.Content)
- if err != nil {
- s.log.Error("failed to write response", "error", err)
-
- return
- }
-
- // Log completion
- duration := time.Since(start)
- s.log.Info("image served",
- "cache_key", imgcache.CacheKey(req),
- "host", req.SourceHost,
- "path", req.SourcePath,
- "format", req.Format,
- "cache_status", resp.CacheStatus,
- "served_bytes", written,
- "duration_ms", duration.Milliseconds(),
- )
+ s.serveImage(w, r, req, expires)
}
}
-// handleImageError converts image service errors to HTTP responses.
-func (s *Handlers) handleImageError(w http.ResponseWriter, err error) {
- switch {
- case errors.Is(err, httpfetcher.ErrSSRFBlocked):
- s.respondError(w, "forbidden", http.StatusForbidden)
- case errors.Is(err, httpfetcher.ErrUpstreamError):
- s.respondError(w, "upstream error", http.StatusBadGateway)
- case errors.Is(err, httpfetcher.ErrUpstreamTimeout):
- s.respondError(w, "upstream timeout", http.StatusGatewayTimeout)
- default:
- s.log.Error("image request failed", "error", err)
- s.respondError(w, "internal error", http.StatusInternalServerError)
+// parseImageEncRequest decrypts the token of an encrypted image URL into an
+// image request and checks it, and returns it with the token's expiry. On a
+// token that is missing, does not decrypt, has expired or asks for something
+// not valid, it writes an error response and returns false.
+func (s *Handlers) parseImageEncRequest(
+ w http.ResponseWriter, r *http.Request,
+) (*imagerequest.Request, time.Time, bool) {
+ // Extract token from URL
+ token := chi.URLParam(r, "token")
+ if token == "" {
+ s.respondError(w, "missing token", http.StatusBadRequest)
+
+ return nil, time.Time{}, false
}
+
+ // Decrypt and validate the payload
+ req, expires, err := s.encGen.Parse(token)
+ if err != nil {
+ if errors.Is(err, encurl.ErrExpired) {
+ s.log.Debug("encrypted URL expired", "error", err)
+ s.respondError(w, "URL has expired", http.StatusGone)
+
+ return nil, time.Time{}, false
+ }
+
+ s.log.Debug("invalid encrypted URL", "error", err)
+ s.respondError(w, "invalid encrypted URL", http.StatusBadRequest)
+
+ return nil, time.Time{}, false
+ }
+
+ // Apply the same dimension and fit-mode bounds as the plain image
+ // route: a sealed payload is trusted for its origin, not for staying
+ // within limits. This refuses an over-limit size before the source is
+ // fetched. An unknown fit mode is a 400 here rather than a 500 from the
+ // processor.
+ err = imagerequest.ValidateImageRequest(req)
+ if err != nil {
+ s.log.Debug("encrypted URL failed validation", "error", err)
+ s.respondError(w, "invalid encrypted URL: "+err.Error(),
+ http.StatusBadRequest)
+
+ return nil, time.Time{}, false
+ }
+
+ return req, expires, true
}
diff --git a/internal/handlers/imageenc_internal_test.go b/internal/handlers/imageenc_internal_test.go
index be2db22..2da0124 100644
--- a/internal/handlers/imageenc_internal_test.go
+++ b/internal/handlers/imageenc_internal_test.go
@@ -6,11 +6,13 @@ import (
"net/http"
"net/http/httptest"
"testing"
+ "time"
"github.com/go-chi/chi/v5"
"sneak.berlin/go/pixa/internal/encurl"
- "sneak.berlin/go/pixa/internal/imgcache"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/seal"
)
// newEncTestServer builds a router serving the encrypted-URL route with a
@@ -20,11 +22,13 @@ import (
func newEncTestServer(t *testing.T) (*encurl.Generator, http.Handler) {
t.Helper()
- encGen, err := encurl.NewGenerator(testSigningKey)
+ keys, err := seal.DeriveKeys(testSigningKey)
if err != nil {
- t.Fatalf("encurl.NewGenerator() error = %v", err)
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
}
+ encGen := encurl.NewGenerator(keys.EncryptedURLKey)
+
h := &Handlers{
log: slog.New(slog.DiscardHandler),
encGen: encGen,
@@ -54,12 +58,15 @@ func TestHandleImageEnc_OverLimitDimension_Returns400(t *testing.T) {
encGen, srv := newEncTestServer(t)
- token, err := encGen.Generate(&encurl.Payload{
- SourceHost: "cdn.example.com",
- SourcePath: "/photo.jpg",
- Width: 100000,
- Height: 100000,
- })
+ token, err := encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: "cdn.example.com",
+ Path: "/photo.jpg",
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 100000, Height: 100000},
+ },
+ }, time.Time{})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
@@ -79,13 +86,16 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) {
encGen, srv := newEncTestServer(t)
- token, err := encGen.Generate(&encurl.Payload{
- SourceHost: "cdn.example.com",
- SourcePath: "/photo.jpg",
- Width: 800,
- Height: 600,
- FitMode: imgcache.FitMode("bogus"),
- })
+ token, err := encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: "cdn.example.com",
+ Path: "/photo.jpg",
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 800, Height: 600},
+ FitMode: imagerequest.Fit("bogus"),
+ },
+ }, time.Time{})
if err != nil {
t.Fatalf("Generate() error = %v", err)
}
@@ -96,3 +106,70 @@ func TestHandleImageEnc_InvalidFitMode_Returns400(t *testing.T) {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
}
}
+
+// TestHandleImageEnc_IfNoneMatch_Returns304 verifies that an image served
+// through an encrypted URL carries an ETag, and that a request whose
+// If-None-Match is that ETag is answered 304 Not Modified with no body.
+func TestHandleImageEnc_IfNoneMatch_Returns304(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+ target := encPhotoURL(t, h)
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, target, nil))
+
+ etag := rec.Header().Get("ETag")
+ t.Logf("GET: %d, ETag %q", rec.Code, etag)
+
+ if rec.Code != http.StatusOK || etag == "" {
+ t.Fatalf("GET: status = %d, ETag = %q, want %d and an ETag",
+ rec.Code, etag, http.StatusOK)
+ }
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+ req.Header.Set("If-None-Match", etag)
+
+ rec = httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+ t.Logf("GET with If-None-Match: %d, %d body bytes", rec.Code, rec.Body.Len())
+
+ if rec.Code != http.StatusNotModified || rec.Body.Len() != 0 {
+ t.Errorf("status = %d with %d body bytes, want %d with none",
+ rec.Code, rec.Body.Len(), http.StatusNotModified)
+ }
+}
+
+// TestHandleImageEnc_HEAD_ReturnsHeadersOnly verifies that HEAD on an
+// encrypted URL is answered 200 with the headers GET sends and no body.
+func TestHandleImageEnc_HEAD_ReturnsHeadersOnly(t *testing.T) {
+ t.Parallel()
+
+ h, _ := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ r := chi.NewRouter()
+ r.Head("/v1/e/{token}/*", h.HandleImageEnc())
+
+ rec := httptest.NewRecorder()
+ r.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodHead, encPhotoURL(t, h), nil))
+ t.Logf("HEAD: %d, headers %v, %d body bytes",
+ rec.Code, rec.Header(), rec.Body.Len())
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
+ }
+
+ for _, name := range []string{
+ "Content-Type", "Content-Length", "Cache-Control", "ETag",
+ } {
+ if rec.Header().Get(name) == "" {
+ t.Errorf("HEAD response has no %s", name)
+ }
+ }
+
+ if rec.Body.Len() != 0 {
+ t.Errorf("HEAD response body has %d bytes, want none", rec.Body.Len())
+ }
+}
diff --git a/internal/handlers/imageenc_token_internal_test.go b/internal/handlers/imageenc_token_internal_test.go
new file mode 100644
index 0000000..09e449f
--- /dev/null
+++ b/internal/handlers/imageenc_token_internal_test.go
@@ -0,0 +1,130 @@
+package handlers
+
+import (
+ "image/jpeg"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/seal"
+)
+
+// requireServedPhoto requires that rec answers 200 with the JPEG at photoPath
+// on signedHost at the 50x50 that encPhotoURL and the generator tests ask for.
+func requireServedPhoto(t *testing.T, rec *httptest.ResponseRecorder) {
+ t.Helper()
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d; body %q",
+ rec.Code, http.StatusOK, rec.Body.String())
+ }
+
+ contentType := rec.Header().Get("Content-Type")
+ if contentType != "image/jpeg" {
+ t.Errorf("Content-Type = %q, want image/jpeg", contentType)
+ }
+
+ img, err := jpeg.DecodeConfig(rec.Body)
+ if err != nil {
+ t.Fatalf("body is not a JPEG: %v", err)
+ }
+
+ if img.Width != 50 || img.Height != 50 {
+ t.Errorf("image is %dx%d, want 50x50", img.Width, img.Height)
+ }
+}
+
+// TestHandleImageEnc_ValidToken_ServesImage verifies that a token made with
+// the signing key serves the image it asks for.
+func TestHandleImageEnc_ValidToken_ServesImage(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, encPhotoURL(t, h), nil))
+
+ requireServedPhoto(t, rec)
+}
+
+// TestHandleImageEnc_RejectedToken verifies that a token that has expired
+// answers 410, and that a token with one character changed, a token cut
+// short, and a token made with another signing key answer 400. The server
+// would serve the photo for a token it accepted.
+func TestHandleImageEnc_RejectedToken(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ photo := imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ },
+ }
+
+ valid, err := h.encGen.Generate(&photo, time.Time{})
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ expired, err := h.encGen.Generate(&photo, time.Now().Add(-time.Minute))
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ keys, err := seal.DeriveKeys("another-signing-key-fedcba9876543210")
+ if err != nil {
+ t.Fatalf("seal.DeriveKeys() error = %v", err)
+ }
+
+ otherGen := encurl.NewGenerator(keys.EncryptedURLKey)
+
+ otherKey, err := otherGen.Generate(&photo, time.Time{})
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ // Changing a character in the middle always changes the decoded bytes;
+ // the last character of unpadded base64 can carry unused bits.
+ middle := len(valid) / 2
+
+ replacement := "A"
+ if valid[middle] == 'A' {
+ replacement = "B"
+ }
+
+ changed := valid[:middle] + replacement + valid[middle+1:]
+
+ tests := []struct {
+ name string
+ token string
+ wantStatus int
+ }{
+ {"expired", expired, http.StatusGone},
+ {"one character changed", changed, http.StatusBadRequest},
+ {"cut short", valid[:middle], http.StatusBadRequest},
+ {"another signing key", otherKey, http.StatusBadRequest},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ rec := getEncToken(srv, tt.token)
+ t.Logf("GET /v1/e/%s/img.jpg: %d %s", tt.token, rec.Code, rec.Body)
+
+ if rec.Code != tt.wantStatus {
+ t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
+ }
+ })
+ }
+}
diff --git a/internal/handlers/jpegxl_default_internal_test.go b/internal/handlers/jpegxl_default_internal_test.go
new file mode 100644
index 0000000..9a5502e
--- /dev/null
+++ b/internal/handlers/jpegxl_default_internal_test.go
@@ -0,0 +1,167 @@
+package handlers
+
+import (
+ "fmt"
+ "log/slog"
+ "maps"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/davidbyttow/govips/v2/vips"
+
+ "sneak.berlin/go/pixa/internal/encurl"
+ "sneak.berlin/go/pixa/internal/imagerequest"
+ "sneak.berlin/go/pixa/internal/signature"
+)
+
+// requireJPEGXL requires that rec answers 200 with a JPEG XL image.
+func requireJPEGXL(t *testing.T, rec *httptest.ResponseRecorder) {
+ t.Helper()
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d; body %q",
+ rec.Code, http.StatusOK, rec.Body.String())
+ }
+
+ if got := rec.Header().Get("Content-Type"); got != jxlType {
+ t.Errorf("Content-Type = %q, want %s", got, jxlType)
+ }
+
+ if got := vips.DetermineImageType(rec.Body.Bytes()); got != vips.ImageTypeJXL {
+ t.Errorf("body is %s, want jxl", vips.ImageTypes[got])
+ }
+}
+
+// TestImageWithoutFormat_ServesJPEGXL verifies that a /v1/image/ URL whose
+// last segment is a size with no format, 50x50 or orig, answers JPEG XL.
+func TestImageWithoutFormat_ServesJPEGXL(t *testing.T) {
+ t.Parallel()
+
+ route := newImageRoute(t, newPhotoFetcher(t, allowlistedHost))
+
+ for _, size := range []string{"50x50", "orig"} {
+ target := "/v1/image/" + allowlistedHost + photoPath + "/" + size
+ requireJPEGXL(t, sendGet(t, route, target))
+ }
+}
+
+// TestImageWithoutFormat_SignedAsJXL verifies that a /v1/image/ URL with no
+// format is signed as jxl: the signature made for the URL ending in .jxl is
+// accepted for the same URL without .jxl.
+func TestImageWithoutFormat_SignedAsJXL(t *testing.T) {
+ t.Parallel()
+
+ route := newImageRoute(t, newPhotoFetcher(t, signedHost))
+ expires := time.Now().Add(time.Hour)
+
+ sig := signature.New(testSigningKey).Sign(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ Format: imagerequest.FormatJXL,
+ Quality: encurl.DefaultQuality,
+ FitMode: imagerequest.FitCover,
+ },
+ }, expires)
+ query := fmt.Sprintf("?sig=%s&exp=%d", sig, expires.Unix())
+
+ for _, size := range []string{"50x50.jxl", "50x50"} {
+ target := "/v1/image/" + signedHost + photoPath + "/" + size + query
+ requireJPEGXL(t, sendGet(t, route, target))
+ }
+}
+
+// TestImageEncWithoutFormat_ServesJPEGXL verifies that an encrypted URL whose
+// token holds no format answers JPEG XL.
+func TestImageEncWithoutFormat_ServesJPEGXL(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+ token, err := h.encGen.Generate(&imagerequest.Request{
+ Source: imagerequest.Source{
+ Host: signedHost,
+ Path: photoPath,
+ },
+ Transform: imagerequest.Transform{
+ Size: imagerequest.Size{Width: 50, Height: 50},
+ },
+ }, time.Time{})
+ if err != nil {
+ t.Fatalf("Generate() error = %v", err)
+ }
+
+ requireJPEGXL(t, getEncToken(srv, token))
+}
+
+// TestGeneratorPage_SelectsJPEGXL verifies that the generator page's format
+// choice is JPEG XL until another is chosen.
+func TestGeneratorPage_SelectsJPEGXL(t *testing.T) {
+ t.Parallel()
+
+ h, srv := newCSRFTestRouter(t)
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.AddCookie(newSessionCookie(t, h))
+
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+
+ if !strings.Contains(rec.Body.String(), `