Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6bed1fda98 | ||
|
|
200997761a | ||
|
|
18b4ee38aa | ||
|
|
721502c7b8 | ||
|
|
2beba15ae7 | ||
|
|
23ec4026f6 | ||
|
|
ef828f71a5 | ||
|
|
f3231a3c5a | ||
|
|
cca2e3f926 | ||
|
|
708a9bec20 | ||
|
|
8314099abd | ||
|
|
8568c17d1b | ||
|
|
625fd42ace | ||
|
|
66e71b4207 | ||
|
|
847ad5b428 | ||
|
|
233a9c05ad | ||
|
|
842372250f | ||
|
|
58d601ea48 | ||
|
|
48f21d4ecf | ||
|
|
f8c437b83f |
+65
-9
@@ -1,12 +1,68 @@
|
||||
# .git is sent without its config. Without a VERSION build argument the
|
||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||
# does not need .git/config; that file can hold a credential, such as a
|
||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
||||
# `certs/server.key` still ship while this file reads as solved. Only
|
||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
||||
# into .gitignore, where `**/` is wrong.
|
||||
#
|
||||
# Matching is case-sensitive, so secrets use character ranges rather
|
||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
||||
#
|
||||
# Extend with this repo's own host-built artifacts, written anchored:
|
||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
||||
# deletes the package directory from the context.
|
||||
|
||||
# Unlike the standard file, which leaves out all of .git, pixa sends
|
||||
# .git without its config. Without a VERSION build argument the stage
|
||||
# that compiles runs `git describe --tags --always` on .git, which does
|
||||
# not need .git/config; that file can hold a credential, such as a
|
||||
# password in a remote URL or the token the CI checkout step stores there.
|
||||
.git/config
|
||||
.gitignore
|
||||
.DS_Store
|
||||
.env*
|
||||
|
||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
||||
# Anchored because it occurs once where agents run at the repo root.
|
||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
||||
# `services/api/.claude/` and must add its own anchored entry.
|
||||
.claude
|
||||
node_modules
|
||||
bin/
|
||||
data/
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Re-include a committed template with a negation if the
|
||||
# build needs one: `!docs/example.env`.
|
||||
**/*.[eE][nN][vV]
|
||||
**/.[eE][nN][vV].*
|
||||
**/.[eE][nN][vV][rR][cC]
|
||||
|
||||
# Private keys and the bundles carrying them. Public certificates
|
||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
||||
**/*.[pP][eE][mM]
|
||||
**/*.[kK][eE][yY]
|
||||
**/*.[pP]12
|
||||
**/*.[pP][fF][xX]
|
||||
**/[iI][dD]_[rR][sS][aA]
|
||||
**/[iI][dD]_[dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
**/[iI][dD]_[eE][dD]25519
|
||||
|
||||
# Dependencies: restored inside the image, never copied in.
|
||||
**/node_modules
|
||||
|
||||
# OS metadata.
|
||||
**/.DS_Store
|
||||
**/Thumbs.db
|
||||
|
||||
# Editor state: never a build input, and it churns COPY.
|
||||
**/*.swp
|
||||
**/*.swo
|
||||
**/*~
|
||||
**/*.bak
|
||||
**/.idea
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# pixa's own entries. Nothing in the build reads .gitignore. On the
|
||||
# host, `make build` writes bin/pixad, and the example config keeps its
|
||||
# state directory in data/.
|
||||
.gitignore
|
||||
/bin
|
||||
/data
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# Every PR adds an entry at the top of TODO.md's Completed Steps; union keeps
|
||||
# both sides instead of conflicting. Git never reports a conflict here: read
|
||||
# the merged entries after every merge or rebase.
|
||||
TODO.md merge=union
|
||||
@@ -11,6 +11,12 @@ Thumbs.db
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
||||
# entry and must not be given a `**/` prefix on the way into one.
|
||||
.claude/
|
||||
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
|
||||
+66
-2
@@ -10,14 +10,20 @@ run:
|
||||
|
||||
linters:
|
||||
default: all
|
||||
enable:
|
||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
||||
# left to `default: all`, because it carries the module policy below.
|
||||
- gomodguard_v2
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
# Deprecated: the warning is attached to the old name, so it is
|
||||
# silenced by disabling that name, not by enabling the successor.
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
@@ -28,6 +34,64 @@ linters:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
depguard:
|
||||
# Test-support code must not be compiled into the shipped binary. A
|
||||
# test-support package exists to hand a test privileges the program
|
||||
# itself must never have, so a file that is not a test must not import
|
||||
# one. Test files, and the files inside a package whose directory name
|
||||
# ends in `test`, are where that code belongs, and are exempt.
|
||||
#
|
||||
# The deny list below is the one part of this file a repository is
|
||||
# expected to extend, and the only part it may. depguard matches an
|
||||
# import path against a list of prefixes, so it cannot be told "any path
|
||||
# whose last segment ends in test"; a repository's own test-support
|
||||
# packages have to be named here one at a time, by full import path,
|
||||
# under a module path that differs from repository to repository. Add
|
||||
# them; change nothing else.
|
||||
rules:
|
||||
test-support:
|
||||
list-mode: lax
|
||||
files:
|
||||
- "$all"
|
||||
- "!$test"
|
||||
- "!**/*test/**"
|
||||
deny:
|
||||
- pkg: net/http/httptest
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
# Only decisions already recorded in the Go package defaults are
|
||||
# listed here. Every entry matches the module path exactly.
|
||||
gomodguard_v2:
|
||||
blocked:
|
||||
- module: github.com/rs/zerolog
|
||||
recommendations:
|
||||
- log/slog
|
||||
reason: "Structured logging is stdlib log/slog."
|
||||
# One entry per pre-fork module path, because the later releases
|
||||
# are separate paths. A prefix match would be shorter but would
|
||||
# also reach github.com/go-redis/redismock, the test double for
|
||||
# the successor these entries recommend.
|
||||
- module: github.com/go-redis/redis
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v7
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v8
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/sergi/go-diff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "No unified diff output; use go-udiff."
|
||||
- module: github.com/hexops/gotextdiff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "Unmaintained fork; use go-udiff."
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
|
||||
-1259
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest
|
||||
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
LDFLAGS := -X main.Version=$(VERSION)
|
||||
@@ -59,6 +59,11 @@ docker:
|
||||
docker-smoke:
|
||||
@script/docker-smoke
|
||||
|
||||
# Measure throughput, latency and peak memory with the default duration and
|
||||
# number of clients (needs Docker and Go; a benchmark, not part of check)
|
||||
loadtest:
|
||||
@script/loadtest
|
||||
|
||||
# Build Docker image tagged pixad:$(VERSION) and pixad:latest
|
||||
docker-versioned:
|
||||
docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
|
||||
|
||||
@@ -18,7 +18,7 @@ make build
|
||||
# run with a config file: copy the example and set a real signing key
|
||||
# (the example placeholder is refused at startup), e.g. with
|
||||
# openssl rand -base64 32
|
||||
cp config.example.yml config.yml
|
||||
cp configs/config.example.yml config.yml
|
||||
$EDITOR config.yml # replace the signing_key placeholder
|
||||
./bin/pixad --config config.yml
|
||||
|
||||
@@ -45,7 +45,8 @@ everything in this list without further settings. The reverse proxy must:
|
||||
Routes);
|
||||
- pass the `Host`, `Origin` and `Referer` headers on unchanged, as pixa refuses
|
||||
a form from those pages unless `Origin` or `Referer` names the host in `Host`,
|
||||
and builds encrypted URLs from `Host`;
|
||||
builds encrypted URLs from `Host`, and checks `Referer` against
|
||||
`referer_blocklist`;
|
||||
- set `X-Forwarded-For` to the client's address, with `trusted_proxies` set to
|
||||
the address pixa sees the proxy's requests come from, so the login limit
|
||||
counts each client by its own address (see `trusted_proxies` under
|
||||
@@ -145,10 +146,11 @@ name.
|
||||
|
||||
Multiple source paths may reference the same content blob; the
|
||||
database tracks references rather than using filesystem refcounting.
|
||||
Toward a target of 1-5k r/s, pixa keeps in memory the content types of
|
||||
the 10,000 transformed images most recently cached or served, so a
|
||||
cache hit on one of them reads only the image file from disk and not
|
||||
the metadata file stored beside it.
|
||||
|
||||
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
||||
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
||||
transformed images most recently cached or served, so a cache hit on one of them
|
||||
reads only the image file from disk and not the metadata file stored beside it.
|
||||
|
||||
### Routes
|
||||
|
||||
@@ -175,11 +177,13 @@ path under `/v1/` answers 200, in maintenance mode too.
|
||||
allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches
|
||||
the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a
|
||||
missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when
|
||||
the upstream host, or a host it redirects to, is `localhost`, ends in
|
||||
`.localhost` or `.local`, or has an address in a blocked network (see
|
||||
`blocked_networks`); 502 when the upstream answered with an error status, and
|
||||
for 5 minutes after that for the same source URL; 503 when pixa is busy or in
|
||||
maintenance mode; 500 for any other failure.
|
||||
the request's `Referer` names a host in `referer_blocklist`, checked before
|
||||
the signature, the cache and the upstream fetch; 403 when the upstream host,
|
||||
or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`,
|
||||
or has an address in a blocked network (see `blocked_networks`); 502 when the
|
||||
upstream answered with an error status, and for 5 minutes after that for the
|
||||
same source URL; 503 when pixa is busy or in maintenance mode; 500 for any
|
||||
other failure.
|
||||
- `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
|
||||
(see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
|
||||
`If-None-Match` matches the image's `ETag`; 400 for a token that does not
|
||||
@@ -192,8 +196,9 @@ path under `/v1/` answers 200, in maintenance mode too.
|
||||
- `GET /.well-known/healthcheck.json` — JSON with `status` (`ok`), `now`,
|
||||
`uptime_seconds`, `uptime_human`, `version`, `appname` and
|
||||
`maintenance_mode`. Needs: nothing. Answers: 200, always.
|
||||
- `GET /static/<file>` — the script the login and generator pages load. Needs:
|
||||
nothing. Answers: 200, or 404 for a file that does not exist.
|
||||
- `GET /static/<file>` — the stylesheet and script the login and generator
|
||||
pages load. Needs: nothing. Answers: 200, or 404 for a file that does not
|
||||
exist.
|
||||
- `GET /metrics` — Prometheus metrics (see Architecture). Needs: HTTP basic
|
||||
authentication with `metrics.username` and `metrics.password`. Answers: 200;
|
||||
401 without them; 404 when they are not set, as the route then does not exist.
|
||||
@@ -391,6 +396,11 @@ and the URL is
|
||||
- **Suffix match**: `.example.com` — matches `cdn.example.com`,
|
||||
`images.example.com`, and `example.com`
|
||||
|
||||
An IP address is matched exactly; write an IPv6 address without brackets. An
|
||||
entry that is neither a host name (letters, digits, hyphens, underscores and
|
||||
dots, with at most one leading dot) nor an IP address, such as one with a port
|
||||
or a `*.` wildcard, aborts startup.
|
||||
|
||||
### Configuration
|
||||
|
||||
Every setting can be given as an environment variable, in a YAML config
|
||||
@@ -414,10 +424,10 @@ pixa finds: `/etc/pixa/config.yml`, `/etc/pixa/config.yaml`,
|
||||
`~/.config/pixa/config.yml`, `~/.config/pixa/config.yaml`, then `config.yml`
|
||||
and `config.yaml` in the working directory. A named file that does not exist,
|
||||
cannot be read or does not parse aborts startup. Of the files pixa looks for on
|
||||
its own, one it finds but cannot read or parse aborts startup; one it cannot
|
||||
find, for any reason, is passed over without a message, even when the file is
|
||||
there in a directory pixa may not enter. With no file, pixa uses the environment
|
||||
and the defaults.
|
||||
its own, only one that does not exist is passed over, without a message. One
|
||||
that pixa cannot read or parse aborts startup, naming the file. So does one in a
|
||||
directory pixa may not enter, whether or not it is there, since pixa cannot
|
||||
tell. With no file, pixa uses the environment and the defaults.
|
||||
|
||||
| Variable | Config key | Meaning |
|
||||
| ------------------------------------ | ------------------------------- | ---------------------------------------------------------------------------- |
|
||||
@@ -427,6 +437,7 @@ and the defaults.
|
||||
| `PIXA_DB_URL` | `db_url` | SQLite database URL; default `state.sqlite3` in the state directory |
|
||||
| `PIXA_CACHE_MAX_BYTES` | `cache_max_bytes` | Disk cache limit in bytes; `0` disables it; default 75% of (free + cached) |
|
||||
| `PIXA_ALLOWLIST_HOSTS` | `allowlist_hosts` | Upstream hosts served without a signature |
|
||||
| `PIXA_REFERER_BLOCKLIST` | `referer_blocklist` | Hosts whose pages the image routes refuse with 403, by `Referer` |
|
||||
| `PIXA_BLOCKED_NETWORKS` | `blocked_networks` | CIDR ranges never fetched from, on top of the built-in ones |
|
||||
| `PIXA_TRUSTED_PROXIES` | `trusted_proxies` | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 |
|
||||
| `PIXA_ALLOW_HTTP` | `allow_http` | Allow plain-HTTP upstreams, for testing only; default `false` |
|
||||
@@ -455,6 +466,18 @@ Key settings in more detail:
|
||||
that has no leading zero and is not the scheme's default. Any other value,
|
||||
including another scheme such as a browser extension's, aborts startup
|
||||
- `allowlist_hosts` — list of allowed upstream hosts
|
||||
- `referer_blocklist` — list of hosts whose pages may not show pixa's images, to
|
||||
stop other sites hotlinking them. Entries are written and matched as for
|
||||
`allowlist_hosts` (see Allowlist patterns), and an entry that is neither a
|
||||
host name nor an IP address aborts startup. A request to `/v1/image/` or
|
||||
`/v1/e/` whose `Referer` header names a listed host is refused with 403 before
|
||||
its signature or token is checked and before the cache or the upstream host is
|
||||
used, so it fetches nothing, and it is refused even when the image is cached.
|
||||
A request with no `Referer`, or one that does not parse as a URL
|
||||
with a host, is served, as many clients send none. So this is easily got
|
||||
around: a site whose pages send no `Referer` (for example with
|
||||
`Referrer-Policy: no-referrer`) is not stopped. It does not apply to the login
|
||||
and generator pages. Default: empty
|
||||
- `blocked_networks` — list of CIDR ranges to refuse for SSRF protection,
|
||||
added to the always-enforced built-in ranges (loopback, private,
|
||||
link-local, CGNAT, benchmark, NAT64, and the like); an invalid CIDR
|
||||
@@ -489,6 +512,12 @@ Key settings in more detail:
|
||||
waits for an upstream connection and for a processing slot (up to 10 seconds
|
||||
each), so keep it longer than `upstream_fetch_timeout` plus 20 seconds
|
||||
- `signing_key` — HMAC secret for URL signatures
|
||||
- `db_url` — the SQLite database to open; omitted, it is
|
||||
`file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)`, which keeps the
|
||||
database in WAL mode. pixa adds `_pragma=busy_timeout(5000)` to any `db_url`,
|
||||
so a write that finds another in progress waits up to five seconds for it
|
||||
instead of failing. WAL mode comes only from the URL: keep
|
||||
`_pragma=journal_mode(WAL)` in one you set
|
||||
- `cache_max_bytes` — disk cache size limit in bytes; `0` disables the
|
||||
disk cache entirely; omitted defaults to 75% of the sum of the free space on
|
||||
the filesystem containing `<state_dir>/cache/` and the bytes of source and
|
||||
@@ -513,7 +542,7 @@ Key settings in more detail:
|
||||
the container unhealthy, and upaas marks a deploy failed when its container
|
||||
is unhealthy. The login and URL generator pages and `/metrics` keep working
|
||||
|
||||
See `config.example.yml` for all options with defaults.
|
||||
See `configs/config.example.yml` for all options with defaults.
|
||||
|
||||
### Architecture
|
||||
|
||||
@@ -548,6 +577,8 @@ them. We provide:
|
||||
- `script/check` — run test, lint, and fmt-check
|
||||
- `script/docker` — build the Docker image tagged via `script/projectname`
|
||||
- `script/docker-smoke` — build the image, start it, wait for it to be healthy
|
||||
- `script/loadtest` — measure pixad's throughput, latency and peak memory; a
|
||||
benchmark, not part of `script/check` (see Load Test)
|
||||
- `script/cibuild` — CI entrypoint: `docker build .` with a new
|
||||
`CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of
|
||||
coming from the build cache, and a green run implies a green repo
|
||||
@@ -556,6 +587,58 @@ them. We provide:
|
||||
- `script/install-precommit` — install the git pre-commit hook that
|
||||
runs `script/precommit`
|
||||
|
||||
## Load Test
|
||||
|
||||
`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how
|
||||
much memory it uses. It is a benchmark, not a check: `script/check` does not run
|
||||
it. It needs Docker and Go.
|
||||
|
||||
```bash
|
||||
script/loadtest # 10 seconds per scenario, 4 clients
|
||||
script/loadtest 30s 32 # 30 seconds per scenario, 32 clients
|
||||
```
|
||||
|
||||
It builds the image with `script/docker` and the load tool,
|
||||
[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario
|
||||
starts a new pixad container and a new origin container, `cmd/loadtest-origin`:
|
||||
an upstream host that answers every path with the same generated 1600x1200 JPEG.
|
||||
vegeta then sends requests from the given number of clients, each sending its
|
||||
next request as soon as its last one is answered, all for an image resized to
|
||||
400x300 WebP:
|
||||
|
||||
- `hit`: the same image every time, put in the cache first;
|
||||
- `miss`: a new source image every time, so pixad fetches and converts each one;
|
||||
- `herd`: each new source image once per client in a row, so that all clients
|
||||
ask for it at the same time and share one fetch and one conversion (see
|
||||
Routes).
|
||||
|
||||
pixad refuses upstream hosts with private or local addresses, so the containers
|
||||
share a Docker network in `203.0.113.0/24`, a range set aside for documentation.
|
||||
A second run on the same Docker host while one is going fails, as it cannot
|
||||
create that network.
|
||||
|
||||
For each scenario the script prints vegeta's report and two lines of its own:
|
||||
|
||||
- `Requests [total, rate, throughput]`: the requests sent, how many were sent
|
||||
per second, and how many were answered successfully per second; the last is
|
||||
the number to compare with the target under Storage;
|
||||
- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request
|
||||
to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th
|
||||
percentiles;
|
||||
- `Status Codes` and `Error Set`: anything other than `200` means the other
|
||||
numbers are not for the scenario described, such as `503` when pixad was busy;
|
||||
- `Bytes In`: `0`, as vegeta is told not to keep the images it receives;
|
||||
- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since
|
||||
its container started, in kB; for `hit` it includes the request that put the
|
||||
image in the cache;
|
||||
- `requests to the origin`: the fetches pixad made: one for `hit`, one per
|
||||
request for `miss`, and one per image for `herd`, that is the requests sent
|
||||
divided by the number of clients.
|
||||
|
||||
The numbers depend on the machine and on whatever else runs on it. The first
|
||||
measurement, made on a shared machine with few clients, is in `TODO.md`; it says
|
||||
nothing about the target.
|
||||
|
||||
## TODO
|
||||
|
||||
See [TODO.md](TODO.md) for the full prioritized task list.
|
||||
|
||||
+270
-75
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-07-06
|
||||
last_modified: 2026-09-08
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
@@ -60,17 +60,28 @@ style conventions are in separate documents:
|
||||
prerequisite since nvm requires bash. yarn is then pinned via
|
||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||
scripts are our own extensions to the standard: `script/check` runs
|
||||
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||
what the git pre-commit hook runs, and it calls `script/check`;
|
||||
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||
target shims to it); and `script/projectname` (literally that filename) simply
|
||||
outputs the project's name. Scripts that need the name call
|
||||
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||
`script/precommit`, not in the hook itself. Model scripts are at
|
||||
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
||||
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
||||
pristine checkout with nothing installed the run dies there, after the
|
||||
containerised gates have passed. **The bootstrap alone is not enough**:
|
||||
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
||||
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
||||
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
||||
source nvm for the pinned node version before invoking it, exactly as
|
||||
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
||||
docker and git then gets through `script/check`. Four further scripts are our
|
||||
own extensions to the standard: `script/check` runs `script/test`,
|
||||
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
||||
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
||||
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
||||
`script/projectname` (literally that filename) simply outputs the project's
|
||||
name. Scripts that need the name call `script/projectname` — e.g.
|
||||
`script/docker` assembles its image tag from it — so those scripts stay
|
||||
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
||||
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
||||
the hook itself. Model scripts are at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||
must document the provided scripts in an **Entrypoints** section (see the
|
||||
README requirements below).
|
||||
@@ -89,87 +100,140 @@ style conventions are in separate documents:
|
||||
contributor should be able to understand the entire development workflow by
|
||||
reading the Makefile.
|
||||
|
||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||
as a build step so the build fails if the branch is not green. For non-server
|
||||
repos, the Dockerfile should bring up a development environment and run
|
||||
`make check`. For server repos, `make check` should run as an early build
|
||||
stage before the final image is assembled. Dockerfiles install development
|
||||
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||
layer stays cached until dependencies change.
|
||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
||||
image cannot be built unless they pass. For non-server repos the final stage
|
||||
brings up a development environment; for server repos it is the runtime image.
|
||||
Dockerfiles install development prerequisites by running `script/bootstrap`
|
||||
rather than duplicating installs inline; COPY `script/` and the dependency
|
||||
manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before
|
||||
running it.
|
||||
|
||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||
repos use a multistage build where linting runs in an independent stage based
|
||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||
then declares an explicit dependency on the lint stage via
|
||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||
linting before proceeding to compilation and tests. This ensures lint failures
|
||||
surface in seconds rather than minutes, without blocking on dependency
|
||||
download or compilation in the build stage.
|
||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
||||
and nothing else:
|
||||
|
||||
The standard pattern for a Go repo Dockerfile is:
|
||||
```sh
|
||||
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
|
||||
docker build --no-cache --target test -t "$(script/projectname)-test" .
|
||||
```
|
||||
|
||||
**A stage that is not the last one in the file is built only when the final
|
||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
||||
two gates are always invoked by name here, and why the final stage carries a
|
||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
||||
and tested nothing.
|
||||
|
||||
**Every `docker build` in `script/` is tagged**, here and in
|
||||
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
|
||||
image behind on every invocation, on every developer host and every CI
|
||||
runner; a tagged one replaces the previous image.
|
||||
|
||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
||||
themselves a `docker build` and would recurse into a daemon that does not
|
||||
exist in a build step. Formatting is the exception and stays on the host:
|
||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
||||
read-only twin.
|
||||
|
||||
**No lint verdict may come from a host invocation of the linter.** On a
|
||||
shared host golangci-lint reads a result cache keyed on file content rather
|
||||
than location, so a second checkout of the same content is served the first
|
||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
||||
a digest-pinned binary, so neither is reachable.
|
||||
|
||||
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
||||
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
||||
the check `RUN` is served from cache, nothing executes, and the build still
|
||||
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
||||
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
||||
four, and there is no fifth — `script/check` runs the two gate phases and
|
||||
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
||||
not evidence that anything ran: a sub-second build reporting success is a
|
||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
||||
and friends destroy a build cache shared with every other build on the host.
|
||||
|
||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||
hash), so lint failures surface in seconds rather than after a full compile,
|
||||
and the test phase is based on the Go image. The canonical Go repo
|
||||
`Dockerfile`:
|
||||
|
||||
```dockerfile
|
||||
# Lint stage — fast feedback on formatting and lint issues
|
||||
# Lint phase
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Build stage
|
||||
# Test phase
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed.
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
WORKDIR /src
|
||||
|
||||
# Force BuildKit to run the lint stage before proceeding
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make test
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage
|
||||
# Runtime stage, and the last one
|
||||
FROM alpine@sha256:...
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
|
||||
Key points:
|
||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||
includes both Go and the linter), so there is no need to install the
|
||||
linter separately.
|
||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||
this line, the build stage would not wait for lint to finish and a lint
|
||||
failure might not fail the overall build.
|
||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
||||
both Go and the linter), so nothing needs installing.
|
||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
||||
and a stage nothing depends on is not built at all, so without these two
|
||||
lines a red gate would not fail the build.
|
||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||
same two copies. A plain `docker build .` builds the last stage's chain
|
||||
and nothing else.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
The lint stage should not depend on the actual build output — it exists to
|
||||
fail fast.
|
||||
- If the project requires CGO or system libraries for linting (e.g.
|
||||
`vips-dev`), install them in the lint stage with `apk add`.
|
||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||
build stage, not the lint stage, because they may require compiled
|
||||
artifacts or heavier dependencies.
|
||||
`vips-dev`), install them in the lint phase with `apk add`.
|
||||
- `ARG VERSION=dev` is declared in the stage that compiles and supplied by
|
||||
`script/docker` and `script/cibuild`; no stage may call `git describe`.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||
Dockerfile already runs `make check`, a successful build implies all checks
|
||||
pass.
|
||||
runs `script/cibuild` on push, and checks out the repo as its only other step.
|
||||
That script bootstraps, runs the gate phases, and then builds the image, so a
|
||||
successful run means every check passed; a bare `docker build .` does not
|
||||
carry the same guarantee, because its gate phases may come from the cache. The
|
||||
image build is uncached and so runs the gate phases a second time. That is the
|
||||
price of the rule above, and it is worth paying: the image that ships is built
|
||||
from a run of its own gates rather than from a cache entry.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
@@ -189,14 +253,21 @@ style conventions are in separate documents:
|
||||
module under test to verify it compiles/parses. There is no excuse for
|
||||
`make test` to be a no-op.
|
||||
|
||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||
Makefile.
|
||||
- `make test` must complete in under 60 seconds. That is the hard cap, and a
|
||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
||||
20 and 60 seconds is still green, but the overage must be filed as an
|
||||
improvement bug against that repo. Add a 90-second timeout to the test
|
||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
||||
one.
|
||||
|
||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||
show full output. This keeps CI logs and `docker build` output clean on
|
||||
success (just package/suite summaries) while providing full diagnostic detail
|
||||
on failure (every test case, every assertion). The general shell pattern:
|
||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
||||
on success (just package/suite summaries) while providing full diagnostic
|
||||
detail on failure (every test case, every assertion). The command lives in the
|
||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
||||
Makefile form below is the same pattern for any repo-local invocation:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@@ -209,11 +280,24 @@ style conventions are in separate documents:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@go test -timeout 30s -race -cover ./... || \
|
||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 30s -race -v ./...; exit 1; }
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
```
|
||||
|
||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
||||
cache, so the target cannot report a pass it did not earn, and the rerun
|
||||
reproduces a failure instead of replaying it. It leaves the build cache
|
||||
alone, so it costs the runtime of the suite and no recompilation.
|
||||
|
||||
Note that this is a second, independent cache, stacked below the Docker
|
||||
layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26)
|
||||
addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes;
|
||||
it does not guarantee `go test` inside that step does any work, because the
|
||||
`GOCACHE` baked into earlier image layers survives into the re-executed
|
||||
step. They are two separate defects requiring two separate fixes, and a fix
|
||||
for one must not be recorded as covering the other.
|
||||
|
||||
Python example:
|
||||
|
||||
```makefile
|
||||
@@ -239,10 +323,83 @@ style conventions are in separate documents:
|
||||
must be in `.gitignore`. No exceptions.
|
||||
|
||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||
Fetch the standard `.gitignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||
a new repo.
|
||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
||||
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
|
||||
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
|
||||
setting up a new repo. These patterns are written to `.gitignore`'s own
|
||||
semantics, in which an unanchored pattern already matches at every depth; they
|
||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
||||
|
||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
||||
across unmodified leaves secrets in the build context.** Docker matches with
|
||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
||||
therefore excludes only the copies at the repository root, while `config/.env`
|
||||
and `certs/server.key` still reach the context and can land in an image layer
|
||||
— which is more dangerous than a short file with no secret patterns at all,
|
||||
because it reads as solved and stops anyone looking. Give every
|
||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
||||
root-anchored entries unprefixed: `.git`, and the repo's own host-built
|
||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
||||
also catches something the build needs, re-include it with a negation
|
||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
||||
other file it covers. Fetch the standard `.dockerignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
||||
it with the repo's own artifacts.
|
||||
|
||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
||||
additional checkout of the repo — so under `COPY . .` the build context
|
||||
inflates by a multiple of the repo and another session's unreviewed work can
|
||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
||||
prefix, because the prefixed form would also delete any nested directory of
|
||||
that name from the build. Anchoring carries a known gap that the canonical
|
||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
||||
file and not the tracker: the directory is created in the agent's working
|
||||
directory, so a repo running agents in subdirectories still ships
|
||||
`services/api/.claude/` and must add its own anchored entry there.
|
||||
|
||||
- **Excluding `.git` means `git describe` cannot run inside any build stage, and
|
||||
it fails quietly there.** In a build stage there is no repository, so
|
||||
`git describe` writes nothing to stdout, `-X main.Version=` comes out empty,
|
||||
the binary reports no version at all, and the build still exits 0. Compute the
|
||||
version on the host and thread it in as a build arg. `script/docker` and
|
||||
`script/cibuild` do this, byte-identically across repos:
|
||||
|
||||
```sh
|
||||
# Own line: a failing command substitution inside an argument does not
|
||||
# trip `set -e`, so the inline form degrades to an empty constant.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$(script/projectname)" .
|
||||
```
|
||||
|
||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
||||
fallback is applied — a live check that fires on a build from an export with
|
||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
||||
Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared
|
||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
||||
the scripts stay byte-identical. One consequence for CI: the standard
|
||||
checkout action clones shallow and fetches no tags, so a repo that embeds a
|
||||
tag-derived version must set `fetch-depth: 0` on its checkout step.
|
||||
|
||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
||||
a probe image that does `COPY . .`, and list what actually landed
|
||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
||||
transfers only the delta from the previous build.
|
||||
|
||||
- **No build artifacts in version control.** Code-derived data (compiled
|
||||
bundles, minified output, generated assets) must never be committed to the
|
||||
@@ -258,9 +415,45 @@ style conventions are in separate documents:
|
||||
- Make all changes on a feature branch. You can do whatever you want on a
|
||||
feature branch.
|
||||
|
||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||
manually by the user. Fetch from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
||||
_NEVER_ be modified by an agent: fetch it from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
||||
because it cannot be written once for every repo: the `deny` list of the
|
||||
`test-support` depguard rule, where a repo names its own test-support packages
|
||||
by full import path. A repo adds entries there and changes nothing else, and a
|
||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
||||
v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base
|
||||
image
|
||||
(`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`,
|
||||
which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the
|
||||
only pin, since no repo installs golangci-lint on the host: bumping the
|
||||
version means changing it and nothing else.
|
||||
|
||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
||||
disagree about what the tool even is. The canonical form:
|
||||
- compares the installed version against the pin over the **whole** version
|
||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
||||
running `2.12.2-rc1` and skips the install;
|
||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
||||
mismatch, so the failure direction is a redundant install and never a
|
||||
skipped one;
|
||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
||||
then through `PATH`, not through the directory the installer wrote to —
|
||||
and fails naming the resolved path, since an install that a shadowing
|
||||
binary hides succeeds while changing nothing any caller sees;
|
||||
- is actually called, and prints the version on both success paths: a
|
||||
function defined and never invoked has the same exit status and the same
|
||||
empty output as one that worked.
|
||||
|
||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
@@ -379,7 +572,9 @@ style conventions are in separate documents:
|
||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||
subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints
|
||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
||||
`cmd/`
|
||||
- `configs/` — configuration templates and examples
|
||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||
- `docs/` — documentation and markdown (README.md stays in root)
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
* branch per issue from `next`
|
||||
* do the work in Next Step
|
||||
* move Next Step to the top of Completed Steps
|
||||
* `TODO.md` merges with git's union merge (`.gitattributes`), which never
|
||||
reports a conflict: read the merged entries after every merge or rebase
|
||||
* move the top item of Future Steps into Next Step
|
||||
* commit (`TODO.md` changes in the same commit as the work)
|
||||
* open a PR based on `next`
|
||||
@@ -25,10 +27,132 @@ The disk cache is now size-bounded with LRU eviction
|
||||
|
||||
# Next Step
|
||||
|
||||
P2: security: referer blacklist
|
||||
P2: security: per-IP rate limiting on the image routes
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]`
|
||||
(`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check`
|
||||
does not run, measures three scenarios, each against a new pixad container and
|
||||
a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss`
|
||||
(a new source image every request) and `herd` (each new source image asked for
|
||||
by all clients at once). For each it prints vegeta's report (requests per
|
||||
second, latency percentiles, status codes), pixad's peak resident memory and
|
||||
the requests that reached the origin. `README.md` says how to run it and read
|
||||
it, and keeps 1-5k r/s as a target not yet measured. First measurement, with
|
||||
the defaults on a shared 48-CPU machine with other work running: a baseline
|
||||
for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95
|
||||
8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so
|
||||
not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100
|
||||
MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms,
|
||||
peak 60 MiB, 188 fetches for 749 requests.
|
||||
- 2026-10-04 `cmd/pixad/main.go` is one call into `internal/` (closes #206):
|
||||
what it did (the command line and its `--config` flag, setting
|
||||
`PIXA_CONFIG_PATH`, ignoring `SIGPIPE`, starting the fx app) is now `Run` in
|
||||
`internal/app`, unchanged, and `main` calls it with `Version`, which the build
|
||||
still sets through `-X main.Version`. That code had no tests to move.
|
||||
- 2026-10-04 `.gitignore` ignores `.claude/` (closes #204): the entry and its
|
||||
comment are copied from the canonical `.gitignore` in `sneak/prompts`,
|
||||
unanchored so it matches at every depth. `.dockerignore` already has
|
||||
`.claude`.
|
||||
- 2026-10-04 `.dockerignore` keeps secrets out at every depth (closes #205): the
|
||||
file is now the standard one from `sneak/prompts`, whose patterns match in
|
||||
every directory and, for environment files and private keys, in any letter
|
||||
case, so a nested `.env` or `server.key` no longer reaches the build context.
|
||||
pixa still sends `.git` without `.git/config` in place of the standard file's
|
||||
`.git` line, and still leaves out `.gitignore`, `/bin` and `/data`.
|
||||
- 2026-10-04 `REPO_POLICIES.md` matches the canonical copy again (closes #196):
|
||||
it is replaced, unchanged, by `prompts/REPO_POLICIES.md` from `sneak/prompts`
|
||||
`main`. The rules it adds that pixa's tree breaks are filed:
|
||||
https://git.eeqj.de/sneak/pixa/issues/202 (lint and tests as `Dockerfile`
|
||||
phases built with `--no-cache`), https://git.eeqj.de/sneak/pixa/issues/203
|
||||
(the workflow's `script/docker-smoke` step),
|
||||
https://git.eeqj.de/sneak/pixa/issues/204 (`.claude/` in `.gitignore`),
|
||||
https://git.eeqj.de/sneak/pixa/issues/205 (`.dockerignore` patterns at every
|
||||
depth), https://git.eeqj.de/sneak/pixa/issues/206 (a thin
|
||||
`cmd/pixad/main.go`) and https://git.eeqj.de/sneak/pixa/issues/208
|
||||
(`fetch-depth: 0` on the CI checkout, so the build sees the tags). Its rule
|
||||
that no build stage runs `git describe` is not followed: pixa takes the
|
||||
version from the `.git` in the build context, per
|
||||
https://git.eeqj.de/sneak/pixa/issues/166, as the copy on `sneak/prompts`
|
||||
`next` already says.
|
||||
- 2026-10-04 an integration test of the image proxy flow (closes #80):
|
||||
`TestImageProxyFlow` in `internal/server` starts the database, handlers and
|
||||
middleware from the constructors `pixad` uses, with a fresh state directory,
|
||||
and replaces only the upstream origin with a local test server. For a resize
|
||||
with a change to JPEG and for `orig`, the first request goes through the
|
||||
router, the real fetcher, libvips, the disk cache and SQLite and answers 200
|
||||
with the right content type and size and `X-Pixa-Cache: MISS`; the second
|
||||
answers `HIT` with the same image and the upstream has had one request; the
|
||||
source and the converted image are then in `cache/sources` and
|
||||
`cache/variants`, with their rows in `source_content`, `source_metadata` and
|
||||
`variant_content`. Two optional fields make this possible, which `pixad` does
|
||||
not set and the config file and environment cannot:
|
||||
`httpfetcher.Config.DialContext` connects in place of the dialer that refuses
|
||||
internal addresses, the URL and redirect checks still running, and
|
||||
`handlers.Params.Fetcher` replaces the fetcher the handlers build.
|
||||
- 2026-10-04 a URL made on the generator page with a `ttl` is tested to
|
||||
expire (closes #199): a new test in `internal/handlers` makes a URL on the
|
||||
generator page with a `ttl` of one second, checks that `/v1/e/` serves it at
|
||||
once, waits two seconds and checks that it then answers 410. The test waits
|
||||
for real, as pixa reads the clock directly when it makes and checks a URL; it
|
||||
waits two seconds because the time a URL expires is kept in whole seconds.
|
||||
Test only.
|
||||
- 2026-10-04 referer blocklist (closes #90): `referer_blocklist`
|
||||
(`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for
|
||||
`allowlist_hosts` with the same matcher; an entry of either list that is
|
||||
neither a host name (letters, digits, hyphens, underscores and dots, with at
|
||||
most one leading dot) nor an IP address, such as one with a port or a `*.`
|
||||
wildcard, aborts startup naming the setting and the entry. Both image routes
|
||||
refuse a request whose `Referer` names a listed host with 403 and a JSON error
|
||||
before the signature, the cache and the upstream fetch, so it fetches nothing
|
||||
and is refused whether or not the image is cached. A request with no
|
||||
`Referer`, or one that does not parse as a URL with a host, is served, so the
|
||||
list is easily got around; `README.md` and `configs/config.example.yml` say
|
||||
so. It does not apply to the login and generator pages.
|
||||
- 2026-10-04 fewer files in the repository root (closes #97):
|
||||
`config.example.yml` moved unchanged to `configs/config.example.yml`, and
|
||||
`README.md`, the comments in `internal/config/config.go` and the startup error
|
||||
for the placeholder signing key name the new path; `scripts/manual-test.sh`
|
||||
and its directory are deleted, as the handler tests in `internal/handlers`
|
||||
cover every check it made except two: fetching a real image from the
|
||||
internet, and a URL made on the generator page with a `ttl` answering 410 once
|
||||
the `ttl` has passed (https://git.eeqj.de/sneak/pixa/issues/199);
|
||||
`CONVENTIONS.md` is deleted, as `REPO_POLICIES.md` links the canonical Go HTTP
|
||||
server conventions.
|
||||
- 2026-10-04 SQLite writes no longer fail with "database is locked" (closes
|
||||
#198): pixa adds `_pragma=busy_timeout(5000)` to every `db_url`, so a write
|
||||
that finds another in progress on another connection waits up to five seconds
|
||||
for it, and the default `db_url` turns on WAL mode with
|
||||
`_pragma=journal_mode(WAL)`. The old default's `_journal_mode=WAL` is not a
|
||||
parameter the driver reads, so the database was never in WAL mode.
|
||||
- 2026-10-04 `TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup`
|
||||
only passes through a periodic pass (closes #189): it slept for three
|
||||
eviction intervals before writing its file, and a startup pass still running
|
||||
then could adopt the file itself. It now holds the test database's only
|
||||
connection until the startup pass waits for it after walking the empty
|
||||
variant directory, writes the file and lets the connection go, as
|
||||
`TestEvictionRunsOnPeriodicSchedule` does, so only a periodic reconciliation
|
||||
pass can adopt the file. Test only.
|
||||
- 2026-10-04 logging in, logging out, the URL generator and `/v1/e/` have
|
||||
handler tests (closes #77): new tests in `internal/handlers`, with no
|
||||
network, check that `GET /` without a login session shows the login form; a
|
||||
wrong key shows it again with an error and sets no session cookie; the right
|
||||
key answers 303 to `/` with a session cookie marked `Secure`, `HttpOnly` and
|
||||
`SameSite=Strict`, with which `GET /` shows the generator page; `GET /logout`
|
||||
answers 303 to `/` with an empty session cookie sent with `Max-Age=0`;
|
||||
`POST /generate` without a login session answers 303 to `/`; `/v1/e/` serves
|
||||
the image for a valid token, answers 410 for an expired one and 400 for one
|
||||
with a character changed, cut short or made with another signing key; and a
|
||||
URL made on the generator page is served by `/v1/e/`. No code changes.
|
||||
- 2026-10-04 `TODO.md` merges with git's union merge (closes #190): a root
|
||||
`.gitattributes`, copied from `sneak/prompts`, marks it `merge=union`, so two
|
||||
branches that each add an entry at the top of Completed Steps merge without a
|
||||
conflict and keep both entries. Git now never reports a conflict in
|
||||
`TODO.md`: a real one keeps both versions of the lines, and two entries that
|
||||
share an identical line can end up one inside the other, which a rebase can
|
||||
do to an entry already on `next`. The Workflow above says to read the merged
|
||||
entries after every merge or rebase.
|
||||
- 2026-10-04 the default `cache_max_bytes` no longer shrinks as the cache fills
|
||||
(closes #184): for an omitted key, the cache works out the limit when it
|
||||
opens, after the database is open, as 75% of the sum of the free space on the
|
||||
@@ -46,6 +170,25 @@ P2: security: referer blacklist
|
||||
files and the eviction pass after it evicts them. No other test in
|
||||
`internal/imgcache` inserts a row by hand after starting the evictor. Test
|
||||
only.
|
||||
- 2026-10-04 a config file pixa cannot read aborts startup (closes #176): of the
|
||||
places pixa looks for its config file on its own, only one where the file does
|
||||
not exist is passed over; any other error, such as a directory on the path
|
||||
that pixa may not enter, aborts startup naming the file, as a file that does
|
||||
not parse already did.
|
||||
- 2026-10-04 `.golangci.yml` re-vendored from the canonical copy (closes #57):
|
||||
the deprecated `gomodguard` is switched off, so lint runs print no
|
||||
deprecation warning; its successor `gomodguard_v2` runs with the shared
|
||||
module block list, and `depguard` keeps `net/http/httptest` out of files that
|
||||
are not tests. The tree needed no code changes.
|
||||
- 2026-10-04 the Content-Security-Policy allows no inline script or style
|
||||
(closes #125): `script-src` and `style-src` are `'self'` only. The generator
|
||||
page's two inline `onclick` handlers moved into
|
||||
`internal/static/generator.js`, attached with `addEventListener`; the bundled
|
||||
Tailwind script, which built styles in the browser, is replaced by a small
|
||||
hand-written `internal/static/style.css` with only the rules the login and
|
||||
generator pages use, the templates carrying a few plain class names in place
|
||||
of Tailwind's. No build step. The pages keep their layout, not every pixel of
|
||||
it.
|
||||
- 2026-10-04 deployment guide and example Caddy config (closes #89):
|
||||
"Deployment" in `README.md` says what the reverse proxy in front of pixa must
|
||||
do (terminate TLS; pass `Host`, `Origin` and `Referer` on unchanged; set
|
||||
@@ -497,7 +640,6 @@ P2: security: referer blacklist
|
||||
# Future Steps
|
||||
|
||||
- P2: security
|
||||
- per-IP rate limiting on the image routes
|
||||
- per-origin rate limiting
|
||||
- P2: HTTP response handling
|
||||
- Last-Modified headers
|
||||
@@ -509,5 +651,5 @@ P2: security: referer blacklist
|
||||
- optional Sentry error reporting
|
||||
- comprehensive request logging
|
||||
- Prometheus performance metrics
|
||||
- integration tests for the image proxy flow
|
||||
- load tests to verify the 1k to 5k req/s target
|
||||
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
||||
shared with other work
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
// Command loadtest-origin is the upstream host script/loadtest points pixad
|
||||
// at; internal/loadtestorigin says what it does.
|
||||
package main
|
||||
|
||||
import "sneak.berlin/go/pixa/internal/loadtestorigin"
|
||||
|
||||
func main() {
|
||||
loadtestorigin.Run()
|
||||
}
|
||||
+2
-61
@@ -1,69 +1,10 @@
|
||||
// Package main is the entry point for the pixad image proxy server.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
"sneak.berlin/go/pixa/internal/server"
|
||||
)
|
||||
import "sneak.berlin/go/pixa/internal/app"
|
||||
|
||||
var Version string //nolint:gochecknoglobals // set by ldflags
|
||||
|
||||
var configPath string //nolint:gochecknoglobals // cobra flag
|
||||
|
||||
func main() {
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "pixad",
|
||||
Short: "Pixa image caching proxy server",
|
||||
Run: run,
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(_ *cobra.Command, _ []string) {
|
||||
globals.Version = Version
|
||||
|
||||
// Set config path in environment if specified via flag
|
||||
if configPath != "" {
|
||||
_ = os.Setenv("PIXA_CONFIG_PATH", configPath)
|
||||
}
|
||||
|
||||
// A write to a closed stdout or stderr must not end the process.
|
||||
signal.Ignore(syscall.SIGPIPE)
|
||||
|
||||
fx.New(
|
||||
fx.Provide(
|
||||
config.New,
|
||||
database.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
logger.New,
|
||||
server.New,
|
||||
middleware.New,
|
||||
healthcheck.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
func(log *logger.Logger) { log.Identify() },
|
||||
func(*server.Server) {},
|
||||
),
|
||||
).Run()
|
||||
app.Run(Version)
|
||||
}
|
||||
|
||||
@@ -34,9 +34,10 @@ maintenance_mode: false
|
||||
state_dir: ./data
|
||||
|
||||
# SQLite database URL (default:
|
||||
# file:<state_dir>/state.sqlite3?_journal_mode=WAL). An empty value aborts
|
||||
# startup; leave the key out to use the default.
|
||||
# db_url: "file:./data/state.sqlite3?_journal_mode=WAL"
|
||||
# file:<state_dir>/state.sqlite3?_pragma=journal_mode(WAL)). pixa adds
|
||||
# _pragma=busy_timeout(5000) to it. An empty value aborts startup; leave the
|
||||
# key out to use the default.
|
||||
# db_url: "file:./data/state.sqlite3?_pragma=journal_mode(WAL)"
|
||||
|
||||
# Image proxy settings
|
||||
# HMAC signing key for URL signatures (required, at least 32 characters)
|
||||
@@ -45,6 +46,8 @@ signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||
|
||||
# Hosts that don't require signatures (default: none)
|
||||
# Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
|
||||
# An entry that is neither a host name nor an IP address (IPv6 without
|
||||
# brackets), such as one with a port or a "*." wildcard, aborts startup.
|
||||
allowlist_hosts:
|
||||
- s3.sneak.cloud
|
||||
- static.sneak.cloud
|
||||
@@ -52,6 +55,16 @@ allowlist_hosts:
|
||||
- github.com
|
||||
- user-images.githubusercontent.com
|
||||
|
||||
# Hosts whose pages may not show pixa's images, written as for
|
||||
# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
|
||||
# names one of them is answered 403 before anything is fetched, even when
|
||||
# the image is cached. A request with no Referer, or one that does not
|
||||
# parse, is served, so a site whose pages send no Referer is not stopped.
|
||||
# The login and generator pages are not covered. (default: none)
|
||||
# referer_blocklist:
|
||||
# - leech.example
|
||||
# - .hotlinker.example
|
||||
|
||||
# Additional CIDR ranges to refuse when fetching upstream, extending the
|
||||
# SSRF protection. These are added to the always-enforced built-in ranges
|
||||
# (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and
|
||||
@@ -0,0 +1,70 @@
|
||||
// Package app reads the pixad command line and runs the server.
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
"sneak.berlin/go/pixa/internal/server"
|
||||
)
|
||||
|
||||
var configPath string //nolint:gochecknoglobals // cobra flag
|
||||
|
||||
// Run reads the command line and runs the server until it stops, with
|
||||
// version as the version pixad logs and reports. It exits the process
|
||||
// with status 1 when the command line is not valid.
|
||||
func Run(version string) {
|
||||
globals.Version = version
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "pixad",
|
||||
Short: "Pixa image caching proxy server",
|
||||
Run: run,
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&configPath, "config", "c", "", "path to config file")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(_ *cobra.Command, _ []string) {
|
||||
// Set config path in environment if specified via flag
|
||||
if configPath != "" {
|
||||
_ = os.Setenv("PIXA_CONFIG_PATH", configPath)
|
||||
}
|
||||
|
||||
// A write to a closed stdout or stderr must not end the process.
|
||||
signal.Ignore(syscall.SIGPIPE)
|
||||
|
||||
fx.New(
|
||||
fx.Provide(
|
||||
config.New,
|
||||
database.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
logger.New,
|
||||
server.New,
|
||||
middleware.New,
|
||||
healthcheck.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
func(log *logger.Logger) { log.Identify() },
|
||||
func(*server.Server) {},
|
||||
),
|
||||
).Run()
|
||||
}
|
||||
+111
-51
@@ -4,16 +4,19 @@ package config
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
@@ -46,6 +49,7 @@ const (
|
||||
keyMetricsPassword = "metrics.password"
|
||||
keySigningKey = "signing_key"
|
||||
keyAllowlistHosts = "allowlist_hosts"
|
||||
keyRefererBlocklist = "referer_blocklist"
|
||||
keyAllowHTTP = "allow_http"
|
||||
keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
|
||||
keyUpstreamConnections = "upstream_connections"
|
||||
@@ -60,7 +64,7 @@ const (
|
||||
)
|
||||
|
||||
// placeholderSigningKey is the dummy signing_key shipped in
|
||||
// config.example.yml. It is 45 characters, so it passes the length
|
||||
// configs/config.example.yml. It is 45 characters, so it passes the length
|
||||
// check, but it is public in this repository and must be rejected at
|
||||
// startup so no deployment ever signs URLs with it.
|
||||
const placeholderSigningKey = "CHANGE_ME_generate_with_openssl_rand_base64_32"
|
||||
@@ -86,7 +90,7 @@ var (
|
||||
errMustBeAtLeastOne = errors.New("must be at least 1")
|
||||
errValueTooShort = errors.New("value too short")
|
||||
errPlaceholderKey = errors.New(
|
||||
"is the placeholder from config.example.yml; " +
|
||||
"is the placeholder from configs/config.example.yml; " +
|
||||
"generate a real key with: openssl rand -base64 32")
|
||||
errMustBeSetTogether = errors.New("must be set together")
|
||||
errMustNotBeNegative = errors.New("must not be negative")
|
||||
@@ -95,12 +99,11 @@ var (
|
||||
"value is null; omit the key entirely to use the default")
|
||||
errValuesNull = errors.New(
|
||||
"value is null; omit a key entirely to use its default")
|
||||
errNotBareHostname = errors.New(
|
||||
"must be a bare hostname without scheme, path, or whitespace")
|
||||
errNoHostnameLabels = errors.New("contains no hostname labels")
|
||||
errNotADuration = errors.New("not a duration such as 30s or 2m")
|
||||
errMustBePositive = errors.New("must be positive")
|
||||
errNotAnOrigin = errors.New(
|
||||
errNotAHost = errors.New("must be a host name such as " +
|
||||
"cdn.example.com or .example.com, or an IP address")
|
||||
errNotADuration = errors.New("not a duration such as 30s or 2m")
|
||||
errMustBePositive = errors.New("must be positive")
|
||||
errNotAnOrigin = errors.New(
|
||||
`not "*" or an origin such as https://example.com`)
|
||||
)
|
||||
|
||||
@@ -128,6 +131,10 @@ type Config struct {
|
||||
AllowHTTP bool // Allow non-TLS upstream (testing only)
|
||||
UpstreamConnectionsPerHost int // Max concurrent connections per upstream host
|
||||
|
||||
// RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
|
||||
// image routes refuse a request whose Referer names a matching host.
|
||||
RefererBlocklist []string
|
||||
|
||||
// UpstreamConnections is the most concurrent connections to all
|
||||
// upstream hosts together, on top of the per-host limit.
|
||||
// MaxConcurrentProcessing is the most images processed at once.
|
||||
@@ -268,7 +275,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
}
|
||||
|
||||
loader := &strictLoader{sc: sc}
|
||||
|
||||
c := &Config{
|
||||
Debug: loader.boolVal(keyDebug, false),
|
||||
MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
|
||||
@@ -296,9 +302,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin),
|
||||
DownstreamTimeout: loader.durationVal(
|
||||
keyDownstreamTimeout, DefaultDownstreamTimeout),
|
||||
CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
|
||||
BlockedNetworks: blockedNetworks,
|
||||
TrustedProxies: trustedProxies,
|
||||
CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
|
||||
BlockedNetworks: blockedNetworks,
|
||||
TrustedProxies: trustedProxies,
|
||||
RefererBlocklist: loader.hostListVal(keyRefererBlocklist),
|
||||
}
|
||||
|
||||
// The default for an omitted cache_max_bytes is worked out when
|
||||
@@ -318,7 +325,8 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
settingName(keyDBURL), errValueEmpty)
|
||||
}
|
||||
|
||||
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_journal_mode=WAL", c.StateDir)
|
||||
// The driver sets the journal mode only through a _pragma parameter.
|
||||
c.DBURL = fmt.Sprintf("file:%s/state.sqlite3?_pragma=journal_mode(WAL)", c.StateDir)
|
||||
}
|
||||
|
||||
if loader.err != nil {
|
||||
@@ -417,7 +425,8 @@ func isKnownConfigKey(key string) bool {
|
||||
keyUpstreamConnectionsPerHost, keyUpstreamConnections,
|
||||
keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks,
|
||||
keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout,
|
||||
keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env":
|
||||
keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist,
|
||||
"env":
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -440,6 +449,7 @@ func envVarNames() map[string]string {
|
||||
keyMetricsPassword: "PIXA_METRICS_PASSWORD",
|
||||
keySigningKey: "PIXA_SIGNING_KEY",
|
||||
keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS",
|
||||
keyRefererBlocklist: "PIXA_REFERER_BLOCKLIST",
|
||||
keyAllowHTTP: "PIXA_ALLOW_HTTP",
|
||||
keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
|
||||
keyUpstreamConnections: "PIXA_UPSTREAM_CONNECTIONS",
|
||||
@@ -551,8 +561,8 @@ func (c *Config) ensureStateDirWritable() error {
|
||||
}
|
||||
|
||||
// validateSigningKey checks that the signing key is present, long
|
||||
// enough, and not the public placeholder from config.example.yml. The
|
||||
// key value itself is never echoed in error messages.
|
||||
// enough, and not the public placeholder from configs/config.example.yml.
|
||||
// The key value itself is never echoed in error messages.
|
||||
func (c *Config) validateSigningKey() error {
|
||||
if c.SigningKey == "" {
|
||||
return fmt.Errorf("%s: %w", settingName(keySigningKey), errValueRequired)
|
||||
@@ -609,7 +619,7 @@ func (c *Config) validate() error {
|
||||
}
|
||||
|
||||
for _, host := range c.AllowlistHosts {
|
||||
err := validateAllowlistHost(host)
|
||||
err := validateHostPattern(keyAllowlistHosts, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -732,25 +742,24 @@ func (c *Config) validateConcurrencyLimits() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAllowlistHost checks that an allowlist_hosts entry is a bare
|
||||
// hostname, optionally with a leading dot for suffix matching. URLs,
|
||||
// paths, and whitespace indicate a misconfigured entry. An entry with
|
||||
// no hostname labels (such as ".") is rejected: the allowlist matcher
|
||||
// treats a leading dot as a suffix pattern, so a bare "." would match
|
||||
// any upstream host written in FQDN trailing-dot form and effectively
|
||||
// disable URL signing.
|
||||
func validateAllowlistHost(host string) error {
|
||||
if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
|
||||
return fmt.Errorf("%s: entry %q %w",
|
||||
settingName(keyAllowlistHosts), host, errNotBareHostname)
|
||||
// hostNamePattern matches a host name: letters, digits, hyphens, underscores
|
||||
// and dots, optionally after one leading dot.
|
||||
var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
|
||||
|
||||
// validateHostPattern checks that an entry of the named key, allowlist_hosts
|
||||
// or referer_blocklist, is an IP address or a host name, the host name
|
||||
// optionally with one leading dot for suffix matching. Anything else, such as
|
||||
// a URL, a port or a "*." wildcard, can never match a host name that resolves,
|
||||
// so it is refused.
|
||||
// So is "." alone: the allowlist matcher would match it against any host
|
||||
// written with a trailing dot, which in allowlist_hosts disables URL signing.
|
||||
func validateHostPattern(key, host string) error {
|
||||
_, err := netip.ParseAddr(host)
|
||||
if err == nil || hostNamePattern.MatchString(host) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.Trim(host, ".") == "" {
|
||||
return fmt.Errorf("%s: entry %q %w",
|
||||
settingName(keyAllowlistHosts), host, errNoHostnameLabels)
|
||||
}
|
||||
|
||||
return nil
|
||||
return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
|
||||
}
|
||||
|
||||
// loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
|
||||
@@ -781,19 +790,27 @@ func loadConfigFile(log *slog.Logger, appName string) (*smartconfig.Config, erro
|
||||
for _, path := range configPaths {
|
||||
cleanPath := filepath.Clean(path)
|
||||
|
||||
// Only a config file that does not exist is skipped, including
|
||||
// one whose path runs through a file, such as under a HOME of
|
||||
// /dev/null. One that cannot be read or does not parse is a
|
||||
// fatal startup error.
|
||||
_, statErr := os.Stat(cleanPath)
|
||||
if statErr == nil {
|
||||
// A config file that exists but does not parse is a fatal
|
||||
// startup error, never something to skip over.
|
||||
sc, err := smartconfig.NewFromConfigPath(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
|
||||
}
|
||||
|
||||
log.Info("loaded config file", "path", path)
|
||||
|
||||
return sc, nil
|
||||
if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
|
||||
continue
|
||||
}
|
||||
|
||||
if statErr != nil {
|
||||
return nil, fmt.Errorf("failed to read config file %s: %w", path, statErr)
|
||||
}
|
||||
|
||||
sc, err := smartconfig.NewFromConfigPath(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
|
||||
}
|
||||
|
||||
log.Info("loaded config file", "path", path)
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
return nil, nil //nolint:nilnil // nil config is valid (use defaults)
|
||||
@@ -872,6 +889,19 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
|
||||
return val
|
||||
}
|
||||
|
||||
func (l *strictLoader) hostListVal(key string) []string {
|
||||
if l.err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
val, err := parseHostList(l.sc, key)
|
||||
if err != nil {
|
||||
l.err = err
|
||||
}
|
||||
|
||||
return val
|
||||
}
|
||||
|
||||
// getString returns the string value for key, or defaultVal if the key
|
||||
// is omitted. A present value that is not a string, or is explicitly
|
||||
// null, is an error.
|
||||
@@ -1169,7 +1199,7 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
return nil, errNullConfigValue(key)
|
||||
}
|
||||
|
||||
entries, err := cidrListEntries(raw, key)
|
||||
entries, err := listEntries(raw, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1189,11 +1219,41 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
// cidrListEntries extracts the raw entries of the named CIDR-list key as
|
||||
// trimmed, non-empty strings, from either a YAML list of strings or a
|
||||
// comma-separated string; an empty string is an empty list, as for
|
||||
// allowlist_hosts. Any other shape is a configuration error.
|
||||
func cidrListEntries(raw any, key string) ([]string, error) {
|
||||
// parseHostList parses the value of the named config key into host patterns,
|
||||
// or returns nil if the key is omitted. It accepts a YAML list of strings or a
|
||||
// comma-separated string. An explicitly null value, a wrong type, an empty
|
||||
// entry, a non-string entry, or an entry validateHostPattern rejects aborts
|
||||
// startup naming the key and the offending value.
|
||||
func parseHostList(sc *smartconfig.Config, key string) ([]string, error) {
|
||||
raw, ok := lookupValue(sc, key)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if raw == nil {
|
||||
return nil, errNullConfigValue(key)
|
||||
}
|
||||
|
||||
entries, err := listEntries(raw, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
err := validateHostPattern(key, entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// listEntries extracts the raw entries of the named list key as trimmed,
|
||||
// non-empty strings, from either a YAML list of strings or a comma-separated
|
||||
// string; an empty string is an empty list, as for allowlist_hosts. Any other
|
||||
// shape is a configuration error.
|
||||
func listEntries(raw any, key string) ([]string, error) {
|
||||
switch val := raw.(type) {
|
||||
case []any:
|
||||
entries := make([]string, 0, len(val))
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
|
||||
_ "modernc.org/sqlite" // SQLite driver registration
|
||||
)
|
||||
|
||||
// validTestSigningKey is a 32-character signing key that satisfies the
|
||||
@@ -94,12 +98,43 @@ func TestOmittedValuesUseDefaults(t *testing.T) {
|
||||
t.Errorf("AllowlistHosts = %v, want empty", c.AllowlistHosts)
|
||||
}
|
||||
|
||||
wantDBURL := "file:" + DefaultStateDir + "/state.sqlite3?_journal_mode=WAL"
|
||||
wantDBURL := "file:" + DefaultStateDir +
|
||||
"/state.sqlite3?_pragma=journal_mode(WAL)"
|
||||
if c.DBURL != wantDBURL {
|
||||
t.Errorf("DBURL = %q, want derived default %q", c.DBURL, wantDBURL)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultDBURLOpensTheDatabaseInWALMode opens the db_url derived from
|
||||
// state_dir with the SQLite driver pixad uses and checks that the database
|
||||
// is in WAL mode: the driver ignores any parameter it does not know.
|
||||
func TestDefaultDBURLOpensTheDatabaseInWALMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+"state_dir: "+t.TempDir()+"\n")
|
||||
if err != nil {
|
||||
t.Fatalf("config with only state_dir set should be valid, got: %v", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", c.DBURL)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open %q: %v", c.DBURL, err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
var journalMode string
|
||||
|
||||
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&journalMode)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the journal mode of %q: %v", c.DBURL, err)
|
||||
}
|
||||
|
||||
if journalMode != "wal" {
|
||||
t.Errorf("journal mode of %q = %q, want wal", c.DBURL, journalMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitValidValuesAreUsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -182,6 +217,25 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with
|
||||
// an underscore, which pixa can fetch from, is accepted as an entry.
|
||||
func TestAllowlistHostsAcceptsUnderscore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts:
|
||||
- my_bucket.example.com
|
||||
- .my_bucket.example.org
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("host names with an underscore should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"my_bucket.example.com", ".my_bucket.example.org"}
|
||||
if !slices.Equal(c.AllowlistHosts, want) {
|
||||
t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want)
|
||||
}
|
||||
}
|
||||
|
||||
// runAbortCases asserts that each case's config aborts startup with an
|
||||
// error message mentioning every expected substring.
|
||||
func runAbortCases(t *testing.T, cases []abortCase) {
|
||||
@@ -284,6 +338,20 @@ func invalidHostAndCredentialCases() []abortCase {
|
||||
keyAllowlistHosts, "example.com/images",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with wildcard",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyAllowlistHosts, "*.example.com",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with port",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyAllowlistHosts, "example.com:8443",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with whitespace",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
|
||||
@@ -564,6 +632,116 @@ func TestMalformedConfigFileAbortsStartup(t *testing.T) {
|
||||
t.Logf("got expected error: %v", err)
|
||||
}
|
||||
|
||||
// TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup checks that a
|
||||
// config file pixa cannot read because it may not enter its directory
|
||||
// aborts startup instead of being passed over.
|
||||
func TestConfigFileInDirectoryPixaMayNotEnterAbortsStartup(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root may enter any directory")
|
||||
}
|
||||
|
||||
home := t.TempDir()
|
||||
configDir := filepath.Join(home, ".config", "pixa-test-nonexistent-app")
|
||||
configPath := filepath.Join(configDir, "config.yml")
|
||||
|
||||
err := os.MkdirAll(configDir, 0o700)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create config directory: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(configPath, []byte(signingKeyLine), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
}
|
||||
|
||||
err = os.Chmod(configDir, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to remove the config directory's permissions: %v", err)
|
||||
}
|
||||
|
||||
// Give the directory back its permissions so t.TempDir can remove it.
|
||||
t.Cleanup(func() {
|
||||
//nolint:gosec // G302: a directory needs its execute bit to be removed
|
||||
_ = os.Chmod(configDir, 0o700)
|
||||
})
|
||||
|
||||
// The ~/.config candidate is the only one that exists: the appname
|
||||
// rules out /etc, and the working directory is empty.
|
||||
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||
t.Setenv("HOME", home)
|
||||
t.Chdir(t.TempDir())
|
||||
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||
if err == nil {
|
||||
t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
|
||||
sc)
|
||||
}
|
||||
|
||||
t.Logf("got expected error: %v", err)
|
||||
|
||||
if !strings.Contains(err.Error(), configPath) {
|
||||
t.Errorf("error %q does not name the config file %s", err.Error(), configPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigFileLinkingToItselfAbortsStartup checks that a config file
|
||||
// pixa cannot read for a reason other than not existing aborts startup,
|
||||
// as root too: a symbolic link to itself fails with "too many levels of
|
||||
// symbolic links".
|
||||
func TestConfigFileLinkingToItselfAbortsStartup(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
|
||||
err := os.Symlink("config.yml", filepath.Join(workDir, "config.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create symbolic link: %v", err)
|
||||
}
|
||||
|
||||
// Only the working directory's config.yml is there: the appname rules
|
||||
// out /etc, and HOME is empty.
|
||||
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||
t.Setenv("HOME", t.TempDir())
|
||||
t.Chdir(workDir)
|
||||
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||
if err == nil {
|
||||
t.Fatalf("config file pixa cannot read must abort startup, got config: %v",
|
||||
sc)
|
||||
}
|
||||
|
||||
t.Logf("got expected error: %v", err)
|
||||
|
||||
if !strings.Contains(err.Error(), "config.yml") {
|
||||
t.Errorf("error %q does not name the config file config.yml", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigPathThroughFileIsPassedOver checks that a config file path
|
||||
// that runs through a file, such as one under a HOME of /dev/null, is
|
||||
// passed over like one that does not exist, since no file can be there.
|
||||
func TestConfigPathThroughFileIsPassedOver(t *testing.T) {
|
||||
// No config file is there: the appname rules out /etc, HOME is
|
||||
// /dev/null, and the working directory is empty.
|
||||
t.Setenv("PIXA_CONFIG_PATH", "")
|
||||
t.Setenv("HOME", os.DevNull)
|
||||
t.Chdir(t.TempDir())
|
||||
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
sc, err := loadConfigFile(log, "pixa-test-nonexistent-app")
|
||||
if err != nil {
|
||||
t.Fatalf("a config path through a file must be passed over, got error: %v",
|
||||
err)
|
||||
}
|
||||
|
||||
if sc != nil {
|
||||
t.Errorf("expected no config file, got config: %v", sc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureStateDirCreatesDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
|
||||
t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example")
|
||||
t.Setenv("PIXA_ALLOW_HTTP", "true")
|
||||
t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
|
||||
t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
|
||||
@@ -93,6 +94,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
|
||||
MetricsPassword: "metricspass",
|
||||
SigningKey: validTestSigningKey,
|
||||
AllowlistHosts: []string{testHostS3, ".example.com"},
|
||||
RefererBlocklist: []string{"hotlinker.example", ".leech.example"},
|
||||
AllowHTTP: true,
|
||||
UpstreamConnectionsPerHost: 5,
|
||||
UpstreamConnections: 10,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
|
||||
// pattern starting with "." and checks both are kept in order.
|
||||
func TestRefererBlocklistParsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- leech.example
|
||||
- .hotlinker.example
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("valid referer_blocklist should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"leech.example", ".hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
|
||||
// the IPv6 one written without brackets, are accepted as entries.
|
||||
func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- 192.0.2.7
|
||||
- "2001:db8::7"
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("IP address entries should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"192.0.2.7", "2001:db8::7"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistAcceptsUnderscore checks that a host name with an
|
||||
// underscore, which a page can be served from, is accepted as an entry.
|
||||
func TestRefererBlocklistAcceptsUnderscore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- my_site.leech.example
|
||||
- .my_site.hotlinker.example
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("host names with an underscore should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"my_site.leech.example", ".my_site.hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
|
||||
// referer.
|
||||
func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine)
|
||||
if err != nil {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
if len(c.RefererBlocklist) != 0 {
|
||||
t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
|
||||
// host, or a value that is not a list of them, aborts startup with an error
|
||||
// naming the key and the entry.
|
||||
func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runAbortCases(t, []abortCase{
|
||||
{
|
||||
name: "entry with a scheme",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - https://leech.example\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "https://leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "entry with a path",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example/page\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "leech.example/page",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wildcard entry",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "*.leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "entry with a port",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "leech.example:8080",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "two leading dots",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "..leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dot only",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
|
||||
},
|
||||
{
|
||||
name: "empty entry",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \"\"\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist},
|
||||
},
|
||||
{
|
||||
name: "entry not a string",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - 42\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, "42"},
|
||||
},
|
||||
{
|
||||
name: "null value",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
|
||||
// takes comma-separated entries, and that an entry in it that is not a host
|
||||
// aborts startup naming the variable and the entry.
|
||||
func TestRefererBlocklistFromEnvironment(t *testing.T) {
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
|
||||
|
||||
c, err := newFromSmartConfig(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"leech.example", ".hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
|
||||
|
||||
_, err = newFromSmartConfig(nil)
|
||||
wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
)
|
||||
|
||||
// TestConcurrentWritesAllSucceed opens a database the way pixad does and
|
||||
// writes to it from several goroutines at once, so the writes run on
|
||||
// separate connections, as one request's writes and the background eviction
|
||||
// pass do. Every write must succeed, none failing with "database is locked",
|
||||
// whether or not db_url already has parameters, and the parameters it has
|
||||
// must still apply.
|
||||
func TestConcurrentWritesAllSucceed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantJournalMode string
|
||||
}{
|
||||
{
|
||||
name: "db_url without parameters",
|
||||
query: "",
|
||||
wantJournalMode: "delete",
|
||||
},
|
||||
{
|
||||
name: "db_url with the WAL parameter",
|
||||
query: "?_pragma=journal_mode(WAL)",
|
||||
wantJournalMode: "wal",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dbURL := "file:" + filepath.Join(t.TempDir(), "state.sqlite3") + tt.query
|
||||
|
||||
d := &Database{
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
config: &config.Config{DBURL: dbURL},
|
||||
}
|
||||
|
||||
err := d.connect(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to connect to %q: %v", dbURL, err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = d.db.Close() })
|
||||
|
||||
writeConcurrently(t, d.db)
|
||||
|
||||
var journalMode string
|
||||
|
||||
err = d.db.QueryRowContext(t.Context(), "PRAGMA journal_mode").
|
||||
Scan(&journalMode)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read the journal mode: %v", err)
|
||||
}
|
||||
|
||||
if journalMode != tt.wantJournalMode {
|
||||
t.Errorf("journal mode = %q, want %q", journalMode, tt.wantJournalMode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeConcurrently runs writeLikeOneRequest from several goroutines at once
|
||||
// and checks that every write was made.
|
||||
func writeConcurrently(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
const (
|
||||
writers = 4
|
||||
requestsEach = 20
|
||||
totalRequests = writers * requestsEach
|
||||
)
|
||||
|
||||
ctx := t.Context()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for writer := range writers {
|
||||
wg.Go(func() {
|
||||
for request := range requestsEach {
|
||||
key := fmt.Sprintf("%d-%d", writer, request)
|
||||
|
||||
err := writeLikeOneRequest(ctx, db, key)
|
||||
if err != nil {
|
||||
t.Errorf("writer %d: %v", writer, err)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
var hits, sources int
|
||||
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT hit_count, (SELECT COUNT(*) FROM source_content)
|
||||
FROM cache_stats WHERE id = 1
|
||||
`).Scan(&hits, &sources)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count the writes: %v", err)
|
||||
}
|
||||
|
||||
if hits != totalRequests || sources != totalRequests {
|
||||
t.Errorf("hit_count = %d and %d source_content rows, want %d of each",
|
||||
hits, sources, totalRequests)
|
||||
}
|
||||
}
|
||||
|
||||
// writeLikeOneRequest makes the writes one request and the eviction pass
|
||||
// make: it counts a cache hit, stores a source, records a transformed image
|
||||
// and deletes that record again.
|
||||
func writeLikeOneRequest(ctx context.Context, db *sql.DB, key string) error {
|
||||
_, err := db.ExecContext(ctx,
|
||||
`UPDATE cache_stats SET hit_count = hit_count + 1 WHERE id = 1`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("counting a cache hit: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO source_content
|
||||
(content_hash, content_type, size_bytes) VALUES (?, 'image/png', 1)`, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storing a source: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, `INSERT INTO variant_content
|
||||
(cache_key, size_bytes, content_type) VALUES (?, 1, 'image/png')`, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recording a transformed image: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx,
|
||||
`DELETE FROM variant_content WHERE cache_key = ?`, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("evicting a transformed image: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -243,7 +243,17 @@ func (s *Database) DB() *sql.DB {
|
||||
}
|
||||
|
||||
func (s *Database) connect(ctx context.Context) error {
|
||||
dbURL := s.config.DBURL
|
||||
// Requests and the eviction pass write on separate connections. With
|
||||
// a busy timeout, a write that finds another one in progress waits up
|
||||
// to five seconds for it instead of failing at once with "database is
|
||||
// locked". The driver runs each _pragma parameter on every connection
|
||||
// it opens.
|
||||
separator := "?"
|
||||
if strings.Contains(s.config.DBURL, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
|
||||
dbURL := s.config.DBURL + separator + "_pragma=busy_timeout(5000)"
|
||||
|
||||
s.log.Info("connecting to database", "url", dbURL)
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
|
||||
const appname = "pixad"
|
||||
|
||||
// Version is populated from main() via ldflags.
|
||||
var Version string //nolint:gochecknoglobals // set from main
|
||||
// Version is set by app.Run to the version main was built with.
|
||||
var Version string //nolint:gochecknoglobals // set by app.Run
|
||||
|
||||
// Globals holds application-wide constants.
|
||||
type Globals struct {
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
"sneak.berlin/go/pixa/internal/session"
|
||||
)
|
||||
|
||||
// formatField is the generator form's format field name.
|
||||
const formatField = "format"
|
||||
|
||||
// Markers telling the login page from the generator page.
|
||||
const (
|
||||
loginForm = `action="/"`
|
||||
loginKeyInput = `name="key"`
|
||||
generatorForm = `action="/generate"`
|
||||
)
|
||||
|
||||
// generatedURLPattern extracts the path of the URL the generator page shows.
|
||||
// The test router runs with debug on, so the URL starts with http, and its
|
||||
// host is httptest's default request host.
|
||||
var generatedURLPattern = regexp.MustCompile(
|
||||
`value="http://example\.com(/v1/e/[^"]+)"`)
|
||||
|
||||
// findSessionCookie returns the session cookie rec sets, or nil if it sets
|
||||
// none.
|
||||
func findSessionCookie(rec *httptest.ResponseRecorder) *http.Cookie {
|
||||
for _, c := range rec.Result().Cookies() {
|
||||
if c.Name == session.CookieName {
|
||||
return c
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestHandleRoot_NoSession_ShowsLoginForm verifies that GET / without a
|
||||
// login session shows the login form.
|
||||
func TestHandleRoot_NoSession_ShowsLoginForm(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, "/", nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
|
||||
t.Errorf("page is not the login form: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginPost_WrongKey_ShowsErrorWithoutSession verifies that a wrong key
|
||||
// shows the login form again with an error, and sets no session cookie.
|
||||
func TestLoginPost_WrongKey_ShowsErrorWithoutSession(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
cookies, token := csrfCredentials(t, srv, nil)
|
||||
|
||||
rec := postForm(srv, "/", cookies, url.Values{
|
||||
loginKeyField: {"wrong-signing-key-fedcba9876543210"},
|
||||
csrfTokenField: {token},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, loginForm) || !strings.Contains(body, loginKeyInput) {
|
||||
t.Errorf("page is not the login form: %s", body)
|
||||
}
|
||||
|
||||
if !strings.Contains(body, "Invalid signing key") {
|
||||
t.Error("login form does not show the error")
|
||||
}
|
||||
|
||||
if c := findSessionCookie(rec); c != nil {
|
||||
t.Errorf("wrong key set a session cookie: %s", c)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginPost_RightKey_SetsSessionCookie verifies that the right key answers
|
||||
// 303 to / with a session cookie marked Secure, HttpOnly and SameSite=Strict,
|
||||
// and that GET / with that cookie shows the generator page.
|
||||
func TestLoginPost_RightKey_SetsSessionCookie(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
cookies, token := csrfCredentials(t, srv, nil)
|
||||
|
||||
rec := postForm(srv, "/", cookies, url.Values{
|
||||
loginKeyField: {testSigningKey},
|
||||
csrfTokenField: {token},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
sessionCookie := findSessionCookie(rec)
|
||||
if sessionCookie == nil {
|
||||
t.Fatal("right key set no session cookie")
|
||||
}
|
||||
|
||||
t.Logf("Set-Cookie: %s", sessionCookie)
|
||||
|
||||
if !sessionCookie.Secure {
|
||||
t.Error("session cookie is not Secure")
|
||||
}
|
||||
|
||||
if !sessionCookie.HttpOnly {
|
||||
t.Error("session cookie is not HttpOnly")
|
||||
}
|
||||
|
||||
if sessionCookie.SameSite != http.SameSiteStrictMode {
|
||||
t.Errorf("session cookie SameSite = %v, want Strict", sessionCookie.SameSite)
|
||||
}
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
||||
req.AddCookie(sessionCookie)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK ||
|
||||
!strings.Contains(rec.Body.String(), generatorForm) {
|
||||
t.Errorf("GET / with the session cookie: status = %d, "+
|
||||
"want %d and the generator page", rec.Code, http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleLogout_ClearsSessionCookie verifies that GET /logout answers 303
|
||||
// to / and replaces the session cookie with an empty one sent with
|
||||
// Max-Age=0, which makes the browser delete it.
|
||||
func TestHandleLogout_ClearsSessionCookie(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, _ := newCSRFTestRouter(t)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, "/logout", nil)
|
||||
req.AddCookie(newSessionCookie(t, h))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleLogout().ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
t.Logf("Set-Cookie: %s", rec.Header().Get("Set-Cookie"))
|
||||
|
||||
sessionCookie := findSessionCookie(rec)
|
||||
if sessionCookie == nil {
|
||||
t.Fatal("logout did not set the session cookie")
|
||||
}
|
||||
|
||||
if sessionCookie.Value != "" {
|
||||
t.Errorf("session cookie value = %q, want empty", sessionCookie.Value)
|
||||
}
|
||||
|
||||
// net/http reads a Max-Age=0 attribute back as MaxAge -1.
|
||||
if sessionCookie.MaxAge != -1 {
|
||||
t.Errorf("session cookie MaxAge = %d, want -1 (Max-Age=0)",
|
||||
sessionCookie.MaxAge)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_NoSession_RedirectsToLogin verifies that POST /generate
|
||||
// with a valid CSRF token but no login session answers 303 to / and makes no
|
||||
// URL.
|
||||
func TestGeneratePost_NoSession_RedirectsToLogin(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, srv := newCSRFTestRouter(t)
|
||||
|
||||
cookies, token := csrfCredentials(t, srv, nil)
|
||||
|
||||
rec := postForm(srv, "/generate", cookies, url.Values{
|
||||
sourceURLField: {testSourceURL},
|
||||
csrfTokenField: {token},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusSeeOther || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status = %d, Location = %q, want %d to /",
|
||||
rec.Code, rec.Header().Get("Location"), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
if strings.Contains(rec.Body.String(), "/v1/e/") {
|
||||
t.Error("a URL was made without a login session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratePost_URLServesImage verifies that the URL the generator page
|
||||
// makes is served by /v1/e/. The image route runs on handlers of its own,
|
||||
// made with the same signing key.
|
||||
func TestGeneratePost_URLServesImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
formatField: {string(imgcache.FormatJPEG)},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
t.Logf("generated URL path: %s", match[1])
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireServedPhoto(t, imageRec)
|
||||
}
|
||||
|
||||
// TestGeneratePost_URLWithTTLExpires verifies that a URL the generator page
|
||||
// makes with a ttl of one second is served by /v1/e/ at once and answers 410
|
||||
// once the ttl has passed.
|
||||
func TestGeneratePost_URLWithTTLExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := generatePost(t, url.Values{
|
||||
sourceURLField: {"https://" + signedHost + photoPath},
|
||||
widthField: {"50"},
|
||||
heightField: {"50"},
|
||||
formatField: {string(imgcache.FormatJPEG)},
|
||||
ttlField: {"1"},
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
|
||||
if match == nil {
|
||||
t.Fatalf("generator page shows no URL: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
imageRec := httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
requireServedPhoto(t, imageRec)
|
||||
|
||||
// The URL keeps the time it expires in whole seconds and is served
|
||||
// through the whole of that second, so a ttl of one second has passed
|
||||
// for certain two seconds after the URL was made.
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
imageRec = httptest.NewRecorder()
|
||||
imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, match[1], nil))
|
||||
|
||||
t.Logf("GET %s after the ttl: %d %q", match[1], imageRec.Code, imageRec.Body)
|
||||
|
||||
if imageRec.Code != http.StatusGone {
|
||||
t.Errorf("status after the ttl = %d, want %d",
|
||||
imageRec.Code, http.StatusGone)
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,79 @@ import (
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
)
|
||||
|
||||
// TestNewCacheConfigFromCacheMaxBytes checks the cache configuration
|
||||
// built from cache_max_bytes: omitted, the cache works out the default
|
||||
// limit; 0 turns the disk cache off; a positive value is the limit,
|
||||
// unchanged.
|
||||
func TestNewCacheConfigFromCacheMaxBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const oneGiB = 1 << 30
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
cacheMaxBytes int64
|
||||
cacheMaxBytesExplicit bool
|
||||
wantMaxBytes int64
|
||||
wantUseDefaultMaxBytes bool
|
||||
wantDisableDiskCache bool
|
||||
}{
|
||||
{
|
||||
name: "cache_max_bytes omitted",
|
||||
cacheMaxBytes: 0,
|
||||
cacheMaxBytesExplicit: false,
|
||||
wantMaxBytes: 0,
|
||||
wantUseDefaultMaxBytes: true,
|
||||
wantDisableDiskCache: false,
|
||||
},
|
||||
{
|
||||
name: "cache_max_bytes: 0",
|
||||
cacheMaxBytes: 0,
|
||||
cacheMaxBytesExplicit: true,
|
||||
wantMaxBytes: 0,
|
||||
wantUseDefaultMaxBytes: false,
|
||||
wantDisableDiskCache: true,
|
||||
},
|
||||
{
|
||||
name: "cache_max_bytes: 1 GiB",
|
||||
cacheMaxBytes: oneGiB,
|
||||
cacheMaxBytesExplicit: true,
|
||||
wantMaxBytes: oneGiB,
|
||||
wantUseDefaultMaxBytes: false,
|
||||
wantDisableDiskCache: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := &config.Config{
|
||||
CacheMaxBytes: tc.cacheMaxBytes,
|
||||
CacheMaxBytesExplicit: tc.cacheMaxBytesExplicit,
|
||||
}
|
||||
|
||||
got := newCacheConfig(cfg, nil)
|
||||
t.Logf("MaxBytes = %d, UseDefaultMaxBytes = %v, DisableDiskCache = %v",
|
||||
got.MaxBytes, got.UseDefaultMaxBytes, got.DisableDiskCache)
|
||||
|
||||
if got.MaxBytes != tc.wantMaxBytes {
|
||||
t.Errorf("MaxBytes = %d, want %d", got.MaxBytes, tc.wantMaxBytes)
|
||||
}
|
||||
|
||||
if got.UseDefaultMaxBytes != tc.wantUseDefaultMaxBytes {
|
||||
t.Errorf("UseDefaultMaxBytes = %v, want %v",
|
||||
got.UseDefaultMaxBytes, tc.wantUseDefaultMaxBytes)
|
||||
}
|
||||
|
||||
if got.DisableDiskCache != tc.wantDisableDiskCache {
|
||||
t.Errorf("DisableDiskCache = %v, want %v",
|
||||
got.DisableDiskCache, tc.wantDisableDiskCache)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDiskCacheOffOnlyForExplicitZeroCacheMaxBytes starts the handlers
|
||||
// once with cache_max_bytes omitted and once with cache_max_bytes: 0,
|
||||
// and checks by whether the cache directories were created that the
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxtest"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
)
|
||||
|
||||
// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as
|
||||
// pixad does, in an fx app that provides no fetcher, and requests an image
|
||||
// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL
|
||||
// check accepts that address; only the dialer that refuses internal
|
||||
// addresses checks blocked_networks, so the answer is 403 only if the
|
||||
// fetcher the handlers build from the config connects with that dialer. Any
|
||||
// other dialer would try to connect until the upstream fetch timeout, which
|
||||
// is short so that the test then fails quickly.
|
||||
func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const host = "192.0.2.10"
|
||||
|
||||
stateDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
SigningKey: testSigningKey,
|
||||
StateDir: stateDir,
|
||||
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||
AllowlistHosts: []string{host},
|
||||
BlockedNetworks: []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
|
||||
UpstreamFetchTimeout: 2 * time.Second,
|
||||
// With no connection slots, the fetch would fail before dialing.
|
||||
UpstreamConnections: config.DefaultUpstreamConnections,
|
||||
}
|
||||
|
||||
var h *Handlers
|
||||
|
||||
app := fxtest.New(t,
|
||||
fx.Supply(cfg),
|
||||
fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New),
|
||||
fx.Populate(&h),
|
||||
)
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
|
||||
rec := sendGet(t, r, photoURL(host))
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "forbidden")
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/allowlist"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
@@ -27,6 +28,11 @@ type Params struct {
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Database *database.Database
|
||||
Config *config.Config
|
||||
|
||||
// Fetcher, when provided, fetches upstream images in place of the
|
||||
// fetcher the handlers build from the config. Only tests provide one;
|
||||
// pixad does not.
|
||||
Fetcher httpfetcher.Fetcher `optional:"true"`
|
||||
}
|
||||
|
||||
// Handlers provides HTTP request handlers.
|
||||
@@ -35,11 +41,16 @@ type Handlers struct {
|
||||
hc *healthcheck.Healthcheck
|
||||
db *database.Database
|
||||
config *config.Config
|
||||
fetcher httpfetcher.Fetcher
|
||||
imgSvc *imgcache.Service
|
||||
imgCache *imgcache.Cache
|
||||
sessMgr *session.Manager
|
||||
encGen *encurl.Generator
|
||||
csrfProtect func(http.Handler) http.Handler
|
||||
|
||||
// refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
|
||||
// reports whether a URL's host is on that list.
|
||||
refererBlocklist *allowlist.HostAllowList
|
||||
}
|
||||
|
||||
// New creates a new Handlers instance.
|
||||
@@ -50,11 +61,13 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
|
||||
}
|
||||
|
||||
s := &Handlers{
|
||||
log: params.Logger.Get(),
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
csrfProtect: csrfProtect,
|
||||
log: params.Logger.Get(),
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
fetcher: params.Fetcher,
|
||||
csrfProtect: csrfProtect,
|
||||
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
@@ -80,20 +93,25 @@ func (s *Handlers) WaitForProcessing(ctx context.Context) int {
|
||||
return s.imgSvc.WaitForProcessing(ctx)
|
||||
}
|
||||
|
||||
// initImageService initializes the image cache and service.
|
||||
func (s *Handlers) initImageService() error {
|
||||
// Create the cache. cache_max_bytes: 0 disables the disk cache
|
||||
// entirely; any other value is the eviction limit in bytes; when
|
||||
// it is omitted, the cache works out the default limit itself.
|
||||
cache, err := imgcache.NewCache(s.db.DB(), imgcache.CacheConfig{
|
||||
StateDir: s.config.StateDir,
|
||||
// newCacheConfig builds the image cache's configuration from cfg.
|
||||
// cache_max_bytes: 0 disables the disk cache entirely; any other value
|
||||
// is the eviction limit in bytes; when it is omitted, the cache works
|
||||
// out the default limit itself.
|
||||
func newCacheConfig(cfg *config.Config, log *slog.Logger) imgcache.CacheConfig {
|
||||
return imgcache.CacheConfig{
|
||||
StateDir: cfg.StateDir,
|
||||
CacheTTL: imgcache.DefaultCacheTTL,
|
||||
NegativeTTL: imgcache.DefaultNegativeTTL,
|
||||
MaxBytes: s.config.CacheMaxBytes,
|
||||
UseDefaultMaxBytes: !s.config.CacheMaxBytesExplicit,
|
||||
DisableDiskCache: s.config.CacheMaxBytesExplicit && s.config.CacheMaxBytes == 0,
|
||||
Logger: s.log,
|
||||
})
|
||||
MaxBytes: cfg.CacheMaxBytes,
|
||||
UseDefaultMaxBytes: !cfg.CacheMaxBytesExplicit,
|
||||
DisableDiskCache: cfg.CacheMaxBytesExplicit && cfg.CacheMaxBytes == 0,
|
||||
Logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
// initImageService initializes the image cache and service.
|
||||
func (s *Handlers) initImageService() error {
|
||||
cache, err := imgcache.NewCache(s.db.DB(), newCacheConfig(s.config, s.log))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -117,10 +135,12 @@ func (s *Handlers) initImageService() error {
|
||||
fetcherCfg.MaxConnections = s.config.UpstreamConnections
|
||||
fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
|
||||
|
||||
// Create the service
|
||||
// Create the service. With no fetcher provided, it builds its own from
|
||||
// fetcherCfg.
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
FetcherConfig: fetcherCfg,
|
||||
Fetcher: s.fetcher,
|
||||
SigningKey: s.config.SigningKey,
|
||||
Allowlist: s.config.AllowlistHosts,
|
||||
MaxConcurrentProcessing: s.config.MaxConcurrentProcessing,
|
||||
|
||||
@@ -21,6 +21,10 @@ import (
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
req, ok := s.parseImageRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
@@ -248,6 +252,23 @@ func cacheControl(expires time.Time) string {
|
||||
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
||||
}
|
||||
|
||||
// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
|
||||
// names a host on referer_blocklist, and reports whether it answered. A request
|
||||
// with no Referer, or one that does not parse as a URL with a host, is not
|
||||
// refused.
|
||||
func (s *Handlers) refuseBlockedReferer(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) bool {
|
||||
referer, err := url.Parse(r.Referer())
|
||||
if err != nil || !s.refererBlocklist.IsAllowed(referer) {
|
||||
return false
|
||||
}
|
||||
|
||||
s.respondError(w, "referer blocked", http.StatusForbidden)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// notModified sets the ETag header to etag and, when the request's
|
||||
// If-None-Match is that ETag, answers 304 Not Modified. It reports whether it
|
||||
// answered. An empty etag sets no header and never answers.
|
||||
|
||||
@@ -22,6 +22,10 @@ import (
|
||||
// browsers identify the content type.
|
||||
func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
start := time.Now()
|
||||
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
)
|
||||
|
||||
// requireServedPhoto requires that rec answers 200 with the JPEG at photoPath
|
||||
// on signedHost at the 50x50 that encPhotoURL and the generator tests ask for.
|
||||
func requireServedPhoto(t *testing.T, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body %q",
|
||||
rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
contentType := rec.Header().Get("Content-Type")
|
||||
if contentType != "image/jpeg" {
|
||||
t.Errorf("Content-Type = %q, want image/jpeg", contentType)
|
||||
}
|
||||
|
||||
img, err := jpeg.DecodeConfig(rec.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("body is not a JPEG: %v", err)
|
||||
}
|
||||
|
||||
if img.Width != 50 || img.Height != 50 {
|
||||
t.Errorf("image is %dx%d, want 50x50", img.Width, img.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleImageEnc_ValidToken_ServesImage verifies that a token made with
|
||||
// the signing key serves the image it asks for.
|
||||
func TestHandleImageEnc_ValidToken_ServesImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, encPhotoURL(t, h), nil))
|
||||
|
||||
requireServedPhoto(t, rec)
|
||||
}
|
||||
|
||||
// TestHandleImageEnc_RejectedToken verifies that a token that has expired
|
||||
// answers 410, and that a token with one character changed, a token cut
|
||||
// short, and a token made with another signing key answer 400. The server
|
||||
// would serve the photo for a token it accepted.
|
||||
func TestHandleImageEnc_RejectedToken(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, srv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
|
||||
|
||||
photo := encurl.Payload{
|
||||
SourceHost: signedHost,
|
||||
SourcePath: photoPath,
|
||||
Width: 50,
|
||||
Height: 50,
|
||||
}
|
||||
|
||||
valid, err := h.encGen.Generate(&photo)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
expiredPhoto := photo
|
||||
expiredPhoto.ExpiresAt = time.Now().Add(-time.Minute).Unix()
|
||||
|
||||
expired, err := h.encGen.Generate(&expiredPhoto)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
otherGen, err := encurl.NewGenerator("another-signing-key-fedcba9876543210")
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
otherKey, err := otherGen.Generate(&photo)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
// Changing a character in the middle always changes the decoded bytes;
|
||||
// the last character of unpadded base64 can carry unused bits.
|
||||
middle := len(valid) / 2
|
||||
|
||||
replacement := "A"
|
||||
if valid[middle] == 'A' {
|
||||
replacement = "B"
|
||||
}
|
||||
|
||||
changed := valid[:middle] + replacement + valid[middle+1:]
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
token string
|
||||
wantStatus int
|
||||
}{
|
||||
{"expired", expired, http.StatusGone},
|
||||
{"one character changed", changed, http.StatusBadRequest},
|
||||
{"cut short", valid[:middle], http.StatusBadRequest},
|
||||
{"another signing key", otherKey, http.StatusBadRequest},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rec := getEncToken(srv, tt.token)
|
||||
t.Logf("GET /v1/e/%s/img.jpg: %d %s", tt.token, rec.Code, rec.Body)
|
||||
|
||||
if rec.Code != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/allowlist"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// blockedReferer is a page on leech.example, which newRefererRoutes puts on
|
||||
// referer_blocklist.
|
||||
const blockedReferer = "https://leech.example/page.html"
|
||||
|
||||
// countingFetcher passes each fetch on to the fetcher it holds and counts it.
|
||||
type countingFetcher struct {
|
||||
httpfetcher.Fetcher
|
||||
|
||||
fetches atomic.Int32
|
||||
}
|
||||
|
||||
// Fetch counts the fetch and passes it on.
|
||||
func (f *countingFetcher) Fetch(
|
||||
ctx context.Context, url string,
|
||||
) (*httpfetcher.FetchResult, error) {
|
||||
f.fetches.Add(1)
|
||||
|
||||
return f.Fetcher.Fetch(ctx, url)
|
||||
}
|
||||
|
||||
// newRefererRoutes returns both image routes of a Handlers whose
|
||||
// referer_blocklist is "leech.example" and ".hotlinker.example", the
|
||||
// Handlers, and the fetcher the routes fetch through. The JPEG at photoPath
|
||||
// exists on allowlistedHost and on signedHost.
|
||||
func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) {
|
||||
t.Helper()
|
||||
|
||||
fetcher := &countingFetcher{
|
||||
Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost),
|
||||
}
|
||||
|
||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||
StateDir: t.TempDir(),
|
||||
CacheTTL: time.Hour,
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewCache() error = %v", err)
|
||||
}
|
||||
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
Fetcher: fetcher,
|
||||
SigningKey: testSigningKey,
|
||||
Allowlist: []string{allowlistedHost},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||
}
|
||||
|
||||
encGen, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
imgSvc: svc,
|
||||
encGen: encGen,
|
||||
refererBlocklist: allowlist.New(
|
||||
[]string{"leech.example", ".hotlinker.example"}),
|
||||
}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
return r, h, fetcher
|
||||
}
|
||||
|
||||
// getWithReferer sends a GET for target to routes with referer as its
|
||||
// Referer header, or with none when referer is empty, and returns the
|
||||
// response.
|
||||
func getWithReferer(
|
||||
t *testing.T, routes http.Handler, target, referer string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||
if referer != "" {
|
||||
req.Header.Set("Referer", referer)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
routes.ServeHTTP(rec, req)
|
||||
t.Logf("GET %s with Referer %q: %d", target, referer, rec.Code)
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// TestRefererBlocklist verifies that both image routes refuse a request whose
|
||||
// Referer names a host on referer_blocklist with 403 and the JSON error,
|
||||
// without fetching from the upstream host, and serve a request with no
|
||||
// Referer, one that does not parse, or one naming any other host. Hosts are
|
||||
// matched as allowlist_hosts matches them.
|
||||
func TestRefererBlocklist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
referer string
|
||||
want int
|
||||
}{
|
||||
{"no referer", "", http.StatusOK},
|
||||
{"unlisted host", "https://unlisted.example/page.html", http.StatusOK},
|
||||
{"unparseable", "%zz", http.StatusOK},
|
||||
{"listed host", blockedReferer, http.StatusForbidden},
|
||||
{"subdomain of listed host", "https://www.leech.example/", http.StatusOK},
|
||||
{"subdomain of dot pattern", "https://www.hotlinker.example/a.html",
|
||||
http.StatusForbidden},
|
||||
{"dot pattern without its dot", "https://hotlinker.example/",
|
||||
http.StatusForbidden},
|
||||
{"host continuing past dot pattern",
|
||||
"https://hotlinker.example.evil.example/", http.StatusOK},
|
||||
}
|
||||
|
||||
// The photo's URL on each image route.
|
||||
photoURLs := map[string]func(t *testing.T, h *Handlers) string{
|
||||
"plain URL": func(t *testing.T, _ *Handlers) string {
|
||||
t.Helper()
|
||||
|
||||
return photoURL(allowlistedHost)
|
||||
},
|
||||
"encrypted URL": encPhotoURL,
|
||||
}
|
||||
|
||||
for urlName, photoURLFor := range photoURLs {
|
||||
for _, tc := range cases {
|
||||
t.Run(urlName+", "+tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
routes, h, fetcher := newRefererRoutes(t)
|
||||
|
||||
rec := getWithReferer(t, routes, photoURLFor(t, h), tc.referer)
|
||||
|
||||
if tc.want == http.StatusOK {
|
||||
requireServedPhoto(t, rec)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
|
||||
|
||||
if n := fetcher.fetches.Load(); n != 0 {
|
||||
t.Errorf("upstream fetched %d times, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockedRefererRefusedWhenImageIsCached verifies that a request whose
|
||||
// Referer is on referer_blocklist is refused even when the image it asks for
|
||||
// is already cached, so the answer does not depend on the cache.
|
||||
func TestBlockedRefererRefusedWhenImageIsCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
routes, h, _ := newRefererRoutes(t)
|
||||
|
||||
for _, target := range []string{photoURL(allowlistedHost), encPhotoURL(t, h)} {
|
||||
requireServedPhoto(t, getWithReferer(t, routes, target, ""))
|
||||
|
||||
rec := getWithReferer(t, routes, target, blockedReferer)
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package httpfetcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built
|
||||
// without DialContext, as pixa builds it, refuses to connect to a local
|
||||
// server.
|
||||
func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
transport := transportOf(t, New(DefaultConfig()))
|
||||
|
||||
addr := srv.Listener.Addr().String()
|
||||
|
||||
_, err := transport.DialContext(testContext(t), "tcp", addr)
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Fatalf("DialContext(%s) error = %v, want ErrSSRFBlocked", addr, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDialContextReplacesOnlyTheDialer checks that a fetcher built with
|
||||
// DialContext connects through it, while the URL check still refuses a
|
||||
// loopback URL and the redirect check a redirect to a link-local address.
|
||||
func TestDialContextReplacesOnlyTheDialer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := startUpstream(t)
|
||||
dialer := &recordingDialer{target: srv.Listener.Addr().String()}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.AllowHTTP = true
|
||||
cfg.DialContext = dialer.dialContext
|
||||
f := New(cfg)
|
||||
|
||||
if body := fetchBody(t, f, "/image"); body != imagePayload {
|
||||
t.Errorf("body = %q, want %q", body, imagePayload)
|
||||
}
|
||||
|
||||
_, err := f.Fetch(testContext(t), "http://127.0.0.1/image")
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err)
|
||||
}
|
||||
|
||||
_, err = f.Fetch(testContext(t), upstreamURL("/redirect/private"))
|
||||
if !errors.Is(err, ErrSSRFBlocked) {
|
||||
t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err)
|
||||
}
|
||||
|
||||
// The upstream server is reached through DialContext, and nothing else
|
||||
// is asked of it.
|
||||
dialed := dialer.dialedAddrs()
|
||||
if len(dialed) == 0 {
|
||||
t.Error("DialContext was never called")
|
||||
}
|
||||
|
||||
for _, addr := range dialed {
|
||||
if addr != net.JoinHostPort(testPublicHost, "80") {
|
||||
t.Errorf("DialContext was asked to connect to %s", addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -137,6 +137,11 @@ type Config struct {
|
||||
// BlockedNetworks are operator-supplied CIDR ranges refused by the
|
||||
// dialer, in addition to the always-enforced built-in ranges.
|
||||
BlockedNetworks []netip.Prefix
|
||||
// DialContext, when set, makes the fetcher's connections in place of
|
||||
// the dialer that refuses internal addresses; the URL and redirect
|
||||
// checks still run. Only tests set it, to reach a local server; the
|
||||
// config file and the environment cannot.
|
||||
DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// DefaultConfig returns a Config with sensible defaults.
|
||||
@@ -190,13 +195,19 @@ func New(config *Config) *HTTPFetcher {
|
||||
config = DefaultConfig()
|
||||
}
|
||||
|
||||
// Create transport with SSRF-safe dialer. The dialer re-resolves and
|
||||
// re-checks at connect time (closing the DNS-rebinding window) against
|
||||
// both the built-in ranges and the operator-supplied blocklist.
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
// Unless config.DialContext replaces it, the transport connects with
|
||||
// the SSRF-safe dialer, which re-resolves and re-checks at connect time
|
||||
// (closing the DNS-rebinding window) against both the built-in ranges
|
||||
// and the operator-supplied blocklist.
|
||||
dialContext := config.DialContext
|
||||
if dialContext == nil {
|
||||
dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return dialSSRFSafe(ctx, network, addr, config.BlockedNetworks)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
DialContext: dialContext,
|
||||
TLSHandshakeTimeout: DefaultTLSTimeout,
|
||||
MaxIdleConns: DefaultMaxIdleConns,
|
||||
IdleConnTimeout: DefaultIdleConnTimeout,
|
||||
|
||||
@@ -847,15 +847,28 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
|
||||
const interval = 100 * time.Millisecond
|
||||
// Hold the test database's only connection, so the startup pass
|
||||
// waits for it after walking the still empty variant directory: the
|
||||
// file written while it waits is first seen by a periodic pass.
|
||||
conn, err := cache.db.Conn(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to take the database connection: %v", err)
|
||||
}
|
||||
|
||||
cache.StartEviction(interval)
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
cache.StartEviction(100 * time.Millisecond)
|
||||
defer func() { _ = cache.StopEviction(t.Context()) }()
|
||||
|
||||
// Let startup reconciliation run and settle on an empty cache
|
||||
// before introducing the untracked file, so the adoption we assert
|
||||
// below can only be the work of a later, periodic pass.
|
||||
time.Sleep(3 * interval)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
|
||||
for cache.db.Stats().WaitCount == 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the startup pass never waited for the database")
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Simulate a variant whose accounting insert failed after the
|
||||
// process was already running and serving requests: the content
|
||||
@@ -864,14 +877,16 @@ func TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup(t *testing.T) {
|
||||
// insert had failed and only the file write had succeeded.
|
||||
untracked := bytes.Repeat([]byte{0x41}, 900)
|
||||
|
||||
_, err := cache.variants.Store(
|
||||
_, err = cache.variants.Store(
|
||||
"aabbccdd0099", bytes.NewReader(untracked), "image/webp",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to store untracked variant file: %v", err)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
_ = conn.Close()
|
||||
|
||||
deadline = time.Now().Add(5 * time.Second)
|
||||
|
||||
var usage int64
|
||||
|
||||
|
||||
@@ -42,7 +42,8 @@ type Service struct {
|
||||
type ServiceConfig struct {
|
||||
// Cache is the cache instance
|
||||
Cache *Cache
|
||||
// FetcherConfig configures the upstream fetcher (ignored if Fetcher is set)
|
||||
// FetcherConfig configures the upstream fetcher built when Fetcher is
|
||||
// not set. Its AllowHTTP and MaxResponseSize are used either way.
|
||||
FetcherConfig *httpfetcher.Config
|
||||
// Fetcher is an optional custom fetcher (for testing)
|
||||
Fetcher httpfetcher.Fetcher
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// Package loadtestorigin is the upstream host script/loadtest points pixad
|
||||
// at, run by cmd/loadtest-origin. It answers every request, whatever its path,
|
||||
// with the same generated JPEG, so each new path is a new source image for
|
||||
// pixad to fetch, and it logs one line per request, so its log counts pixad's
|
||||
// fetches.
|
||||
package loadtestorigin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
listenAddress = ":80"
|
||||
readHeaderTimeout = 10 * time.Second
|
||||
imageWidth = 1600
|
||||
imageHeight = 1200
|
||||
jpegQuality = 85
|
||||
)
|
||||
|
||||
// Run makes the image and serves it on port 80 until the server fails, then
|
||||
// exits the process with status 1.
|
||||
func Run() {
|
||||
photo, err := makeJPEG()
|
||||
if err != nil {
|
||||
slog.Error("cannot make the image", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: listenAddress,
|
||||
Handler: newHandler(photo),
|
||||
ReadHeaderTimeout: readHeaderTimeout,
|
||||
}
|
||||
|
||||
err = server.ListenAndServe()
|
||||
slog.Error("server stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// newHandler answers every request with photo and logs the request's path.
|
||||
func newHandler(photo []byte) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
slog.Info("request", "path", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(photo)
|
||||
})
|
||||
}
|
||||
|
||||
// makeJPEG draws colour gradients crossed with a fine pattern, so the image
|
||||
// has detail to decode and does not compress to almost nothing.
|
||||
func makeJPEG() ([]byte, error) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight))
|
||||
|
||||
// red and green count up from 0 to 255 and wrap around, along each row
|
||||
// and down the image.
|
||||
var green uint8
|
||||
|
||||
for y := range imageHeight {
|
||||
var red uint8
|
||||
|
||||
for x := range imageWidth {
|
||||
img.SetRGBA(x, y, color.RGBA{
|
||||
R: red, G: green, B: red ^ green, A: math.MaxUint8,
|
||||
})
|
||||
red++
|
||||
}
|
||||
|
||||
green++
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package loadtestorigin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEveryPathServesTheSameJPEG checks that the origin answers any path with
|
||||
// 200 and the same JPEG, so every new path script/loadtest asks pixad for is
|
||||
// a valid source image.
|
||||
func TestEveryPathServesTheSameJPEG(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
photo, err := makeJPEG()
|
||||
if err != nil {
|
||||
t.Fatalf("makeJPEG: %v", err)
|
||||
}
|
||||
|
||||
size, err := jpeg.DecodeConfig(bytes.NewReader(photo))
|
||||
if err != nil {
|
||||
t.Fatalf("the image does not decode as a JPEG: %v", err)
|
||||
}
|
||||
|
||||
if size.Width != imageWidth || size.Height != imageHeight {
|
||||
t.Errorf("the image is %dx%d, want %dx%d",
|
||||
size.Width, size.Height, imageWidth, imageHeight)
|
||||
}
|
||||
|
||||
handler := newHandler(photo)
|
||||
|
||||
for _, path := range []string{"/", "/miss/1.jpg", "/herd/2.jpg"} {
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, path, nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("%s: status = %d, want %d", path, rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "image/jpeg" {
|
||||
t.Errorf("%s: Content-Type = %q, want image/jpeg", path, ct)
|
||||
}
|
||||
|
||||
if !bytes.Equal(rec.Body.Bytes(), photo) {
|
||||
t.Errorf("%s: the body is not the image", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,13 +35,10 @@ const HSTSValue = "max-age=31536000; includeSubDomains"
|
||||
|
||||
// ContentSecurityPolicyValue is the Content-Security-Policy header value.
|
||||
// default-src 'self' is the baseline and frame-ancestors 'none' is the primary
|
||||
// clickjacking control. 'unsafe-inline' is required in script-src and style-src
|
||||
// because the served templates carry inline onclick handlers (generator page)
|
||||
// and the bundled Tailwind asset injects a runtime <style> element; dropping it
|
||||
// needs template changes outside this issue's scope.
|
||||
// clickjacking control.
|
||||
const ContentSecurityPolicyValue = "default-src 'self'; " +
|
||||
"script-src 'self' 'unsafe-inline'; " +
|
||||
"style-src 'self' 'unsafe-inline'; " +
|
||||
"script-src 'self'; " +
|
||||
"style-src 'self'; " +
|
||||
"object-src 'none'; " +
|
||||
"base-uri 'self'; " +
|
||||
"form-action 'self'; " +
|
||||
|
||||
@@ -325,6 +325,13 @@ func TestSecurityHeaders_PolicyHeaders(t *testing.T) {
|
||||
|
||||
handler.ServeHTTP(rec, req)
|
||||
|
||||
// The login and generator pages load their script and stylesheet from
|
||||
// /static, so the policy allows no inline script or style.
|
||||
csp := rec.Header().Get("Content-Security-Policy")
|
||||
if strings.Contains(csp, "unsafe-inline") {
|
||||
t.Errorf("Content-Security-Policy allows unsafe-inline: %q", csp)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
header string
|
||||
want string
|
||||
@@ -333,8 +340,8 @@ func TestSecurityHeaders_PolicyHeaders(t *testing.T) {
|
||||
{
|
||||
"Content-Security-Policy",
|
||||
"default-src 'self'; " +
|
||||
"script-src 'self' 'unsafe-inline'; " +
|
||||
"style-src 'self' 'unsafe-inline'; " +
|
||||
"script-src 'self'; " +
|
||||
"style-src 'self'; " +
|
||||
"object-src 'none'; " +
|
||||
"base-uri 'self'; " +
|
||||
"form-action 'self'; " +
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"go.uber.org/fx/fxtest"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/globals"
|
||||
"sneak.berlin/go/pixa/internal/handlers"
|
||||
"sneak.berlin/go/pixa/internal/healthcheck"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/logger"
|
||||
"sneak.berlin/go/pixa/internal/middleware"
|
||||
)
|
||||
|
||||
// upstreamHost is the upstream host of the image URLs below. It is a
|
||||
// documentation address (RFC 5737), which the fetcher's URL check accepts as
|
||||
// public; the fetcher's dial function connects it to the test upstream server.
|
||||
const upstreamHost = "192.0.2.10"
|
||||
|
||||
// TestImageProxyFlow requests images through pixa's router, handlers,
|
||||
// upstream fetcher, image processor, disk cache and database, with only the
|
||||
// upstream origin replaced by a local test server. The first request for a URL
|
||||
// is fetched and converted; the second is served from the cache without
|
||||
// another upstream request. The source and the converted image are then on
|
||||
// disk, with their rows in the database.
|
||||
func TestImageProxyFlow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := encodeTestPNG(t, 64, 48)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sizeFormat string // the <size>.<format> part of the image URL
|
||||
contentType string
|
||||
decodeConfig func(io.Reader) (image.Config, error)
|
||||
width, height int
|
||||
}{
|
||||
{"resize and convert to JPEG", "32x24.jpeg", "image/jpeg",
|
||||
jpeg.DecodeConfig, 32, 24},
|
||||
{"orig", "orig.orig", "image/png", png.DecodeConfig, 64, 48},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var upstreamRequests atomic.Int32
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
upstreamRequests.Add(1)
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
_, _ = w.Write(source)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
s, db, stateDir := startImageProxy(t, upstream)
|
||||
target := "/v1/image/" + upstreamHost + "/photo.png/" + tt.sizeFormat
|
||||
|
||||
first := getImage(t, s, target)
|
||||
|
||||
if got := first.Header().Get("X-Pixa-Cache"); got != "MISS" {
|
||||
t.Errorf("first X-Pixa-Cache = %q, want MISS", got)
|
||||
}
|
||||
|
||||
if got := first.Header().Get("Content-Type"); got != tt.contentType {
|
||||
t.Errorf("Content-Type = %q, want %q", got, tt.contentType)
|
||||
}
|
||||
|
||||
decoded, err := tt.decodeConfig(bytes.NewReader(first.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("decoding the image: %v", err)
|
||||
}
|
||||
|
||||
if decoded.Width != tt.width || decoded.Height != tt.height {
|
||||
t.Errorf("image is %dx%d, want %dx%d",
|
||||
decoded.Width, decoded.Height, tt.width, tt.height)
|
||||
}
|
||||
|
||||
second := getImage(t, s, target)
|
||||
|
||||
if got := second.Header().Get("X-Pixa-Cache"); got != "HIT" {
|
||||
t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
|
||||
}
|
||||
|
||||
if !bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
|
||||
t.Error("the second response is not the image the first served")
|
||||
}
|
||||
|
||||
if got := upstreamRequests.Load(); got != 1 {
|
||||
t.Errorf("upstream received %d requests, want 1", got)
|
||||
}
|
||||
|
||||
checkSourceCached(t, db, stateDir, source)
|
||||
checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// startImageProxy starts the components pixad's fx app builds, from a config
|
||||
// with a fresh state directory and upstreamHost on the allowlist, and with an
|
||||
// upstream fetcher that connects every upstream address to upstream. It
|
||||
// returns the server with its routes, the database and the state directory.
|
||||
func startImageProxy(
|
||||
t *testing.T, upstream *httptest.Server,
|
||||
) (*Server, *sql.DB, string) {
|
||||
t.Helper()
|
||||
|
||||
stateDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
SigningKey: testSigningKey,
|
||||
StateDir: stateDir,
|
||||
DBURL: "file:" + filepath.Join(stateDir, "state.sqlite3"),
|
||||
AllowlistHosts: []string{upstreamHost},
|
||||
// The test upstream server has no TLS.
|
||||
AllowHTTP: true,
|
||||
// A limit of its own, so the cache does not size itself from the
|
||||
// host's free disk space.
|
||||
CacheMaxBytes: 64 << 20,
|
||||
CacheMaxBytesExplicit: true,
|
||||
UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize,
|
||||
DownstreamTimeout: config.DefaultDownstreamTimeout,
|
||||
}
|
||||
|
||||
fetcherCfg := httpfetcher.DefaultConfig()
|
||||
fetcherCfg.AllowHTTP = true
|
||||
fetcherCfg.DialContext = func(
|
||||
ctx context.Context, network, _ string,
|
||||
) (net.Conn, error) {
|
||||
var dialer net.Dialer
|
||||
|
||||
return dialer.DialContext(ctx, network, upstream.Listener.Addr().String())
|
||||
}
|
||||
fetcher := httpfetcher.New(fetcherCfg)
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
mw *middleware.Middleware
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
app := fxtest.New(t,
|
||||
fx.Supply(cfg),
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
database.New,
|
||||
healthcheck.New,
|
||||
handlers.New,
|
||||
middleware.New,
|
||||
func() httpfetcher.Fetcher { return fetcher },
|
||||
),
|
||||
fx.Populate(&h, &mw, &db),
|
||||
)
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// Requests go straight to the router, as in newTestServer; the server's
|
||||
// own start hook, which listens on a port, is left out.
|
||||
s := &Server{config: cfg, mw: mw, h: h}
|
||||
s.SetupRoutes()
|
||||
|
||||
return s, db.DB(), stateDir
|
||||
}
|
||||
|
||||
// getImage sends a GET for target to s and fails unless it answers 200.
|
||||
func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||
t.Context(), http.MethodGet, target, nil))
|
||||
t.Logf("GET %s: %d, X-Pixa-Cache %s",
|
||||
target, rec.Code, rec.Header().Get("X-Pixa-Cache"))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s status = %d, want %d; body %s",
|
||||
target, rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// checkSourceCached checks that source is stored under its SHA-256 in
|
||||
// cache/sources, recorded in source_content, and that the source URL's row in
|
||||
// source_metadata points at it.
|
||||
func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
|
||||
t.Helper()
|
||||
|
||||
sum := sha256.Sum256(source)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
|
||||
checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
|
||||
source)
|
||||
|
||||
var rows int
|
||||
|
||||
err := db.QueryRowContext(t.Context(),
|
||||
"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
|
||||
).Scan(&rows)
|
||||
if err != nil || rows != 1 {
|
||||
t.Errorf("source_content rows for the source = %d (error %v), want 1",
|
||||
rows, err)
|
||||
}
|
||||
|
||||
var metadataHash string
|
||||
|
||||
err = db.QueryRowContext(t.Context(),
|
||||
`SELECT content_hash FROM source_metadata
|
||||
WHERE source_host = ? AND source_path = ?`,
|
||||
upstreamHost, "/photo.png",
|
||||
).Scan(&metadataHash)
|
||||
if err != nil || metadataHash != hash {
|
||||
t.Errorf("source_metadata content_hash = %q (error %v), want %q",
|
||||
metadataHash, err, hash)
|
||||
}
|
||||
}
|
||||
|
||||
// checkVariantCached checks that the converted image served is recorded in
|
||||
// variant_content with contentType, and stored under its cache key in
|
||||
// cache/variants.
|
||||
func checkVariantCached(
|
||||
t *testing.T, db *sql.DB, stateDir string, served []byte, contentType string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
var cacheKey, storedType string
|
||||
|
||||
err := db.QueryRowContext(t.Context(),
|
||||
"SELECT cache_key, content_type FROM variant_content",
|
||||
).Scan(&cacheKey, &storedType)
|
||||
if err != nil {
|
||||
t.Fatalf("variant_content row: %v", err)
|
||||
}
|
||||
|
||||
if storedType != contentType {
|
||||
t.Errorf("variant_content content_type = %q, want %q",
|
||||
storedType, contentType)
|
||||
}
|
||||
|
||||
checkFile(t, filepath.Join(stateDir, "cache", "variants",
|
||||
cacheKey[0:2], cacheKey[2:4], cacheKey), served)
|
||||
}
|
||||
|
||||
// checkFile checks that the file at path holds want.
|
||||
func checkFile(t *testing.T, path string, want []byte) {
|
||||
t.Helper()
|
||||
|
||||
//nolint:gosec // G304: a path under the test's state directory
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Errorf("reading %s: %v", path, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("%s holds %d bytes that are not the %d expected",
|
||||
path, len(got), len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// encodeTestPNG returns an opaque width x height PNG of one color.
|
||||
func encodeTestPNG(t *testing.T, width, height int) []byte {
|
||||
t.Helper()
|
||||
|
||||
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
img.Set(x, y, color.RGBA{R: 200, G: 40, B: 40, A: 255})
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := png.Encode(&buf, img)
|
||||
if err != nil {
|
||||
t.Fatalf("encoding the test PNG: %v", err)
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func (s *Server) SetupRoutes() {
|
||||
// Robots.txt
|
||||
s.router.Get("/robots.txt", s.h.HandleRobotsTxt())
|
||||
|
||||
// Static files (Tailwind CSS, etc.)
|
||||
// The login and generator pages' stylesheet and script
|
||||
s.router.Handle("/static/*", http.StripPrefix("/static/", static.Handler()))
|
||||
|
||||
// Login/generator UI. The form routes carry CSRF protection; the
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
// Generator page: a click on the generated URL selects it, and the Copy
|
||||
// button copies it. Both are on the page only once a URL has been generated.
|
||||
const generatedURL = document.getElementById("generated-url");
|
||||
|
||||
if (generatedURL) {
|
||||
generatedURL.addEventListener("click", () => generatedURL.select());
|
||||
document.getElementById("copy-url").addEventListener("click", () => {
|
||||
navigator.clipboard.writeText(generatedURL.value);
|
||||
});
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
//go:embed *.js
|
||||
//go:embed *.css *.js
|
||||
var files embed.FS
|
||||
|
||||
// FS returns the embedded filesystem containing static files.
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
/* The login and generator pages. */
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
background: #f3f4f6;
|
||||
font-family: system-ui, sans-serif;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: 1.5rem;
|
||||
line-height: 2rem;
|
||||
font-weight: 700;
|
||||
color: #1f2937;
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
margin-bottom: 0.25rem;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
color: #374151;
|
||||
}
|
||||
|
||||
input,
|
||||
select {
|
||||
width: 100%;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border: 1px solid #d1d5db;
|
||||
border-radius: 0.375rem;
|
||||
box-shadow: 0 1px 2px rgb(0 0 0 / 5%);
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
input:focus,
|
||||
select:focus {
|
||||
outline: none;
|
||||
border-color: #3b82f6;
|
||||
box-shadow: 0 0 0 2px #3b82f6;
|
||||
}
|
||||
|
||||
button {
|
||||
width: 100%;
|
||||
padding: 0.5rem 1rem;
|
||||
border: none;
|
||||
border-radius: 0.375rem;
|
||||
background: #2563eb;
|
||||
color: #fff;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.15s;
|
||||
}
|
||||
|
||||
button:hover {
|
||||
background: #1d4ed8;
|
||||
}
|
||||
|
||||
button:focus {
|
||||
outline: 2px solid #3b82f6;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
form > * + * {
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.card {
|
||||
padding: 1.5rem;
|
||||
border-radius: 0.5rem;
|
||||
background: #fff;
|
||||
box-shadow:
|
||||
0 4px 6px -1px rgb(0 0 0 / 10%),
|
||||
0 2px 4px -2px rgb(0 0 0 / 10%);
|
||||
}
|
||||
|
||||
.error {
|
||||
margin-bottom: 1rem;
|
||||
padding: 0.75rem 1rem;
|
||||
border: 1px solid #f87171;
|
||||
border-radius: 0.25rem;
|
||||
background: #fee2e2;
|
||||
color: #b91c1c;
|
||||
}
|
||||
|
||||
/* Login page: the card centred on the screen. */
|
||||
|
||||
.login {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.login .card {
|
||||
width: 100%;
|
||||
max-width: 28rem;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.login h1 {
|
||||
margin-bottom: 1.5rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Generator page. */
|
||||
|
||||
.page {
|
||||
max-width: 42rem;
|
||||
margin: 0 auto;
|
||||
padding: 2rem 1rem;
|
||||
}
|
||||
|
||||
header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
header a {
|
||||
font-size: 0.875rem;
|
||||
color: #4b5563;
|
||||
}
|
||||
|
||||
header a:hover {
|
||||
color: #1f2937;
|
||||
}
|
||||
|
||||
.result {
|
||||
margin-bottom: 1.5rem;
|
||||
padding: 1rem;
|
||||
border: 1px solid #bbf7d0;
|
||||
border-radius: 0.5rem;
|
||||
background: #f0fdf4;
|
||||
}
|
||||
|
||||
.result h2 {
|
||||
margin: 0 0 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
color: #166534;
|
||||
}
|
||||
|
||||
.result div {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.result input {
|
||||
flex: 1;
|
||||
border-color: #86efac;
|
||||
box-shadow: none;
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.result button {
|
||||
width: auto;
|
||||
padding: 0.5rem 0.75rem;
|
||||
background: #16a34a;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.result button:hover {
|
||||
background: #15803d;
|
||||
}
|
||||
|
||||
.result p {
|
||||
margin: 0.5rem 0 0;
|
||||
font-size: 0.75rem;
|
||||
color: #16a34a;
|
||||
}
|
||||
|
||||
.columns {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.note {
|
||||
margin-top: 1rem;
|
||||
font-size: 0.75rem;
|
||||
color: #6b7280;
|
||||
text-align: center;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -4,52 +4,47 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Pixa - URL Generator</title>
|
||||
<script src="/static/tailwind.js"></script>
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
<body class="bg-gray-100 min-h-screen">
|
||||
<div class="max-w-2xl mx-auto py-8 px-4">
|
||||
<div class="flex justify-between items-center mb-8">
|
||||
<h1 class="text-2xl font-bold text-gray-800">Pixa URL Generator</h1>
|
||||
<a href="/logout" class="text-sm text-gray-600 hover:text-gray-800 underline">
|
||||
<body>
|
||||
<div class="page">
|
||||
<header>
|
||||
<h1>Pixa URL Generator</h1>
|
||||
<a href="/logout">
|
||||
Logout
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{{if .GeneratedURL}}
|
||||
<div class="bg-green-50 border border-green-200 rounded-lg p-4 mb-6">
|
||||
<h2 class="text-sm font-medium text-green-800 mb-2">Generated URL</h2>
|
||||
<div class="flex gap-2">
|
||||
<div class="result">
|
||||
<h2>Generated URL</h2>
|
||||
<div>
|
||||
<input
|
||||
type="text"
|
||||
readonly
|
||||
value="{{.GeneratedURL}}"
|
||||
id="generated-url"
|
||||
class="flex-1 px-3 py-2 bg-white border border-green-300 rounded-md text-sm font-mono"
|
||||
onclick="this.select()"
|
||||
>
|
||||
<button
|
||||
onclick="navigator.clipboard.writeText(document.getElementById('generated-url').value)"
|
||||
class="px-3 py-2 bg-green-600 text-white rounded-md hover:bg-green-700 text-sm"
|
||||
>
|
||||
<button id="copy-url">
|
||||
Copy
|
||||
</button>
|
||||
</div>
|
||||
<p class="text-xs text-green-600 mt-2">
|
||||
<p>
|
||||
Expires: {{.ExpiresAt}}
|
||||
</p>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{if .Error}}
|
||||
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-6">
|
||||
<div class="error">
|
||||
{{.Error}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<form method="POST" action="/generate" class="bg-white rounded-lg shadow-md p-6 space-y-4">
|
||||
<form method="POST" action="/generate" class="card">
|
||||
{{ .CSRFField }}
|
||||
<div>
|
||||
<label for="url" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="url">
|
||||
Source URL
|
||||
</label>
|
||||
<input
|
||||
@@ -59,13 +54,12 @@
|
||||
required
|
||||
placeholder="https://example.com/image.jpg"
|
||||
value="{{.FormURL}}"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-2 gap-4">
|
||||
<div class="columns">
|
||||
<div>
|
||||
<label for="width" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="width">
|
||||
Width
|
||||
</label>
|
||||
<input
|
||||
@@ -76,11 +70,10 @@
|
||||
max="8192"
|
||||
value="{{if .FormWidth}}{{.FormWidth}}{{else}}0{{end}}"
|
||||
placeholder="0 = original"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
</div>
|
||||
<div>
|
||||
<label for="height" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="height">
|
||||
Height
|
||||
</label>
|
||||
<input
|
||||
@@ -91,21 +84,16 @@
|
||||
max="8192"
|
||||
value="{{if .FormHeight}}{{.FormHeight}}{{else}}0{{end}}"
|
||||
placeholder="0 = original"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-2 gap-4">
|
||||
<div class="columns">
|
||||
<div>
|
||||
<label for="format" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="format">
|
||||
Format
|
||||
</label>
|
||||
<select
|
||||
id="format"
|
||||
name="format"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
<select id="format" name="format">
|
||||
<option value="orig" {{if eq .FormFormat "orig"}}selected{{end}}>Original</option>
|
||||
<option value="jpeg" {{if eq .FormFormat "jpeg"}}selected{{end}}>JPEG</option>
|
||||
<option value="png" {{if eq .FormFormat "png"}}selected{{end}}>PNG</option>
|
||||
@@ -115,14 +103,10 @@
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label for="quality" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="quality">
|
||||
Quality
|
||||
</label>
|
||||
<select
|
||||
id="quality"
|
||||
name="quality"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
<select id="quality" name="quality">
|
||||
<option value="25" {{if eq .FormQuality "25"}}selected{{end}}>Potato</option>
|
||||
<option value="50" {{if eq .FormQuality "50"}}selected{{end}}>Low</option>
|
||||
<option value="70" {{if eq .FormQuality "70"}}selected{{end}}>Medium</option>
|
||||
@@ -132,16 +116,12 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-2 gap-4">
|
||||
<div class="columns">
|
||||
<div>
|
||||
<label for="fit" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="fit">
|
||||
Fit Mode
|
||||
</label>
|
||||
<select
|
||||
id="fit"
|
||||
name="fit"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
<select id="fit" name="fit">
|
||||
<option value="cover" {{if eq .FormFit "cover"}}selected{{end}}>Cover</option>
|
||||
<option value="contain" {{if eq .FormFit "contain"}}selected{{end}}>Contain</option>
|
||||
<option value="fill" {{if eq .FormFit "fill"}}selected{{end}}>Fill</option>
|
||||
@@ -150,14 +130,10 @@
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label for="ttl" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="ttl">
|
||||
Expires In
|
||||
</label>
|
||||
<select
|
||||
id="ttl"
|
||||
name="ttl"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
<select id="ttl" name="ttl">
|
||||
<option value="0" {{if or (eq .FormTTL "0") (eq .FormTTL "")}}selected{{end}}>Never</option>
|
||||
<option value="60" {{if eq .FormTTL "60"}}selected{{end}}>1 minute</option>
|
||||
<option value="3600" {{if eq .FormTTL "3600"}}selected{{end}}>1 hour</option>
|
||||
@@ -169,17 +145,15 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
class="w-full bg-blue-600 text-white py-2 px-4 rounded-md hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 transition-colors"
|
||||
>
|
||||
<button type="submit">
|
||||
Generate Encrypted URL
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<p class="text-xs text-gray-500 mt-4 text-center">
|
||||
<p class="note">
|
||||
Generated URLs are encrypted and cannot be modified. They will expire at the specified time.
|
||||
</p>
|
||||
</div>
|
||||
<script src="/static/generator.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -4,22 +4,22 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Pixa - Login</title>
|
||||
<script src="/static/tailwind.js"></script>
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
<body class="bg-gray-100 min-h-screen flex items-center justify-center">
|
||||
<div class="bg-white p-8 rounded-lg shadow-md w-full max-w-md">
|
||||
<h1 class="text-2xl font-bold text-gray-800 mb-6 text-center">Pixa Image Proxy</h1>
|
||||
<body class="login">
|
||||
<div class="card">
|
||||
<h1>Pixa Image Proxy</h1>
|
||||
|
||||
{{if .Error}}
|
||||
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded mb-4">
|
||||
<div class="error">
|
||||
{{.Error}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<form method="POST" action="/" class="space-y-4">
|
||||
<form method="POST" action="/">
|
||||
{{ .CSRFField }}
|
||||
<div>
|
||||
<label for="key" class="block text-sm font-medium text-gray-700 mb-1">
|
||||
<label for="key">
|
||||
Signing Key
|
||||
</label>
|
||||
<input
|
||||
@@ -28,15 +28,11 @@
|
||||
name="key"
|
||||
required
|
||||
autocomplete="current-password"
|
||||
class="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
placeholder="Enter your signing key"
|
||||
>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="submit"
|
||||
class="w-full bg-blue-600 text-white py-2 px-4 rounded-md hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 transition-colors"
|
||||
>
|
||||
<button type="submit">
|
||||
Login
|
||||
</button>
|
||||
</form>
|
||||
|
||||
Executable
+175
@@ -0,0 +1,175 @@
|
||||
#!/bin/sh
|
||||
# script/loadtest: measure pixad's throughput, latency and peak memory.
|
||||
#
|
||||
# script/loadtest [duration [clients]] (defaults: 10s and 4)
|
||||
#
|
||||
# A benchmark, not a check: script/check does not run it. It needs Docker
|
||||
# and Go. It builds the image with script/docker and builds vegeta, the
|
||||
# load tool, from a pinned commit. Each scenario then gets a new pixad
|
||||
# container and a new origin container (cmd/loadtest-origin, which answers
|
||||
# every path with the same JPEG), and vegeta sends requests for <duration>
|
||||
# from <clients> clients at once, each asking for an image resized to
|
||||
# 400x300 WebP:
|
||||
#
|
||||
# hit the same image every time, put in the cache first
|
||||
# miss a new source image every time
|
||||
# herd each new source image once per client in a row, so that all
|
||||
# clients ask for it at the same time
|
||||
#
|
||||
# For each, it prints vegeta's report, pixad's peak resident memory and
|
||||
# how many requests reached the origin. README.md says how to read them.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
# vegeta v12.13.0, 2026-10-04
|
||||
VEGETA_COMMIT=4b240c3089fa4aa10816542d64a74294d974211f
|
||||
|
||||
# pixad refuses upstream hosts with private or local addresses, so the
|
||||
# containers share a network in 203.0.113.0/24, a range set aside for
|
||||
# documentation (RFC 5737) that pixad does not refuse and that is never
|
||||
# routed on the internet.
|
||||
SUBNET=203.0.113.0/24
|
||||
|
||||
usage() {
|
||||
echo "usage: script/loadtest [duration [clients]]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
main() {
|
||||
duration="${1:-10s}"
|
||||
clients="${2:-4}"
|
||||
# The duration is a whole number, not zero (vegeta takes 0 to mean no
|
||||
# end), followed by ms, s, m or h.
|
||||
case "$duration" in
|
||||
*ms) number="${duration%ms}" ;;
|
||||
*s | *m | *h) number="${duration%?}" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
case "$number" in
|
||||
"" | *[!0-9]*) usage ;;
|
||||
esac
|
||||
[ "$number" -gt 0 ] || usage
|
||||
case "$clients" in
|
||||
*[!0-9]* | 0) usage ;;
|
||||
esac
|
||||
|
||||
cd "$ROOT"
|
||||
run="pixa-loadtest-$$"
|
||||
tmp="$(mktemp -d)"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 1' HUP INT TERM
|
||||
|
||||
"$SCRIPT_DIR/docker"
|
||||
# The image's ID, so a build elsewhere that moves the tag does not
|
||||
# change what a later scenario starts.
|
||||
image="$(docker image inspect --format '{{.Id}}' \
|
||||
"$("$SCRIPT_DIR/projectname")")"
|
||||
|
||||
GOBIN="$tmp" go install "github.com/tsenart/vegeta/v12@$VEGETA_COMMIT"
|
||||
# The origin runs in a container, so it is built for the Docker host.
|
||||
CGO_ENABLED=0 GOOS=linux \
|
||||
GOARCH="$(docker version --format '{{.Server.Arch}}')" \
|
||||
go build -o "$tmp/loadtest-origin" ./cmd/loadtest-origin
|
||||
|
||||
docker network create --subnet "$SUBNET" "$run" >/dev/null
|
||||
|
||||
start_containers
|
||||
# Put the image the hit scenario asks for in the cache.
|
||||
docker exec "$run-pixad" wget -q -O /dev/null \
|
||||
"http://localhost:8080/v1/image/origin/hit.jpg/400x300.webp"
|
||||
attack hit hit_targets
|
||||
stop_containers
|
||||
|
||||
start_containers
|
||||
attack miss miss_targets
|
||||
stop_containers
|
||||
|
||||
start_containers
|
||||
attack herd herd_targets
|
||||
stop_containers
|
||||
}
|
||||
|
||||
# start_containers starts a new origin and a new pixad, and waits up to 30
|
||||
# seconds for pixad's health check to pass.
|
||||
start_containers() {
|
||||
docker run -d --name "$run-origin" \
|
||||
--network "$run" --network-alias origin \
|
||||
-v "$tmp/loadtest-origin:/usr/local/bin/loadtest-origin:ro" \
|
||||
--entrypoint /usr/local/bin/loadtest-origin "$image" >/dev/null
|
||||
docker run -d --name "$run-pixad" \
|
||||
--network "$run" -p 127.0.0.1::8080 --health-interval=1s \
|
||||
-e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \
|
||||
-e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \
|
||||
"$image" >/dev/null
|
||||
|
||||
waited=0
|
||||
until [ "$(docker inspect --format '{{.State.Health.Status}}' \
|
||||
"$run-pixad")" = healthy ]; do
|
||||
if [ "$waited" -ge 30 ]; then
|
||||
echo "loadtest: pixad not healthy after 30 seconds; its log:" >&2
|
||||
docker logs "$run-pixad" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
pixa="http://$(docker port "$run-pixad" 8080/tcp)"
|
||||
}
|
||||
|
||||
stop_containers() {
|
||||
docker rm -f "$run-pixad" "$run-origin" >/dev/null
|
||||
}
|
||||
|
||||
# attack <scenario> <targets>: send the requests <targets> prints and
|
||||
# report on them.
|
||||
attack() {
|
||||
echo
|
||||
echo "== $1: $clients clients for $duration"
|
||||
"$2" | "$tmp/vegeta" attack -lazy -rate 0 -workers "$clients" \
|
||||
-max-workers "$clients" -duration "$duration" -max-body 0 |
|
||||
"$tmp/vegeta" report
|
||||
# pixad is process 1 in its container: the entrypoint execs it.
|
||||
echo "pixad peak memory (VmHWM):" \
|
||||
"$(docker exec "$run-pixad" awk '/^VmHWM:/ { print $2, $3 }' \
|
||||
/proc/1/status)"
|
||||
echo "requests to the origin:" \
|
||||
"$(docker logs "$run-origin" 2>&1 | grep -c ' request ')"
|
||||
}
|
||||
|
||||
# The targets functions print vegeta targets until vegeta stops reading.
|
||||
|
||||
hit_targets() {
|
||||
while :; do
|
||||
echo "GET $pixa/v1/image/origin/hit.jpg/400x300.webp"
|
||||
done
|
||||
}
|
||||
|
||||
miss_targets() {
|
||||
i=0
|
||||
while :; do
|
||||
i=$((i + 1))
|
||||
echo "GET $pixa/v1/image/origin/miss/$i.jpg/400x300.webp"
|
||||
done
|
||||
}
|
||||
|
||||
herd_targets() {
|
||||
i=0
|
||||
while :; do
|
||||
i=$((i + 1))
|
||||
n=0
|
||||
while [ "$n" -lt "$clients" ]; do
|
||||
n=$((n + 1))
|
||||
echo "GET $pixa/v1/image/origin/herd/$i.jpg/400x300.webp"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$run-pixad" "$run-origin" >/dev/null 2>&1 || :
|
||||
docker network rm "$run" >/dev/null 2>&1 || :
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,147 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Manual test script for pixa server
|
||||
# Requires: server running on localhost:8080
|
||||
#
|
||||
set -e
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
SIGNING_KEY="${SIGNING_KEY:-test-signing-key-for-development-only}"
|
||||
TEST_IMAGE_URL="https://s3.sneak.cloud/sneak-public/2021/2021-04-18.untitled.a7r4.07723.jpg"
|
||||
COOKIE_JAR=$(mktemp)
|
||||
|
||||
cleanup() {
|
||||
rm -f "$COOKIE_JAR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
pass() {
|
||||
echo "✓ PASS: $1"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "✗ FAIL: $1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "=== Pixa Manual Test Suite ==="
|
||||
echo "Base URL: $BASE_URL"
|
||||
echo ""
|
||||
|
||||
# Test 1: Healthcheck
|
||||
echo "--- Test 1: Healthcheck endpoint ---"
|
||||
HEALTH=$(curl -sf "$BASE_URL/.well-known/healthcheck.json")
|
||||
if echo "$HEALTH" | grep -q '"status"'; then
|
||||
pass "Healthcheck returns status"
|
||||
else
|
||||
fail "Healthcheck did not return expected response"
|
||||
fi
|
||||
|
||||
# Test 2: Login page displays
|
||||
echo "--- Test 2: Login page (GET /) ---"
|
||||
LOGIN_PAGE=$(curl -sf "$BASE_URL/")
|
||||
if echo "$LOGIN_PAGE" | grep -qi "password\|login\|sign"; then
|
||||
pass "Login page displays password form"
|
||||
else
|
||||
fail "Login page did not display expected content"
|
||||
fi
|
||||
|
||||
# Test 3: Wrong password shows error
|
||||
echo "--- Test 3: Login with wrong password ---"
|
||||
WRONG_LOGIN=$(curl -sf -X POST "$BASE_URL/" -d "key=wrong-key" -c "$COOKIE_JAR")
|
||||
if echo "$WRONG_LOGIN" | grep -qi "invalid\|error\|incorrect\|wrong"; then
|
||||
pass "Wrong password shows error message"
|
||||
else
|
||||
fail "Wrong password did not show error"
|
||||
fi
|
||||
|
||||
# Test 4: Correct password redirects to generator
|
||||
echo "--- Test 4: Login with correct signing key ---"
|
||||
curl -sf -X POST "$BASE_URL/" -d "key=$SIGNING_KEY" -c "$COOKIE_JAR" -b "$COOKIE_JAR" -L -o /dev/null
|
||||
GENERATOR_PAGE=$(curl -sf "$BASE_URL/" -b "$COOKIE_JAR")
|
||||
if echo "$GENERATOR_PAGE" | grep -qi "generate\|url\|source\|logout"; then
|
||||
pass "Correct password shows generator page"
|
||||
else
|
||||
fail "Generator page not displayed after login"
|
||||
fi
|
||||
|
||||
# Test 5: Generate encrypted URL
|
||||
echo "--- Test 5: Generate encrypted URL ---"
|
||||
GEN_RESULT=$(curl -sf -X POST "$BASE_URL/generate" -b "$COOKIE_JAR" \
|
||||
-d "url=$TEST_IMAGE_URL" \
|
||||
-d "width=800" \
|
||||
-d "height=600" \
|
||||
-d "format=jpeg" \
|
||||
-d "quality=85" \
|
||||
-d "fit=cover" \
|
||||
-d "ttl=3600")
|
||||
if echo "$GEN_RESULT" | grep -q "/v1/e/"; then
|
||||
pass "Encrypted URL generated"
|
||||
# Extract the encrypted URL
|
||||
ENC_URL=$(echo "$GEN_RESULT" | grep -o '/v1/e/[^"<]*' | head -1)
|
||||
echo " Generated URL: $ENC_URL"
|
||||
else
|
||||
fail "Failed to generate encrypted URL"
|
||||
fi
|
||||
|
||||
# Test 6: Fetch image via encrypted URL
|
||||
echo "--- Test 6: Fetch image via encrypted URL ---"
|
||||
if [ -n "$ENC_URL" ]; then
|
||||
HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$BASE_URL$ENC_URL")
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
pass "Encrypted URL returns image (HTTP 200)"
|
||||
else
|
||||
fail "Encrypted URL returned HTTP $HTTP_CODE"
|
||||
fi
|
||||
else
|
||||
fail "No encrypted URL to test"
|
||||
fi
|
||||
|
||||
# Test 7: Fetch image via allowlisted host (direct proxy)
|
||||
echo "--- Test 7: Fetch image via direct proxy (allowlisted host) ---"
|
||||
# URL format: /v1/image/<host>/<path>/<WxH>.<format>
|
||||
PROXY_PATH="/v1/image/s3.sneak.cloud/sneak-public/2021/2021-04-18.untitled.a7r4.07723.jpg/400x300.jpeg"
|
||||
HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$BASE_URL$PROXY_PATH")
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
pass "Direct proxy returns image (HTTP 200)"
|
||||
else
|
||||
fail "Direct proxy returned HTTP $HTTP_CODE"
|
||||
fi
|
||||
|
||||
# Test 8: Logout
|
||||
echo "--- Test 8: Logout ---"
|
||||
curl -sf "$BASE_URL/logout" -b "$COOKIE_JAR" -c "$COOKIE_JAR" -L -o /dev/null
|
||||
AFTER_LOGOUT=$(curl -sf "$BASE_URL/" -b "$COOKIE_JAR")
|
||||
if echo "$AFTER_LOGOUT" | grep -qi "password\|login"; then
|
||||
pass "Logout redirects to login page"
|
||||
else
|
||||
fail "Logout did not redirect to login"
|
||||
fi
|
||||
|
||||
# Test 9: Generate short-TTL URL and verify expiration
|
||||
echo "--- Test 9: Expired URL returns 410 ---"
|
||||
# Login again
|
||||
curl -sf -X POST "$BASE_URL/" -d "key=$SIGNING_KEY" -c "$COOKIE_JAR" -b "$COOKIE_JAR" -L -o /dev/null
|
||||
# Generate URL with 1 second TTL
|
||||
GEN_RESULT=$(curl -sf -X POST "$BASE_URL/generate" -b "$COOKIE_JAR" \
|
||||
-d "url=$TEST_IMAGE_URL" \
|
||||
-d "width=100" \
|
||||
-d "height=100" \
|
||||
-d "format=jpeg" \
|
||||
-d "ttl=1")
|
||||
SHORT_URL=$(echo "$GEN_RESULT" | grep -o '/v1/e/[^"<]*' | head -1)
|
||||
if [ -n "$SHORT_URL" ]; then
|
||||
echo " Waiting 2 seconds for URL to expire..."
|
||||
sleep 2
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$BASE_URL$SHORT_URL")
|
||||
if [ "$HTTP_CODE" = "410" ]; then
|
||||
pass "Expired URL returns 410 Gone"
|
||||
else
|
||||
fail "Expired URL returned HTTP $HTTP_CODE (expected 410)"
|
||||
fi
|
||||
else
|
||||
fail "Could not generate short-TTL URL"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== All tests passed! ==="
|
||||
Reference in New Issue
Block a user