diff --git a/Makefile b/Makefile index 708d5e5..ee52f5c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks +.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") LDFLAGS := -X main.Version=$(VERSION) @@ -59,6 +59,11 @@ docker: docker-smoke: @script/docker-smoke +# Measure throughput, latency and peak memory with the default duration and +# number of clients (needs Docker and Go; a benchmark, not part of check) +loadtest: + @script/loadtest + # Build Docker image tagged pixad:$(VERSION) and pixad:latest docker-versioned: docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest . diff --git a/README.md b/README.md index 8597c53..ac26fa2 100644 --- a/README.md +++ b/README.md @@ -146,10 +146,11 @@ name. Multiple source paths may reference the same content blob; the database tracks references rather than using filesystem refcounting. -Toward a target of 1-5k r/s, pixa keeps in memory the content types of -the 10,000 transformed images most recently cached or served, so a -cache hit on one of them reads only the image file from disk and not -the metadata file stored beside it. + +pixa's target is 1-5k r/s, which has not been measured at that rate (see Load +Test). Toward it, pixa keeps in memory the content types of the 10,000 +transformed images most recently cached or served, so a cache hit on one of them +reads only the image file from disk and not the metadata file stored beside it. ### Routes @@ -576,6 +577,8 @@ them. We provide: - `script/check` — run test, lint, and fmt-check - `script/docker` — build the Docker image tagged via `script/projectname` - `script/docker-smoke` — build the image, start it, wait for it to be healthy +- `script/loadtest` — measure pixad's throughput, latency and peak memory; a + benchmark, not part of `script/check` (see Load Test) - `script/cibuild` — CI entrypoint: `docker build .` with a new `CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of coming from the build cache, and a green run implies a green repo @@ -584,6 +587,58 @@ them. We provide: - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` +## Load Test + +`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how +much memory it uses. It is a benchmark, not a check: `script/check` does not run +it. It needs Docker and Go. + +```bash +script/loadtest # 10 seconds per scenario, 4 clients +script/loadtest 30s 32 # 30 seconds per scenario, 32 clients +``` + +It builds the image with `script/docker` and the load tool, +[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario +starts a new pixad container and a new origin container, `cmd/loadtest-origin`: +an upstream host that answers every path with the same generated 1600x1200 JPEG. +vegeta then sends requests from the given number of clients, each sending its +next request as soon as its last one is answered, all for an image resized to +400x300 WebP: + +- `hit`: the same image every time, put in the cache first; +- `miss`: a new source image every time, so pixad fetches and converts each one; +- `herd`: each new source image once per client in a row, so that all clients + ask for it at the same time and share one fetch and one conversion (see + Routes). + +pixad refuses upstream hosts with private or local addresses, so the containers +share a Docker network in `203.0.113.0/24`, a range set aside for documentation. +A second run on the same Docker host while one is going fails, as it cannot +create that network. + +For each scenario the script prints vegeta's report and two lines of its own: + +- `Requests [total, rate, throughput]`: the requests sent, how many were sent + per second, and how many were answered successfully per second; the last is + the number to compare with the target under Storage; +- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request + to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th + percentiles; +- `Status Codes` and `Error Set`: anything other than `200` means the other + numbers are not for the scenario described, such as `503` when pixad was busy; +- `Bytes In`: `0`, as vegeta is told not to keep the images it receives; +- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since + its container started, in kB; for `hit` it includes the request that put the + image in the cache; +- `requests to the origin`: the fetches pixad made: one for `hit`, one per + request for `miss`, and one per image for `herd`, that is the requests sent + divided by the number of clients. + +The numbers depend on the machine and on whatever else runs on it. The first +measurement, made on a shared machine with few clients, is in `TODO.md`; it says +nothing about the target. + ## TODO See [TODO.md](TODO.md) for the full prioritized task list. diff --git a/TODO.md b/TODO.md index 7fecf2b..a7cb363 100644 --- a/TODO.md +++ b/TODO.md @@ -31,6 +31,21 @@ P2: security: per-IP rate limiting on the image routes # Completed Steps +- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]` + (`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check` + does not run, measures three scenarios, each against a new pixad container and + a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss` + (a new source image every request) and `herd` (each new source image asked for + by all clients at once). For each it prints vegeta's report (requests per + second, latency percentiles, status codes), pixad's peak resident memory and + the requests that reached the origin. `README.md` says how to run it and read + it, and keeps 1-5k r/s as a target not yet measured. First measurement, with + the defaults on a shared 48-CPU machine with other work running: a baseline + for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95 + 8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so + not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100 + MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms, + peak 60 MiB, 188 fetches for 749 requests. - 2026-10-04 `REPO_POLICIES.md` matches the canonical copy again (closes #196): it is replaced, unchanged, by `prompts/REPO_POLICIES.md` from `sneak/prompts` `main`. The rules it adds that pixa's tree breaks are filed: @@ -621,4 +636,5 @@ P2: security: per-IP rate limiting on the image routes - optional Sentry error reporting - comprehensive request logging - Prometheus performance metrics - - load tests to verify the 1k to 5k req/s target + - measure the 1k to 5k req/s target with `script/loadtest` on a machine not + shared with other work diff --git a/cmd/loadtest-origin/main.go b/cmd/loadtest-origin/main.go new file mode 100644 index 0000000..19dfe5e --- /dev/null +++ b/cmd/loadtest-origin/main.go @@ -0,0 +1,84 @@ +// Command loadtest-origin is the upstream host script/loadtest points pixad +// at. It answers every request, whatever its path, with the same generated +// JPEG, so each new path is a new source image for pixad to fetch, and it logs +// one line per request, so its log counts pixad's fetches. +package main + +import ( + "bytes" + "image" + "image/color" + "image/jpeg" + "log/slog" + "math" + "net/http" + "os" + "time" +) + +const ( + listenAddress = ":80" + readHeaderTimeout = 10 * time.Second + imageWidth = 1600 + imageHeight = 1200 + jpegQuality = 85 +) + +func main() { + photo, err := makeJPEG() + if err != nil { + slog.Error("cannot make the image", "error", err) + os.Exit(1) + } + + server := &http.Server{ + Addr: listenAddress, + Handler: newHandler(photo), + ReadHeaderTimeout: readHeaderTimeout, + } + + err = server.ListenAndServe() + slog.Error("server stopped", "error", err) + os.Exit(1) +} + +// newHandler answers every request with photo and logs the request's path. +func newHandler(photo []byte) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + slog.Info("request", "path", r.URL.Path) + w.Header().Set("Content-Type", "image/jpeg") + _, _ = w.Write(photo) + }) +} + +// makeJPEG draws colour gradients crossed with a fine pattern, so the image +// has detail to decode and does not compress to almost nothing. +func makeJPEG() ([]byte, error) { + img := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight)) + + // red and green count up from 0 to 255 and wrap around, along each row + // and down the image. + var green uint8 + + for y := range imageHeight { + var red uint8 + + for x := range imageWidth { + img.SetRGBA(x, y, color.RGBA{ + R: red, G: green, B: red ^ green, A: math.MaxUint8, + }) + red++ + } + + green++ + } + + var buf bytes.Buffer + + err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/script/loadtest b/script/loadtest new file mode 100755 index 0000000..3639899 --- /dev/null +++ b/script/loadtest @@ -0,0 +1,162 @@ +#!/bin/sh +# script/loadtest: measure pixad's throughput, latency and peak memory. +# +# script/loadtest [duration [clients]] (defaults: 10s and 4) +# +# A benchmark, not a check: script/check does not run it. It needs Docker +# and Go. It builds the image with script/docker and builds vegeta, the +# load tool, from a pinned commit. Each scenario then gets a new pixad +# container and a new origin container (cmd/loadtest-origin, which answers +# every path with the same JPEG), and vegeta sends requests for +# from clients at once, each asking for an image resized to +# 400x300 WebP: +# +# hit the same image every time, put in the cache first +# miss a new source image every time +# herd each new source image once per client in a row, so that all +# clients ask for it at the same time +# +# For each, it prints vegeta's report, pixad's peak resident memory and +# how many requests reached the origin. README.md says how to read them. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +# vegeta v12.13.0, 2026-10-04 +VEGETA_COMMIT=4b240c3089fa4aa10816542d64a74294d974211f + +# pixad refuses upstream hosts with private or local addresses, so the +# containers share a network in 203.0.113.0/24, a range set aside for +# documentation (RFC 5737) that pixad does not refuse and that is never +# routed on the internet. +SUBNET=203.0.113.0/24 + +main() { + duration="${1:-10s}" + clients="${2:-4}" + case "$clients" in + *[!0-9]* | 0) + echo "usage: script/loadtest [duration [clients]]" >&2 + exit 2 + ;; + esac + + cd "$ROOT" + run="pixa-loadtest-$$" + tmp="$(mktemp -d)" + trap cleanup EXIT + trap 'exit 1' HUP INT TERM + + "$SCRIPT_DIR/docker" + # The image's ID, so a build elsewhere that moves the tag does not + # change what a later scenario starts. + image="$(docker image inspect --format '{{.Id}}' \ + "$("$SCRIPT_DIR/projectname")")" + + GOBIN="$tmp" go install "github.com/tsenart/vegeta/v12@$VEGETA_COMMIT" + # The origin runs in a container, so it is built for the Docker host. + CGO_ENABLED=0 GOOS=linux \ + GOARCH="$(docker version --format '{{.Server.Arch}}')" \ + go build -o "$tmp/loadtest-origin" ./cmd/loadtest-origin + + docker network create --subnet "$SUBNET" "$run" >/dev/null + + start_containers + # Put the image the hit scenario asks for in the cache. + docker exec "$run-pixad" wget -q -O /dev/null \ + "http://localhost:8080/v1/image/origin/hit.jpg/400x300.webp" + attack hit hit_targets + stop_containers + + start_containers + attack miss miss_targets + stop_containers + + start_containers + attack herd herd_targets + stop_containers +} + +# start_containers starts a new origin and a new pixad, and waits up to 30 +# seconds for pixad's health check to pass. +start_containers() { + docker run -d --name "$run-origin" \ + --network "$run" --network-alias origin \ + -v "$tmp/loadtest-origin:/usr/local/bin/loadtest-origin:ro" \ + --entrypoint /usr/local/bin/loadtest-origin "$image" >/dev/null + docker run -d --name "$run-pixad" \ + --network "$run" -p 127.0.0.1::8080 --health-interval=1s \ + -e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \ + -e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \ + "$image" >/dev/null + + waited=0 + until [ "$(docker inspect --format '{{.State.Health.Status}}' \ + "$run-pixad")" = healthy ]; do + if [ "$waited" -ge 30 ]; then + echo "loadtest: pixad not healthy after 30 seconds; its log:" >&2 + docker logs "$run-pixad" >&2 + exit 1 + fi + sleep 1 + waited=$((waited + 1)) + done + pixa="http://$(docker port "$run-pixad" 8080/tcp)" +} + +stop_containers() { + docker rm -f "$run-pixad" "$run-origin" >/dev/null +} + +# attack : send the requests prints and +# report on them. +attack() { + echo + echo "== $1: $clients clients for $duration" + "$2" | "$tmp/vegeta" attack -lazy -rate 0 -workers "$clients" \ + -max-workers "$clients" -duration "$duration" -max-body 0 | + "$tmp/vegeta" report + # pixad is process 1 in its container: the entrypoint execs it. + echo "pixad peak memory (VmHWM):" \ + "$(docker exec "$run-pixad" awk '/^VmHWM:/ { print $2, $3 }' \ + /proc/1/status)" + echo "requests to the origin:" \ + "$(docker logs "$run-origin" 2>&1 | grep -c ' request ')" +} + +# The targets functions print vegeta targets until vegeta stops reading. + +hit_targets() { + while :; do + echo "GET $pixa/v1/image/origin/hit.jpg/400x300.webp" + done +} + +miss_targets() { + i=0 + while :; do + i=$((i + 1)) + echo "GET $pixa/v1/image/origin/miss/$i.jpg/400x300.webp" + done +} + +herd_targets() { + i=0 + while :; do + i=$((i + 1)) + n=0 + while [ "$n" -lt "$clients" ]; do + n=$((n + 1)) + echo "GET $pixa/v1/image/origin/herd/$i.jpg/400x300.webp" + done + done +} + +cleanup() { + docker rm -f "$run-pixad" "$run-origin" >/dev/null 2>&1 || : + docker network rm "$run" >/dev/null 2>&1 || : + rm -rf "$tmp" +} + +main "$@"