Add script/loadtest to measure throughput, latency and memory (closes #81)
script/loadtest [duration [clients]], or make loadtest, is a benchmark that script/check does not run. It builds the image and vegeta from a pinned commit, then for each scenario starts a new pixad container and a new cmd/loadtest-origin container, an upstream host that answers every path with one generated JPEG: a cached image (hit), a new source image per request (miss), and each new source image asked for by all clients at once (herd). It prints vegeta's report, pixad's peak resident memory and the requests the origin got. The containers share a network in 203.0.113.0/24, as pixad refuses private and local upstream addresses. README.md says how to run and read it; TODO.md records a small baseline. Model: opus-5-5
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks
|
||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest
|
||||
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
LDFLAGS := -X main.Version=$(VERSION)
|
||||
@@ -59,6 +59,11 @@ docker:
|
||||
docker-smoke:
|
||||
@script/docker-smoke
|
||||
|
||||
# Measure throughput, latency and peak memory with the default duration and
|
||||
# number of clients (needs Docker and Go; a benchmark, not part of check)
|
||||
loadtest:
|
||||
@script/loadtest
|
||||
|
||||
# Build Docker image tagged pixad:$(VERSION) and pixad:latest
|
||||
docker-versioned:
|
||||
docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
|
||||
|
||||
@@ -146,10 +146,11 @@ name.
|
||||
|
||||
Multiple source paths may reference the same content blob; the
|
||||
database tracks references rather than using filesystem refcounting.
|
||||
Toward a target of 1-5k r/s, pixa keeps in memory the content types of
|
||||
the 10,000 transformed images most recently cached or served, so a
|
||||
cache hit on one of them reads only the image file from disk and not
|
||||
the metadata file stored beside it.
|
||||
|
||||
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
||||
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
||||
transformed images most recently cached or served, so a cache hit on one of them
|
||||
reads only the image file from disk and not the metadata file stored beside it.
|
||||
|
||||
### Routes
|
||||
|
||||
@@ -576,6 +577,8 @@ them. We provide:
|
||||
- `script/check` — run test, lint, and fmt-check
|
||||
- `script/docker` — build the Docker image tagged via `script/projectname`
|
||||
- `script/docker-smoke` — build the image, start it, wait for it to be healthy
|
||||
- `script/loadtest` — measure pixad's throughput, latency and peak memory; a
|
||||
benchmark, not part of `script/check` (see Load Test)
|
||||
- `script/cibuild` — CI entrypoint: `docker build .` with a new
|
||||
`CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of
|
||||
coming from the build cache, and a green run implies a green repo
|
||||
@@ -584,6 +587,58 @@ them. We provide:
|
||||
- `script/install-precommit` — install the git pre-commit hook that
|
||||
runs `script/precommit`
|
||||
|
||||
## Load Test
|
||||
|
||||
`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how
|
||||
much memory it uses. It is a benchmark, not a check: `script/check` does not run
|
||||
it. It needs Docker and Go.
|
||||
|
||||
```bash
|
||||
script/loadtest # 10 seconds per scenario, 4 clients
|
||||
script/loadtest 30s 32 # 30 seconds per scenario, 32 clients
|
||||
```
|
||||
|
||||
It builds the image with `script/docker` and the load tool,
|
||||
[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario
|
||||
starts a new pixad container and a new origin container, `cmd/loadtest-origin`:
|
||||
an upstream host that answers every path with the same generated 1600x1200 JPEG.
|
||||
vegeta then sends requests from the given number of clients, each sending its
|
||||
next request as soon as its last one is answered, all for an image resized to
|
||||
400x300 WebP:
|
||||
|
||||
- `hit`: the same image every time, put in the cache first;
|
||||
- `miss`: a new source image every time, so pixad fetches and converts each one;
|
||||
- `herd`: each new source image once per client in a row, so that all clients
|
||||
ask for it at the same time and share one fetch and one conversion (see
|
||||
Routes).
|
||||
|
||||
pixad refuses upstream hosts with private or local addresses, so the containers
|
||||
share a Docker network in `203.0.113.0/24`, a range set aside for documentation.
|
||||
A second run on the same Docker host while one is going fails, as it cannot
|
||||
create that network.
|
||||
|
||||
For each scenario the script prints vegeta's report and two lines of its own:
|
||||
|
||||
- `Requests [total, rate, throughput]`: the requests sent, how many were sent
|
||||
per second, and how many were answered successfully per second; the last is
|
||||
the number to compare with the target under Storage;
|
||||
- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request
|
||||
to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th
|
||||
percentiles;
|
||||
- `Status Codes` and `Error Set`: anything other than `200` means the other
|
||||
numbers are not for the scenario described, such as `503` when pixad was busy;
|
||||
- `Bytes In`: `0`, as vegeta is told not to keep the images it receives;
|
||||
- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since
|
||||
its container started, in kB; for `hit` it includes the request that put the
|
||||
image in the cache;
|
||||
- `requests to the origin`: the fetches pixad made: one for `hit`, one per
|
||||
request for `miss`, and one per image for `herd`, that is the requests sent
|
||||
divided by the number of clients.
|
||||
|
||||
The numbers depend on the machine and on whatever else runs on it. The first
|
||||
measurement, made on a shared machine with few clients, is in `TODO.md`; it says
|
||||
nothing about the target.
|
||||
|
||||
## TODO
|
||||
|
||||
See [TODO.md](TODO.md) for the full prioritized task list.
|
||||
|
||||
@@ -31,6 +31,21 @@ P2: security: per-IP rate limiting on the image routes
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]`
|
||||
(`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check`
|
||||
does not run, measures three scenarios, each against a new pixad container and
|
||||
a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss`
|
||||
(a new source image every request) and `herd` (each new source image asked for
|
||||
by all clients at once). For each it prints vegeta's report (requests per
|
||||
second, latency percentiles, status codes), pixad's peak resident memory and
|
||||
the requests that reached the origin. `README.md` says how to run it and read
|
||||
it, and keeps 1-5k r/s as a target not yet measured. First measurement, with
|
||||
the defaults on a shared 48-CPU machine with other work running: a baseline
|
||||
for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95
|
||||
8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so
|
||||
not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100
|
||||
MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms,
|
||||
peak 60 MiB, 188 fetches for 749 requests.
|
||||
- 2026-10-04 `.dockerignore` keeps secrets out at every depth (closes #205): the
|
||||
file is now the standard one from `sneak/prompts`, whose patterns match in
|
||||
every directory and, for environment files and private keys, in any letter
|
||||
@@ -627,4 +642,5 @@ P2: security: per-IP rate limiting on the image routes
|
||||
- optional Sentry error reporting
|
||||
- comprehensive request logging
|
||||
- Prometheus performance metrics
|
||||
- load tests to verify the 1k to 5k req/s target
|
||||
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
||||
shared with other work
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Command loadtest-origin is the upstream host script/loadtest points pixad
|
||||
// at. It answers every request, whatever its path, with the same generated
|
||||
// JPEG, so each new path is a new source image for pixad to fetch, and it logs
|
||||
// one line per request, so its log counts pixad's fetches.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
listenAddress = ":80"
|
||||
readHeaderTimeout = 10 * time.Second
|
||||
imageWidth = 1600
|
||||
imageHeight = 1200
|
||||
jpegQuality = 85
|
||||
)
|
||||
|
||||
func main() {
|
||||
photo, err := makeJPEG()
|
||||
if err != nil {
|
||||
slog.Error("cannot make the image", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: listenAddress,
|
||||
Handler: newHandler(photo),
|
||||
ReadHeaderTimeout: readHeaderTimeout,
|
||||
}
|
||||
|
||||
err = server.ListenAndServe()
|
||||
slog.Error("server stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// newHandler answers every request with photo and logs the request's path.
|
||||
func newHandler(photo []byte) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
slog.Info("request", "path", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(photo)
|
||||
})
|
||||
}
|
||||
|
||||
// makeJPEG draws colour gradients crossed with a fine pattern, so the image
|
||||
// has detail to decode and does not compress to almost nothing.
|
||||
func makeJPEG() ([]byte, error) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight))
|
||||
|
||||
// red and green count up from 0 to 255 and wrap around, along each row
|
||||
// and down the image.
|
||||
var green uint8
|
||||
|
||||
for y := range imageHeight {
|
||||
var red uint8
|
||||
|
||||
for x := range imageWidth {
|
||||
img.SetRGBA(x, y, color.RGBA{
|
||||
R: red, G: green, B: red ^ green, A: math.MaxUint8,
|
||||
})
|
||||
red++
|
||||
}
|
||||
|
||||
green++
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
Executable
+162
@@ -0,0 +1,162 @@
|
||||
#!/bin/sh
|
||||
# script/loadtest: measure pixad's throughput, latency and peak memory.
|
||||
#
|
||||
# script/loadtest [duration [clients]] (defaults: 10s and 4)
|
||||
#
|
||||
# A benchmark, not a check: script/check does not run it. It needs Docker
|
||||
# and Go. It builds the image with script/docker and builds vegeta, the
|
||||
# load tool, from a pinned commit. Each scenario then gets a new pixad
|
||||
# container and a new origin container (cmd/loadtest-origin, which answers
|
||||
# every path with the same JPEG), and vegeta sends requests for <duration>
|
||||
# from <clients> clients at once, each asking for an image resized to
|
||||
# 400x300 WebP:
|
||||
#
|
||||
# hit the same image every time, put in the cache first
|
||||
# miss a new source image every time
|
||||
# herd each new source image once per client in a row, so that all
|
||||
# clients ask for it at the same time
|
||||
#
|
||||
# For each, it prints vegeta's report, pixad's peak resident memory and
|
||||
# how many requests reached the origin. README.md says how to read them.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
# vegeta v12.13.0, 2026-10-04
|
||||
VEGETA_COMMIT=4b240c3089fa4aa10816542d64a74294d974211f
|
||||
|
||||
# pixad refuses upstream hosts with private or local addresses, so the
|
||||
# containers share a network in 203.0.113.0/24, a range set aside for
|
||||
# documentation (RFC 5737) that pixad does not refuse and that is never
|
||||
# routed on the internet.
|
||||
SUBNET=203.0.113.0/24
|
||||
|
||||
main() {
|
||||
duration="${1:-10s}"
|
||||
clients="${2:-4}"
|
||||
case "$clients" in
|
||||
*[!0-9]* | 0)
|
||||
echo "usage: script/loadtest [duration [clients]]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
cd "$ROOT"
|
||||
run="pixa-loadtest-$$"
|
||||
tmp="$(mktemp -d)"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 1' HUP INT TERM
|
||||
|
||||
"$SCRIPT_DIR/docker"
|
||||
# The image's ID, so a build elsewhere that moves the tag does not
|
||||
# change what a later scenario starts.
|
||||
image="$(docker image inspect --format '{{.Id}}' \
|
||||
"$("$SCRIPT_DIR/projectname")")"
|
||||
|
||||
GOBIN="$tmp" go install "github.com/tsenart/vegeta/v12@$VEGETA_COMMIT"
|
||||
# The origin runs in a container, so it is built for the Docker host.
|
||||
CGO_ENABLED=0 GOOS=linux \
|
||||
GOARCH="$(docker version --format '{{.Server.Arch}}')" \
|
||||
go build -o "$tmp/loadtest-origin" ./cmd/loadtest-origin
|
||||
|
||||
docker network create --subnet "$SUBNET" "$run" >/dev/null
|
||||
|
||||
start_containers
|
||||
# Put the image the hit scenario asks for in the cache.
|
||||
docker exec "$run-pixad" wget -q -O /dev/null \
|
||||
"http://localhost:8080/v1/image/origin/hit.jpg/400x300.webp"
|
||||
attack hit hit_targets
|
||||
stop_containers
|
||||
|
||||
start_containers
|
||||
attack miss miss_targets
|
||||
stop_containers
|
||||
|
||||
start_containers
|
||||
attack herd herd_targets
|
||||
stop_containers
|
||||
}
|
||||
|
||||
# start_containers starts a new origin and a new pixad, and waits up to 30
|
||||
# seconds for pixad's health check to pass.
|
||||
start_containers() {
|
||||
docker run -d --name "$run-origin" \
|
||||
--network "$run" --network-alias origin \
|
||||
-v "$tmp/loadtest-origin:/usr/local/bin/loadtest-origin:ro" \
|
||||
--entrypoint /usr/local/bin/loadtest-origin "$image" >/dev/null
|
||||
docker run -d --name "$run-pixad" \
|
||||
--network "$run" -p 127.0.0.1::8080 --health-interval=1s \
|
||||
-e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \
|
||||
-e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \
|
||||
"$image" >/dev/null
|
||||
|
||||
waited=0
|
||||
until [ "$(docker inspect --format '{{.State.Health.Status}}' \
|
||||
"$run-pixad")" = healthy ]; do
|
||||
if [ "$waited" -ge 30 ]; then
|
||||
echo "loadtest: pixad not healthy after 30 seconds; its log:" >&2
|
||||
docker logs "$run-pixad" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
pixa="http://$(docker port "$run-pixad" 8080/tcp)"
|
||||
}
|
||||
|
||||
stop_containers() {
|
||||
docker rm -f "$run-pixad" "$run-origin" >/dev/null
|
||||
}
|
||||
|
||||
# attack <scenario> <targets>: send the requests <targets> prints and
|
||||
# report on them.
|
||||
attack() {
|
||||
echo
|
||||
echo "== $1: $clients clients for $duration"
|
||||
"$2" | "$tmp/vegeta" attack -lazy -rate 0 -workers "$clients" \
|
||||
-max-workers "$clients" -duration "$duration" -max-body 0 |
|
||||
"$tmp/vegeta" report
|
||||
# pixad is process 1 in its container: the entrypoint execs it.
|
||||
echo "pixad peak memory (VmHWM):" \
|
||||
"$(docker exec "$run-pixad" awk '/^VmHWM:/ { print $2, $3 }' \
|
||||
/proc/1/status)"
|
||||
echo "requests to the origin:" \
|
||||
"$(docker logs "$run-origin" 2>&1 | grep -c ' request ')"
|
||||
}
|
||||
|
||||
# The targets functions print vegeta targets until vegeta stops reading.
|
||||
|
||||
hit_targets() {
|
||||
while :; do
|
||||
echo "GET $pixa/v1/image/origin/hit.jpg/400x300.webp"
|
||||
done
|
||||
}
|
||||
|
||||
miss_targets() {
|
||||
i=0
|
||||
while :; do
|
||||
i=$((i + 1))
|
||||
echo "GET $pixa/v1/image/origin/miss/$i.jpg/400x300.webp"
|
||||
done
|
||||
}
|
||||
|
||||
herd_targets() {
|
||||
i=0
|
||||
while :; do
|
||||
i=$((i + 1))
|
||||
n=0
|
||||
while [ "$n" -lt "$clients" ]; do
|
||||
n=$((n + 1))
|
||||
echo "GET $pixa/v1/image/origin/herd/$i.jpg/400x300.webp"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$run-pixad" "$run-origin" >/dev/null 2>&1 || :
|
||||
docker network rm "$run" >/dev/null 2>&1 || :
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user