Compare commits
4
Commits
468439a229
...
60939f4c84
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
60939f4c84 | ||
|
|
55cf7f4fac | ||
|
|
c434581a54 | ||
|
|
f77faf13de |
@@ -68,5 +68,6 @@
|
|||||||
/data
|
/data
|
||||||
|
|
||||||
# Local config files, kept out of git because they can hold the signing key.
|
# Local config files, kept out of git because they can hold the signing key.
|
||||||
|
**/[cC][oO][nN][fF][iI][gG].[yY][mM][lL]
|
||||||
**/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL]
|
**/[cC][oO][nN][fF][iI][gG].[yY][aA][mM][lL]
|
||||||
**/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL]
|
**/[cC][oO][nN][fF][iI][gG].[dD][eE][vV].[yY][mM][lL]
|
||||||
|
|||||||
@@ -6,5 +6,10 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
# actions/checkout v4.2.2, 2026-02-22
|
# actions/checkout v4.2.2, 2026-02-22
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||||
|
# The default clone is shallow and has no tags, so the
|
||||||
|
# version the build takes from `git describe` would be a
|
||||||
|
# bare commit; this fetches the whole history with its tags.
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
- run: script/cibuild
|
- run: script/cibuild
|
||||||
- run: script/docker-smoke
|
- run: script/docker-smoke
|
||||||
|
|||||||
@@ -37,5 +37,6 @@ node_modules/
|
|||||||
*.sqlite3
|
*.sqlite3
|
||||||
|
|
||||||
# Local dev configs
|
# Local dev configs
|
||||||
|
config.yml
|
||||||
config.yaml
|
config.yaml
|
||||||
config.dev.yml
|
config.dev.yml
|
||||||
|
|||||||
+38
-30
@@ -1,55 +1,62 @@
|
|||||||
# Lint stage
|
# Lint phase. script/lint builds it alone. The linter is run directly:
|
||||||
# Same image as Dockerfile.lint: change both pins together.
|
# `make lint` and script/lint are themselves a docker build.
|
||||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
# golangci/golangci-lint:v2.12.2, 2026-10-04
|
||||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
|
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
|
|
||||||
|
# The linter compiles every package, and govips needs the libvips
|
||||||
|
# headers for that. REPO_POLICIES.md has the lint phase install them
|
||||||
|
# itself; this image is Debian, so with apt-get rather than apk.
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
|
# Test phase. script/test builds it alone.
|
||||||
|
# golang:1.25.4-alpine, 2026-02-25
|
||||||
|
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS test
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# script/bootstrap installs the build dependencies and downloads the Go
|
# script/bootstrap --cgo installs the build dependencies (a C compiler
|
||||||
# modules. Only script/, go.mod and go.sum are copied first, so this
|
# and the libvips and libheif headers) and downloads the Go modules.
|
||||||
# layer is reused until one of them changes.
|
|
||||||
COPY script/ ./script/
|
COPY script/ ./script/
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN script/bootstrap
|
RUN script/bootstrap --cgo
|
||||||
|
|
||||||
# Copy source code
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Tells script/lint it is inside a container, so it runs the linter.
|
# Without -v first; on a failure, again with -v for the details, and
|
||||||
ENV container=docker
|
# the step fails even if the second run passes.
|
||||||
|
RUN go test -count=1 -timeout 90s -race -cover ./... || \
|
||||||
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
|
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
# Run formatting check and linter. script/cibuild and script/docker pass
|
# Build stage. Nothing is wanted from the two phases above: these copies
|
||||||
# a new CHECK_EPOCH on every run, and each check step names it in its
|
# make BuildKit build them first, so this stage runs only when lint and
|
||||||
# command, so a new value reruns the step instead of reusing a cached
|
# test passed.
|
||||||
# success that checked nothing. A plain `docker build .` leaves it empty
|
|
||||||
# and reuses the check steps only for an identical build context.
|
|
||||||
ARG CHECK_EPOCH
|
|
||||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
|
|
||||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make lint
|
|
||||||
|
|
||||||
# Build stage
|
|
||||||
# golang:1.25.4-alpine, 2026-02-25
|
# golang:1.25.4-alpine, 2026-02-25
|
||||||
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
FROM golang:1.25.4-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS builder
|
||||||
|
|
||||||
# Depend on lint stage passing
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
COPY --from=test /src/go.sum /dev/null
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# Build dependencies and Go modules, as in the lint stage
|
# Build dependencies and Go modules, as in the test phase
|
||||||
COPY script/ ./script/
|
COPY script/ ./script/
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN script/bootstrap
|
RUN script/bootstrap --cgo
|
||||||
|
|
||||||
# Copy source code
|
# Copy source code
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Run tests; a new CHECK_EPOCH reruns them, as in the lint stage.
|
|
||||||
ARG CHECK_EPOCH
|
|
||||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
|
||||||
|
|
||||||
# VERSION is declared here, not earlier: a new value reruns only the
|
# VERSION is declared here, not earlier: a new value reruns only the
|
||||||
# build, not script/bootstrap or the tests. Given none, the version is
|
# build, not script/bootstrap. Given none, the version is
|
||||||
# `git describe --tags --always` of the .git in the build context (git
|
# `git describe --tags --always` of the .git in the build context (git
|
||||||
# comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH
|
# comes from script/bootstrap): the tag on a tagged commit, tag-N-gHASH
|
||||||
# after one, the short commit when no tag is reachable. A context that
|
# after one, the short commit when no tag is reachable. A context that
|
||||||
@@ -68,7 +75,8 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
|||||||
-ldflags "-s -w -X main.Version=${version}" \
|
-ldflags "-s -w -X main.Version=${version}" \
|
||||||
-o /pixad ./cmd/pixad
|
-o /pixad ./cmd/pixad
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage, and the last one: a plain `docker build .` builds this
|
||||||
|
# stage and what it depends on, and nothing else.
|
||||||
# alpine:3.21, 2026-02-25
|
# alpine:3.21, 2026-02-25
|
||||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||||
|
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
# Dockerfile.lint: the container script/lint builds to run golangci-lint,
|
|
||||||
# which is never installed on the host. Pinned to the same image as the
|
|
||||||
# Dockerfile lint stage: change both pins together, or the two run
|
|
||||||
# different linter versions.
|
|
||||||
#
|
|
||||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
|
||||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# pixa is CGO/libvips: the type-aware linters compile every package, so
|
|
||||||
# this image needs the same C libraries the build does. script/bootstrap
|
|
||||||
# installs them and downloads the Go modules. Only script/, go.mod and
|
|
||||||
# go.sum are copied first; they settle this layer's result, so it may
|
|
||||||
# safely be reused between runs.
|
|
||||||
COPY script/ ./script/
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN script/bootstrap
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# Tells script/lint it is inside a container, so it runs the linter.
|
|
||||||
ENV container=docker
|
|
||||||
|
|
||||||
# script/lint passes a different CACHEBUST on every run, and BuildKit
|
|
||||||
# keys every RUN after this ARG on its value, so the lint step always
|
|
||||||
# runs instead of returning a cached success that linted nothing.
|
|
||||||
#
|
|
||||||
# Go's and golangci-lint's caches (/root/.cache, hundreds of MB) go on a
|
|
||||||
# tmpfs that is discarded after the step. Written into the layer, they
|
|
||||||
# would pile up as build cache on every run, since no later run, with
|
|
||||||
# its new CACHEBUST, can reuse that layer.
|
|
||||||
ARG CACHEBUST
|
|
||||||
RUN --mount=type=tmpfs,target=/root/.cache script/lint
|
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks
|
.PHONY: bootstrap setup check lint test fmt fmt-check build clean docker docker-smoke docker-versioned docker-test devserver devserver-stop hooks loadtest
|
||||||
|
|
||||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||||
LDFLAGS := -X main.Version=$(VERSION)
|
LDFLAGS := -X main.Version=$(VERSION)
|
||||||
|
|
||||||
# Use nix-shell to provide CGO dependencies unless they are already available
|
# Use nix-shell to provide CGO dependencies unless they are already available
|
||||||
# (e.g. inside a Docker build or an existing nix-shell).
|
# (e.g. inside an existing nix-shell).
|
||||||
HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null)
|
HAS_PKGCONFIG := $(shell command -v pkg-config 2>/dev/null)
|
||||||
ifdef HAS_PKGCONFIG
|
ifdef HAS_PKGCONFIG
|
||||||
NIX_RUN_PREFIX =
|
NIX_RUN_PREFIX =
|
||||||
@@ -32,11 +32,11 @@ fmt-check:
|
|||||||
fmt:
|
fmt:
|
||||||
@script/fmt
|
@script/fmt
|
||||||
|
|
||||||
# Run linter
|
# Run linter (the lint phase of the Dockerfile)
|
||||||
lint:
|
lint:
|
||||||
@script/lint
|
@script/lint
|
||||||
|
|
||||||
# Run tests (30-second timeout)
|
# Run tests (the test phase of the Dockerfile)
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
@@ -59,20 +59,25 @@ docker:
|
|||||||
docker-smoke:
|
docker-smoke:
|
||||||
@script/docker-smoke
|
@script/docker-smoke
|
||||||
|
|
||||||
# Build Docker image tagged pixad:$(VERSION) and pixad:latest
|
# Measure throughput, latency and peak memory with the default duration and
|
||||||
docker-versioned:
|
# number of clients (needs Docker and Go; a benchmark, not part of check)
|
||||||
docker build --build-arg VERSION=$(VERSION) -t pixad:$(VERSION) -t pixad:latest .
|
loadtest:
|
||||||
|
@script/loadtest
|
||||||
|
|
||||||
# Run tests in Docker (needed for CGO/libvips)
|
# Build Docker image as `make docker` does, and also tag it pixa:$(VERSION)
|
||||||
|
docker-versioned:
|
||||||
|
@script/docker
|
||||||
|
docker tag pixa pixa:$(VERSION)
|
||||||
|
|
||||||
|
# Run tests in Docker, as `make test` does
|
||||||
docker-test:
|
docker-test:
|
||||||
docker build --target builder --build-arg VERSION=$(VERSION) -t pixad-builder .
|
@script/test
|
||||||
docker run --rm pixad-builder sh -c "CGO_ENABLED=1 GOTOOLCHAIN=auto go test -v ./..."
|
|
||||||
|
|
||||||
# Run local dev server in Docker
|
# Run local dev server in Docker
|
||||||
devserver: docker-versioned devserver-stop
|
devserver: docker-versioned devserver-stop
|
||||||
docker run -d --name pixad-dev -p 8080:8080 \
|
docker run -d --name pixad-dev -p 8080:8080 \
|
||||||
-v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
|
-v $(CURDIR)/config.dev.yml:/etc/pixa/config.yml:ro \
|
||||||
pixad:latest
|
pixa:latest
|
||||||
@echo "pixad running at http://localhost:8080"
|
@echo "pixad running at http://localhost:8080"
|
||||||
|
|
||||||
# Stop dev server
|
# Stop dev server
|
||||||
|
|||||||
@@ -92,8 +92,11 @@ another part of pixa failed to stop. A request not finished by then is cut off.
|
|||||||
|
|
||||||
Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
|
Outside Docker, pixa needs libvips (the image has 8.15) and libheif to run, as
|
||||||
it uses libvips through CGO; building it also needs their development files,
|
it uses libvips through CGO; building it also needs their development files,
|
||||||
`pkg-config` and a C compiler. `script/bootstrap` installs all of these with
|
`pkg-config` and a C compiler. `script/bootstrap --cgo` installs all of these
|
||||||
nix, apt, brew or apk.
|
with nix, apt, brew or apk, as the `Dockerfile` does where it compiles pixa.
|
||||||
|
Plain `script/bootstrap`, which `script/setup` and `script/cibuild` run,
|
||||||
|
installs only git, make and Go: the checks compile pixa in Docker, so the host
|
||||||
|
needs none of the C libraries. Docker itself must already be installed.
|
||||||
|
|
||||||
## Running under upaas
|
## Running under upaas
|
||||||
|
|
||||||
@@ -146,10 +149,11 @@ name.
|
|||||||
|
|
||||||
Multiple source paths may reference the same content blob; the
|
Multiple source paths may reference the same content blob; the
|
||||||
database tracks references rather than using filesystem refcounting.
|
database tracks references rather than using filesystem refcounting.
|
||||||
Toward a target of 1-5k r/s, pixa keeps in memory the content types of
|
|
||||||
the 10,000 transformed images most recently cached or served, so a
|
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
|
||||||
cache hit on one of them reads only the image file from disk and not
|
Test). Toward it, pixa keeps in memory the content types of the 10,000
|
||||||
the metadata file stored beside it.
|
transformed images most recently cached or served, so a cache hit on one of them
|
||||||
|
reads only the image file from disk and not the metadata file stored beside it.
|
||||||
|
|
||||||
### Routes
|
### Routes
|
||||||
|
|
||||||
@@ -564,26 +568,90 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
|||||||
development workflow, and the Makefile targets are thin shims that call
|
development workflow, and the Makefile targets are thin shims that call
|
||||||
them. We provide:
|
them. We provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (idempotent)
|
- `script/bootstrap` — install git, make and Go and download the Go modules
|
||||||
|
(idempotent); with `--cgo`, also the C compiler and the libvips and libheif
|
||||||
|
libraries that compiling pixa needs
|
||||||
- `script/setup` — make a fresh clone ready for development
|
- `script/setup` — make a fresh clone ready for development
|
||||||
(bootstrap, then install-precommit)
|
(bootstrap, then install-precommit)
|
||||||
- `script/projectname` — output the project name ("pixa")
|
- `script/projectname` — output the project name ("pixa")
|
||||||
- `script/test` — run the test suite
|
- `script/test` — run the test suite: build the `test` phase of the
|
||||||
- `script/lint` — run golangci-lint, always in a container (builds
|
`Dockerfile`, tagged `pixa-test`
|
||||||
`Dockerfile.lint` when run outside one)
|
- `script/lint` — run golangci-lint: build the `lint` phase of the `Dockerfile`,
|
||||||
|
tagged `pixa-lint`; the linter never runs on the host
|
||||||
- `script/fmt` — format all code (writes)
|
- `script/fmt` — format all code (writes)
|
||||||
- `script/fmt-check` — check formatting (read-only)
|
- `script/fmt-check` — check formatting (read-only), on the host
|
||||||
- `script/check` — run test, lint, and fmt-check
|
- `script/check` — run test, lint, and fmt-check
|
||||||
- `script/docker` — build the Docker image tagged via `script/projectname`
|
- `script/docker` — build the Docker image tagged via `script/projectname`, with
|
||||||
|
the version from `git describe`; the image's build stage depends on the `lint`
|
||||||
|
and `test` phases, so this runs them too
|
||||||
- `script/docker-smoke` — build the image, start it, wait for it to be healthy
|
- `script/docker-smoke` — build the image, start it, wait for it to be healthy
|
||||||
- `script/cibuild` — CI entrypoint: `docker build .` with a new
|
- `script/loadtest` — measure pixad's throughput, latency and peak memory; a
|
||||||
`CHECK_EPOCH` on every run, so the Dockerfile's checks run instead of
|
benchmark, not part of `script/check` (see Load Test)
|
||||||
coming from the build cache, and a green run implies a green repo
|
- `script/cibuild` — CI entrypoint: run `script/bootstrap` (without `--cgo`),
|
||||||
|
then `script/check`, then build the image as `script/docker` does
|
||||||
- `script/precommit` — pre-commit checks (`go mod tidy` guard, then
|
- `script/precommit` — pre-commit checks (`go mod tidy` guard, then
|
||||||
`script/check`)
|
`script/check`)
|
||||||
- `script/install-precommit` — install the git pre-commit hook that
|
- `script/install-precommit` — install the git pre-commit hook that
|
||||||
runs `script/precommit`
|
runs `script/precommit`
|
||||||
|
|
||||||
|
Every `docker build` in these scripts passes `--no-cache`, so the lint and test
|
||||||
|
phases run on every build instead of coming from the build cache.
|
||||||
|
`script/check`, `script/cibuild`, `script/docker`, `script/lint`, `script/test`,
|
||||||
|
`script/setup` and `script/install-precommit` are the standard copies from
|
||||||
|
`sneak/prompts`, kept identical to them.
|
||||||
|
|
||||||
|
## Load Test
|
||||||
|
|
||||||
|
`script/loadtest` (or `make loadtest`) measures how fast pixad answers and how
|
||||||
|
much memory it uses. It is a benchmark, not a check: `script/check` does not run
|
||||||
|
it. It needs Docker and Go.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
script/loadtest # 10 seconds per scenario, 4 clients
|
||||||
|
script/loadtest 30s 32 # 30 seconds per scenario, 32 clients
|
||||||
|
```
|
||||||
|
|
||||||
|
It builds the image with `script/docker` and the load tool,
|
||||||
|
[vegeta](https://github.com/tsenart/vegeta), from a pinned commit. Each scenario
|
||||||
|
starts a new pixad container and a new origin container, `cmd/loadtest-origin`:
|
||||||
|
an upstream host that answers every path with the same generated 1600x1200 JPEG.
|
||||||
|
vegeta then sends requests from the given number of clients, each sending its
|
||||||
|
next request as soon as its last one is answered, all for an image resized to
|
||||||
|
400x300 WebP:
|
||||||
|
|
||||||
|
- `hit`: the same image every time, put in the cache first;
|
||||||
|
- `miss`: a new source image every time, so pixad fetches and converts each one;
|
||||||
|
- `herd`: each new source image once per client in a row, so that all clients
|
||||||
|
ask for it at the same time and share one fetch and one conversion (see
|
||||||
|
Routes).
|
||||||
|
|
||||||
|
pixad refuses upstream hosts with private or local addresses, so the containers
|
||||||
|
share a Docker network in `203.0.113.0/24`, a range set aside for documentation.
|
||||||
|
A second run on the same Docker host while one is going fails, as it cannot
|
||||||
|
create that network.
|
||||||
|
|
||||||
|
For each scenario the script prints vegeta's report and two lines of its own:
|
||||||
|
|
||||||
|
- `Requests [total, rate, throughput]`: the requests sent, how many were sent
|
||||||
|
per second, and how many were answered successfully per second; the last is
|
||||||
|
the number to compare with the target under Storage;
|
||||||
|
- `Latencies [min, mean, 50, 90, 95, 99, max]`: the time from sending a request
|
||||||
|
to the end of its answer; `50`, `95` and `99` are the 50th, 95th and 99th
|
||||||
|
percentiles;
|
||||||
|
- `Status Codes` and `Error Set`: anything other than `200` means the other
|
||||||
|
numbers are not for the scenario described, such as `503` when pixad was busy;
|
||||||
|
- `Bytes In`: `0`, as vegeta is told not to keep the images it receives;
|
||||||
|
- `pixad peak memory (VmHWM)`: the peak resident memory of pixad's process since
|
||||||
|
its container started, in kB; for `hit` it includes the request that put the
|
||||||
|
image in the cache;
|
||||||
|
- `requests to the origin`: the fetches pixad made: one for `hit`, one per
|
||||||
|
request for `miss`, and one per image for `herd`, that is the requests sent
|
||||||
|
divided by the number of clients.
|
||||||
|
|
||||||
|
The numbers depend on the machine and on whatever else runs on it. The first
|
||||||
|
measurement, made on a shared machine with few clients, is in `TODO.md`; it says
|
||||||
|
nothing about the target.
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
See [TODO.md](TODO.md) for the full prioritized task list.
|
See [TODO.md](TODO.md) for the full prioritized task list.
|
||||||
|
|||||||
@@ -31,6 +31,48 @@ P2: security: per-IP rate limiting on the image routes
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-05 lint and tests run as the `lint` and `test` phases of the
|
||||||
|
`Dockerfile`, built with `--no-cache` (closes #202): `script/check`,
|
||||||
|
`script/cibuild`, `script/docker`, `script/lint`, `script/test`,
|
||||||
|
`script/setup` and `script/install-precommit` are now the copies from
|
||||||
|
`sneak/prompts` `main`, unchanged. The `lint` phase runs golangci-lint from
|
||||||
|
the image `REPO_POLICIES.md` names, with `libvips-dev` from `apt-get`; the
|
||||||
|
`test` phase runs the tests with a 90-second timeout; the build stage depends
|
||||||
|
on both. `Dockerfile.lint` and the `CHECK_EPOCH` build argument are gone, the
|
||||||
|
formatting check runs on the host, and `make docker-versioned` and
|
||||||
|
`make docker-test` call the scripts. `script/bootstrap`, `script/fmt`,
|
||||||
|
`script/fmt-check`, `script/precommit` and `script/projectname` stay pixa's
|
||||||
|
own. `script/bootstrap` installs git, make and Go, refreshing apt's package
|
||||||
|
lists before its first apt install; with `--cgo`, which only the `test` phase
|
||||||
|
and the build stage pass, it also installs the C compiler and the libvips and
|
||||||
|
libheif libraries. The stage that compiles still takes the version from
|
||||||
|
`git describe` when no `VERSION` is given, per
|
||||||
|
https://git.eeqj.de/sneak/pixa/issues/166, so the copied scripts' comment that
|
||||||
|
`.dockerignore` leaves out `.git` does not hold for pixa.
|
||||||
|
- 2026-10-04 load test (closes #81): `script/loadtest [duration [clients]]`
|
||||||
|
(`make loadtest`, defaults `10s` and `4`), a benchmark that `script/check`
|
||||||
|
does not run, measures three scenarios, each against a new pixad container and
|
||||||
|
a new upstream host, `cmd/loadtest-origin`: `hit` (one cached image), `miss`
|
||||||
|
(a new source image every request) and `herd` (each new source image asked for
|
||||||
|
by all clients at once). For each it prints vegeta's report (requests per
|
||||||
|
second, latency percentiles, status codes), pixad's peak resident memory and
|
||||||
|
the requests that reached the origin. `README.md` says how to run it and read
|
||||||
|
it, and keeps 1-5k r/s as a target not yet measured. First measurement, with
|
||||||
|
the defaults on a shared 48-CPU machine with other work running: a baseline
|
||||||
|
for later changes, not a test of the target. `hit` 1413 r/s, p50 0.7 ms, p95
|
||||||
|
8.7 ms, p99 44 ms, peak 53 MiB (4 clients that each wait for their answer, so
|
||||||
|
not pixad's limit); `miss` 70 r/s, p50 52 ms, p95 91 ms, p99 122 ms, peak 100
|
||||||
|
MiB, one fetch per request; `herd` 74 r/s, p50 52 ms, p95 69 ms, p99 111 ms,
|
||||||
|
peak 60 MiB, 188 fetches for 749 requests.
|
||||||
|
- 2026-10-04 the CI checkout fetches the tags (closes #208): the checkout step
|
||||||
|
in `.gitea/workflows/check.yml` sets `fetch-depth: 0`, as `REPO_POLICIES.md`
|
||||||
|
asks of a repo that takes its version from the tags, so a CI build of a tagged
|
||||||
|
commit stamps the tag from `git describe` instead of a bare commit.
|
||||||
|
- 2026-10-04 `config.yml` stays out of git and the Docker build context (closes
|
||||||
|
#212): `.gitignore` now ignores `config.yml`, the config file Getting Started
|
||||||
|
creates with the signing key, and `.dockerignore` leaves it out in every
|
||||||
|
directory and in any letter case, as it already did `config.yaml` and
|
||||||
|
`config.dev.yml`.
|
||||||
- 2026-10-04 local config files stay out of the Docker build context (closes
|
- 2026-10-04 local config files stay out of the Docker build context (closes
|
||||||
#211): `.dockerignore` now leaves out `config.yaml` and `config.dev.yml` in
|
#211): `.dockerignore` now leaves out `config.yaml` and `config.dev.yml` in
|
||||||
every directory and in any letter case, the local config files `.gitignore`
|
every directory and in any letter case, the local config files `.gitignore`
|
||||||
@@ -643,4 +685,5 @@ P2: security: per-IP rate limiting on the image routes
|
|||||||
- optional Sentry error reporting
|
- optional Sentry error reporting
|
||||||
- comprehensive request logging
|
- comprehensive request logging
|
||||||
- Prometheus performance metrics
|
- Prometheus performance metrics
|
||||||
- load tests to verify the 1k to 5k req/s target
|
- measure the 1k to 5k req/s target with `script/loadtest` on a machine not
|
||||||
|
shared with other work
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
// Command loadtest-origin is the upstream host script/loadtest points pixad
|
||||||
|
// at; internal/loadtestorigin says what it does.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "sneak.berlin/go/pixa/internal/loadtestorigin"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
loadtestorigin.Run()
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
// Package loadtestorigin is the upstream host script/loadtest points pixad
|
||||||
|
// at, run by cmd/loadtest-origin. It answers every request, whatever its path,
|
||||||
|
// with the same generated JPEG, so each new path is a new source image for
|
||||||
|
// pixad to fetch, and it logs one line per request, so its log counts pixad's
|
||||||
|
// fetches.
|
||||||
|
package loadtestorigin
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"image"
|
||||||
|
"image/color"
|
||||||
|
"image/jpeg"
|
||||||
|
"log/slog"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
listenAddress = ":80"
|
||||||
|
readHeaderTimeout = 10 * time.Second
|
||||||
|
imageWidth = 1600
|
||||||
|
imageHeight = 1200
|
||||||
|
jpegQuality = 85
|
||||||
|
)
|
||||||
|
|
||||||
|
// Run makes the image and serves it on port 80 until the server fails, then
|
||||||
|
// exits the process with status 1.
|
||||||
|
func Run() {
|
||||||
|
photo, err := makeJPEG()
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("cannot make the image", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: listenAddress,
|
||||||
|
Handler: newHandler(photo),
|
||||||
|
ReadHeaderTimeout: readHeaderTimeout,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = server.ListenAndServe()
|
||||||
|
slog.Error("server stopped", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newHandler answers every request with photo and logs the request's path.
|
||||||
|
func newHandler(photo []byte) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slog.Info("request", "path", r.URL.Path)
|
||||||
|
w.Header().Set("Content-Type", "image/jpeg")
|
||||||
|
_, _ = w.Write(photo)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeJPEG draws colour gradients crossed with a fine pattern, so the image
|
||||||
|
// has detail to decode and does not compress to almost nothing.
|
||||||
|
func makeJPEG() ([]byte, error) {
|
||||||
|
img := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight))
|
||||||
|
|
||||||
|
// red and green count up from 0 to 255 and wrap around, along each row
|
||||||
|
// and down the image.
|
||||||
|
var green uint8
|
||||||
|
|
||||||
|
for y := range imageHeight {
|
||||||
|
var red uint8
|
||||||
|
|
||||||
|
for x := range imageWidth {
|
||||||
|
img.SetRGBA(x, y, color.RGBA{
|
||||||
|
R: red, G: green, B: red ^ green, A: math.MaxUint8,
|
||||||
|
})
|
||||||
|
red++
|
||||||
|
}
|
||||||
|
|
||||||
|
green++
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return buf.Bytes(), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
package loadtestorigin
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"image/jpeg"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEveryPathServesTheSameJPEG checks that the origin answers any path with
|
||||||
|
// 200 and the same JPEG, so every new path script/loadtest asks pixad for is
|
||||||
|
// a valid source image.
|
||||||
|
func TestEveryPathServesTheSameJPEG(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
photo, err := makeJPEG()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("makeJPEG: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
size, err := jpeg.DecodeConfig(bytes.NewReader(photo))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the image does not decode as a JPEG: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if size.Width != imageWidth || size.Height != imageHeight {
|
||||||
|
t.Errorf("the image is %dx%d, want %dx%d",
|
||||||
|
size.Width, size.Height, imageWidth, imageHeight)
|
||||||
|
}
|
||||||
|
|
||||||
|
handler := newHandler(photo)
|
||||||
|
|
||||||
|
for _, path := range []string{"/", "/miss/1.jpg", "/herd/2.jpg"} {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, path, nil))
|
||||||
|
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Errorf("%s: status = %d, want %d", path, rec.Code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ct := rec.Header().Get("Content-Type"); ct != "image/jpeg" {
|
||||||
|
t.Errorf("%s: Content-Type = %q, want image/jpeg", path, ct)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(rec.Body.Bytes(), photo) {
|
||||||
|
t.Errorf("%s: the body is not the image", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+26
-7
@@ -4,11 +4,15 @@
|
|||||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
# make, or go). The linter is never installed on the host: golangci-lint
|
# make, or go). The linter is never installed on the host: golangci-lint
|
||||||
# runs only inside a container, Dockerfile.lint or the Dockerfile lint
|
# runs only in the lint phase of the Dockerfile (see script/lint).
|
||||||
# stage (see script/lint). A C compiler and the CGO image libraries
|
#
|
||||||
# (pkg-config, vips, libheif) are installed for the govips bindings.
|
# script/bootstrap git, make and Go, all the host needs: the
|
||||||
# Both Dockerfiles run this script too, so their build dependencies are
|
# checks compile pixa in Docker
|
||||||
# the ones listed here.
|
# script/bootstrap --cgo also a C compiler and the CGO image
|
||||||
|
# libraries (pkg-config, vips, libheif) for
|
||||||
|
# the govips bindings, to compile pixa; the
|
||||||
|
# Dockerfile's test phase and build stage
|
||||||
|
# run this
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -35,6 +39,10 @@ detect_pkgmgr() {
|
|||||||
if [ "$(id -u)" != "0" ]; then
|
if [ "$(id -u)" != "0" ]; then
|
||||||
SUDO="sudo"
|
SUDO="sudo"
|
||||||
fi
|
fi
|
||||||
|
# This runs before the first install only. A fresh image, such
|
||||||
|
# as a CI runner's, has no package lists, and apt-get install
|
||||||
|
# finds no package without them.
|
||||||
|
$SUDO apt-get update
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,7 +79,16 @@ ensure_cgo_deps() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "usage: script/bootstrap [--cgo]" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
|
case "$*" in
|
||||||
|
"" | --cgo) ;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
# Base tooling
|
# Base tooling
|
||||||
@@ -81,8 +98,10 @@ main() {
|
|||||||
# Go toolchain
|
# Go toolchain
|
||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
|
|
||||||
# CGO image libraries
|
# CGO image libraries, only where pixa is compiled
|
||||||
ensure_cgo_deps
|
if [ "$*" = "--cgo" ]; then
|
||||||
|
ensure_cgo_deps
|
||||||
|
fi
|
||||||
|
|
||||||
go mod download
|
go mod download
|
||||||
|
|
||||||
|
|||||||
+3
-2
@@ -1,7 +1,8 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||||
# extension to scripts-to-rule-them-all. Must not modify any files.
|
# extension to scripts-to-rule-them-all. test and lint are Docker
|
||||||
# Generic: usually needs no adaptation.
|
# phases; fmt-check is native, because a formatter writes the working
|
||||||
|
# tree. Must not modify any files.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
+20
-9
@@ -1,18 +1,29 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/cibuild: run the CI build. The Dockerfile runs the checks
|
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||||
# (make fmt-check, lint, test) as build steps. This script passes a new
|
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||||
# CHECK_EPOCH on every run, so Docker runs those steps instead of
|
# the formatter on the host, which a pristine checkout cannot do.
|
||||||
# reusing cached results: a successful run means the checks ran and
|
# --no-cache for the same reason as script/docker: the gate phases the
|
||||||
# passed on this tree. Generic: needs no adaptation. The Gitea workflow
|
# final stage depends on are RUN steps, and a cached one is a check that
|
||||||
# runs this on push.
|
# did not run.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
epoch="$(date +%s)$$"
|
"$SCRIPT_DIR/bootstrap"
|
||||||
docker build --build-arg CHECK_EPOCH="$epoch" .
|
"$SCRIPT_DIR/check"
|
||||||
|
# Own line: a failing command substitution inside an argument does
|
||||||
|
# not trip `set -e`, so the inline form degrades silently to an
|
||||||
|
# empty constant. VERSION is computed here because .dockerignore
|
||||||
|
# excludes .git, so `git describe` in a build stage yields an empty
|
||||||
|
# version without failing.
|
||||||
|
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||||
|
[ -n "$version" ] || version="unknown"
|
||||||
|
docker build --no-cache \
|
||||||
|
--build-arg VERSION="$version" \
|
||||||
|
-t "$("$SCRIPT_DIR/projectname")" .
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+12
-6
@@ -1,9 +1,8 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/docker: build the Docker image tagged with the project name.
|
# script/docker: build the Docker image tagged with the project name.
|
||||||
# Identical in all repos; the tag comes from script/projectname. Like
|
# Identical in all repos; the tag comes from script/projectname.
|
||||||
# script/cibuild, it passes a new CHECK_EPOCH, so the build runs the
|
# --no-cache because the gate phases the final stage depends on are RUN
|
||||||
# checks instead of reusing cached results. Generic: needs no
|
# steps, and a cached one is a check that did not run.
|
||||||
# adaptation.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
@@ -11,8 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
epoch="$(date +%s)$$"
|
# Own line: a failing command substitution inside an argument does
|
||||||
docker build --build-arg CHECK_EPOCH="$epoch" \
|
# not trip `set -e`, so the inline form degrades silently to an
|
||||||
|
# empty constant. VERSION is computed here because .dockerignore
|
||||||
|
# excludes .git, so `git describe` in a build stage yields an empty
|
||||||
|
# version without failing.
|
||||||
|
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||||
|
[ -n "$version" ] || version="unknown"
|
||||||
|
docker build --no-cache \
|
||||||
|
--build-arg VERSION="$version" \
|
||||||
-t "$("$SCRIPT_DIR/projectname")" .
|
-t "$("$SCRIPT_DIR/projectname")" .
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/install-precommit: install the git pre-commit hook that runs
|
# script/install-precommit: install the git pre-commit hook that runs
|
||||||
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
||||||
# Generic: needs no adaptation.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
hook=".git/hooks/pre-commit"
|
||||||
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
|
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
|
||||||
chmod +x .git/hooks/pre-commit
|
chmod +x .git/hooks/pre-commit
|
||||||
echo "pre-commit hook installed: runs script/precommit"
|
echo "pre-commit hook installed: runs script/precommit"
|
||||||
|
|||||||
+13
-27
@@ -1,37 +1,23 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/lint: run golangci-lint over the whole tree. This is the only
|
# script/lint: run the linter. Linting is a phase of the Dockerfile and
|
||||||
# way the linter is run, everywhere; it is never installed on the host.
|
# this builds that phase alone; the linter is never installed or run on
|
||||||
|
# a developer host, where a shared result cache and a host-global lock
|
||||||
|
# make its answer untrustworthy.
|
||||||
#
|
#
|
||||||
# Inside a container it runs the linter. Anywhere else it builds
|
# The phase is not the last stage in the file, so it is built only when
|
||||||
# Dockerfile.lint, whose last step runs this script again inside that
|
# --target names it. --no-cache because a cached lint layer is a lint
|
||||||
# container.
|
# that did not run. The tag makes each build replace the previous image
|
||||||
#
|
# instead of leaving a dangling one behind.
|
||||||
# Dockerfile.lint and the Dockerfile lint stage set container=docker
|
|
||||||
# (the systemd convention for marking a container) to say where we are.
|
|
||||||
# /.dockerenv cannot: it is missing inside build steps, and present on
|
|
||||||
# hosts that are themselves containers.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
if [ "${container:-}" = docker ]; then
|
docker build --no-cache \
|
||||||
# `golangci-lint config verify` is not run: it fetches its JSON
|
--target lint \
|
||||||
# schema over an unpinned live HTTPS call, which REPO_POLICIES.md
|
-t "$("$SCRIPT_DIR/projectname")-lint" .
|
||||||
# forbids.
|
|
||||||
echo "Running linter..."
|
|
||||||
golangci-lint run --config .golangci.yml ./...
|
|
||||||
else
|
|
||||||
# A new CACHEBUST on every run means the lint step is never
|
|
||||||
# served from cache (see Dockerfile.lint). The cacheonly output
|
|
||||||
# leaves no image behind.
|
|
||||||
docker build \
|
|
||||||
--progress=plain \
|
|
||||||
--build-arg CACHEBUST="$(date +%s)-$$" \
|
|
||||||
--output=type=cacheonly \
|
|
||||||
-f Dockerfile.lint .
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Executable
+177
@@ -0,0 +1,177 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/loadtest: measure pixad's throughput, latency and peak memory.
|
||||||
|
#
|
||||||
|
# script/loadtest [duration [clients]] (defaults: 10s and 4)
|
||||||
|
#
|
||||||
|
# A benchmark, not a check: script/check does not run it. It needs Docker
|
||||||
|
# and Go. It builds the image with script/docker and builds vegeta, the
|
||||||
|
# load tool, from a pinned commit. Each scenario then gets a new pixad
|
||||||
|
# container and a new origin container (cmd/loadtest-origin, which answers
|
||||||
|
# every path with the same JPEG), and vegeta sends requests for <duration>
|
||||||
|
# from <clients> clients at once, each asking for an image resized to
|
||||||
|
# 400x300 WebP:
|
||||||
|
#
|
||||||
|
# hit the same image every time, put in the cache first
|
||||||
|
# miss a new source image every time
|
||||||
|
# herd each new source image once per client in a row, so that all
|
||||||
|
# clients ask for it at the same time
|
||||||
|
#
|
||||||
|
# For each, it prints vegeta's report, pixad's peak resident memory and
|
||||||
|
# how many requests reached the origin. README.md says how to read them.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
# vegeta v12.13.0, 2026-10-04
|
||||||
|
VEGETA_COMMIT=4b240c3089fa4aa10816542d64a74294d974211f
|
||||||
|
|
||||||
|
# pixad refuses upstream hosts with private or local addresses, so the
|
||||||
|
# containers share a network in 203.0.113.0/24, a range set aside for
|
||||||
|
# documentation (RFC 5737) that pixad does not refuse and that is never
|
||||||
|
# routed on the internet.
|
||||||
|
SUBNET=203.0.113.0/24
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "usage: script/loadtest [duration [clients]]" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
duration="${1:-10s}"
|
||||||
|
clients="${2:-4}"
|
||||||
|
# The duration is a whole number, not zero (vegeta takes 0 to mean no
|
||||||
|
# end), followed by ms, s, m or h.
|
||||||
|
case "$duration" in
|
||||||
|
*ms) number="${duration%ms}" ;;
|
||||||
|
*s | *m | *h) number="${duration%?}" ;;
|
||||||
|
*) usage ;;
|
||||||
|
esac
|
||||||
|
case "$number" in
|
||||||
|
"" | *[!0-9]*) usage ;;
|
||||||
|
esac
|
||||||
|
[ "$number" -gt 0 ] || usage
|
||||||
|
# The number of clients is a whole number that does not start with 0,
|
||||||
|
# which also refuses zero: vegeta reads a leading 0 as octal.
|
||||||
|
case "$clients" in
|
||||||
|
*[!0-9]* | 0*) usage ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
run="pixa-loadtest-$$"
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap cleanup EXIT
|
||||||
|
trap 'exit 1' HUP INT TERM
|
||||||
|
|
||||||
|
"$SCRIPT_DIR/docker"
|
||||||
|
# The image's ID, so a build elsewhere that moves the tag does not
|
||||||
|
# change what a later scenario starts.
|
||||||
|
image="$(docker image inspect --format '{{.Id}}' \
|
||||||
|
"$("$SCRIPT_DIR/projectname")")"
|
||||||
|
|
||||||
|
GOBIN="$tmp" go install "github.com/tsenart/vegeta/v12@$VEGETA_COMMIT"
|
||||||
|
# The origin runs in a container, so it is built for the Docker host.
|
||||||
|
CGO_ENABLED=0 GOOS=linux \
|
||||||
|
GOARCH="$(docker version --format '{{.Server.Arch}}')" \
|
||||||
|
go build -o "$tmp/loadtest-origin" ./cmd/loadtest-origin
|
||||||
|
|
||||||
|
docker network create --subnet "$SUBNET" "$run" >/dev/null
|
||||||
|
|
||||||
|
start_containers
|
||||||
|
# Put the image the hit scenario asks for in the cache.
|
||||||
|
docker exec "$run-pixad" wget -q -O /dev/null \
|
||||||
|
"http://localhost:8080/v1/image/origin/hit.jpg/400x300.webp"
|
||||||
|
attack hit hit_targets
|
||||||
|
stop_containers
|
||||||
|
|
||||||
|
start_containers
|
||||||
|
attack miss miss_targets
|
||||||
|
stop_containers
|
||||||
|
|
||||||
|
start_containers
|
||||||
|
attack herd herd_targets
|
||||||
|
stop_containers
|
||||||
|
}
|
||||||
|
|
||||||
|
# start_containers starts a new origin and a new pixad, and waits up to 30
|
||||||
|
# seconds for pixad's health check to pass.
|
||||||
|
start_containers() {
|
||||||
|
docker run -d --name "$run-origin" \
|
||||||
|
--network "$run" --network-alias origin \
|
||||||
|
-v "$tmp/loadtest-origin:/usr/local/bin/loadtest-origin:ro" \
|
||||||
|
--entrypoint /usr/local/bin/loadtest-origin "$image" >/dev/null
|
||||||
|
docker run -d --name "$run-pixad" \
|
||||||
|
--network "$run" -p 127.0.0.1::8080 --health-interval=1s \
|
||||||
|
-e PIXA_SIGNING_KEY="$(head -c 32 /dev/urandom | base64)" \
|
||||||
|
-e PIXA_ALLOWLIST_HOSTS=origin -e PIXA_ALLOW_HTTP=true \
|
||||||
|
"$image" >/dev/null
|
||||||
|
|
||||||
|
waited=0
|
||||||
|
until [ "$(docker inspect --format '{{.State.Health.Status}}' \
|
||||||
|
"$run-pixad")" = healthy ]; do
|
||||||
|
if [ "$waited" -ge 30 ]; then
|
||||||
|
echo "loadtest: pixad not healthy after 30 seconds; its log:" >&2
|
||||||
|
docker logs "$run-pixad" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
waited=$((waited + 1))
|
||||||
|
done
|
||||||
|
pixa="http://$(docker port "$run-pixad" 8080/tcp)"
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_containers() {
|
||||||
|
docker rm -f "$run-pixad" "$run-origin" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# attack <scenario> <targets>: send the requests <targets> prints and
|
||||||
|
# report on them.
|
||||||
|
attack() {
|
||||||
|
echo
|
||||||
|
echo "== $1: $clients clients for $duration"
|
||||||
|
"$2" | "$tmp/vegeta" attack -lazy -rate 0 -workers "$clients" \
|
||||||
|
-max-workers "$clients" -duration "$duration" -max-body 0 |
|
||||||
|
"$tmp/vegeta" report
|
||||||
|
# pixad is process 1 in its container: the entrypoint execs it.
|
||||||
|
echo "pixad peak memory (VmHWM):" \
|
||||||
|
"$(docker exec "$run-pixad" awk '/^VmHWM:/ { print $2, $3 }' \
|
||||||
|
/proc/1/status)"
|
||||||
|
echo "requests to the origin:" \
|
||||||
|
"$(docker logs "$run-origin" 2>&1 | grep -c ' request ')"
|
||||||
|
}
|
||||||
|
|
||||||
|
# The targets functions print vegeta targets until vegeta stops reading.
|
||||||
|
|
||||||
|
hit_targets() {
|
||||||
|
while :; do
|
||||||
|
echo "GET $pixa/v1/image/origin/hit.jpg/400x300.webp"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
miss_targets() {
|
||||||
|
i=0
|
||||||
|
while :; do
|
||||||
|
i=$((i + 1))
|
||||||
|
echo "GET $pixa/v1/image/origin/miss/$i.jpg/400x300.webp"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
herd_targets() {
|
||||||
|
i=0
|
||||||
|
while :; do
|
||||||
|
i=$((i + 1))
|
||||||
|
n=0
|
||||||
|
while [ "$n" -lt "$clients" ]; do
|
||||||
|
n=$((n + 1))
|
||||||
|
echo "GET $pixa/v1/image/origin/herd/$i.jpg/400x300.webp"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
docker rm -f "$run-pixad" "$run-origin" >/dev/null 2>&1 || :
|
||||||
|
docker network rm "$run" >/dev/null 2>&1 || :
|
||||||
|
rm -rf "$tmp"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
+1
-2
@@ -1,7 +1,6 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/setup: set up the repo for development after a fresh clone:
|
# script/setup: set up the repo for development after a fresh clone:
|
||||||
# installs dependencies (script/bootstrap) and the git pre-commit hook.
|
# installs dependencies and the git pre-commit hook.
|
||||||
# Add any repo-specific initialization (db init, .env template) here.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|||||||
+10
-18
@@ -1,27 +1,19 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/test: run the test suite. CGO dependencies (pkg-config, vips,
|
# script/test: run the test suite. Testing is a phase of the Dockerfile
|
||||||
# libheif) come from nix-shell when not already available (e.g. inside
|
# and this builds that phase alone, on the same terms as script/lint:
|
||||||
# a Docker build or an existing nix-shell).
|
# --target because a phase that is not the last stage is built only when
|
||||||
|
# named, --no-cache because a cached test layer is a test that did not
|
||||||
|
# run, and a tag so each build replaces the previous image.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
run_with_cgo_deps() {
|
|
||||||
if command -v pkg-config >/dev/null 2>&1; then
|
|
||||||
sh -c "$1"
|
|
||||||
else
|
|
||||||
nix-shell -p pkg-config vips libheif git --run "$1"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
echo "Running tests..."
|
docker build --no-cache \
|
||||||
# Run without -v first for clean output on success; on failure rerun
|
--target test \
|
||||||
# with -v for full diagnostics, then exit non-zero (REPO_POLICIES.md
|
-t "$("$SCRIPT_DIR/projectname")-test" .
|
||||||
# conditional-verbose-rerun pattern). The first run already proved the
|
|
||||||
# tests broken, so the build fails even if the rerun happens to pass.
|
|
||||||
run_with_cgo_deps "CGO_ENABLED=1 go test -timeout 30s -race -cover ./... || { echo '--- Rerunning with -v for details ---'; CGO_ENABLED=1 go test -timeout 30s -race -v ./...; exit 1; }"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user