1 Commits
Author SHA1 Message Date
clawbot 2bf52ba7ff fix(backend): rate-limit and cap report ingest, drop wildcard CORS (closes #20)
check / check (push) Successful in 58s
POST /api/v1/reports stays unauthenticated but is bounded. Each client
address, as the trusted-proxy logic resolves it, may send
REPORTS_PER_MINUTE reports a minute (default 60, all at once if it
likes), using golang.org/x/time/rate; past that it gets 429 with
Retry-After. Buckets that have refilled are dropped once a minute, so
idle addresses do not pile up. reportbuf refuses a report that would
take the report files past DATA_DIR_MAX_BYTES (default 1 GiB) with
ErrFull, answered with 507; the count starts from the files already in
DATA_DIR, and reports not yet written count at their uncompressed size.
CORS adds nothing unless CORS_ALLOWED_ORIGINS lists origins. A limit
that is not a positive number stops the server from starting.

Model: opus-5-5
2026-09-29 00:18:52 +00:00
26 changed files with 246 additions and 328 deletions
+1 -1
View File
@@ -6,5 +6,5 @@ jobs:
steps:
# actions/checkout v4.2.2, 2026-02-22
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
# script/cibuild builds the image, whose stages run every check.
# script/cibuild builds both images.
- run: script/cibuild
+5 -68
View File
@@ -1,51 +1,5 @@
# The one image netwatch ships: nginx serves the built frontend and
# passes /api/ and /.well-known/healthcheck to netwatch-server, the Go
# backend, which runs in the same container on loopback only.
# bin/entrypoint.sh starts and watches both.
# Lint stage — fast feedback on formatting and lint issues. The
# golangci/golangci-lint image ships Go, gofmt, make and the linter, so
# nothing is installed here. The root make lint builds this stage alone.
# golangci/golangci-lint:v2.12.2 (2026-08-10)
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
WORKDIR /src
COPY backend/go.mod backend/go.sum ./
RUN go mod download
COPY backend/ .
RUN make fmt-check
RUN make lint
# Backend build stage
# golang:1.25-alpine (2026-02-27)
FROM golang:1.25-alpine@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
RUN apk add --no-cache make
WORKDIR /src
# Force BuildKit to run the lint stage before proceeding. BuildKit runs
# stages in parallel by default; without this no-op copy a lint failure
# would not gate compilation.
COPY --from=lint /src/go.sum /dev/null
COPY backend/go.mod backend/go.sum ./
RUN go mod download
COPY backend/ .
RUN make test
# make build is a shim around backend/script/build, the one definition
# of the build command:
# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=... -X main.Buildarch=..."
# That script reads VERSION from the environment, so it is handed over
# there rather than as a make variable.
ARG VERSION=dev
RUN VERSION="${VERSION}" make build
# Frontend stage
# node:22-alpine as of 2026-02-22
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS build
WORKDIR /app
COPY package.json yarn.lock ./
RUN yarn install --frozen-lockfile
@@ -54,33 +8,16 @@ COPY . .
# make frontend-check is the frontend half of make check (test + lint +
# fmt-check); its test step is the production yarn build, so this both
# produces dist/ and gates the image on lint/fmt-check/test regressions.
# This node stage has neither Go nor Docker; the lint and builder stages
# above gate the backend half.
# This node stage has neither Go nor Docker for the other half, which
# Dockerfile.backend gates; script/cibuild builds both images.
RUN make frontend-check
# Runtime stage
# nginx:stable-alpine as of 2026-02-22
FROM nginx@sha256:15e96e59aa3b0aada3a121296e3bce117721f42d88f5f64217ef4b18f458c6ab
# netwatch-server runs as this user, which owns the report directory.
# nginx keeps the image's own arrangement: its main process runs as
# root, its worker processes as the nginx user.
RUN addgroup -g 1000 -S netwatch && \
adduser -u 1000 -S netwatch -G netwatch
RUN rm /etc/nginx/conf.d/default.conf
COPY nginx.conf /etc/nginx/conf.d/netwatch.conf
COPY --from=frontend /app/dist /usr/share/nginx/html
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
ENV DATA_DIR=/data/reports
RUN mkdir -p /data/reports && chown -R netwatch:netwatch /data
VOLUME /data
COPY --from=build /app/dist /usr/share/nginx/html
EXPOSE 8080
# The nginx image stops its container with SIGQUIT; the entrypoint
# acts on TERM and INT.
STOPSIGNAL SIGTERM
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]
+49
View File
@@ -0,0 +1,49 @@
# Lint stage — fast feedback on formatting and lint issues. The
# golangci/golangci-lint image ships Go, gofmt, make and the linter, so
# nothing is installed here.
# golangci/golangci-lint:v2.12.2 (2026-08-10)
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
WORKDIR /src
COPY backend/go.mod backend/go.sum ./
RUN go mod download
COPY backend/ .
RUN make fmt-check
RUN make lint
# Build stage
# golang:1.25-alpine (2026-02-27)
FROM golang:1.25-alpine@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
RUN apk add --no-cache make
WORKDIR /src
# Force BuildKit to run the lint stage before proceeding. BuildKit runs
# stages in parallel by default; without this no-op copy a lint failure
# would not gate compilation.
COPY --from=lint /src/go.sum /dev/null
COPY backend/go.mod backend/go.sum ./
RUN go mod download
COPY backend/ .
RUN make test
# make build is a shim around backend/script/build, the one definition
# of the build command:
# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=... -X main.Buildarch=..."
# That script reads VERSION from the environment, so it is handed over
# there rather than as a make variable.
ARG VERSION=dev
RUN VERSION="${VERSION}" make build
# Runtime stage
# alpine:3.23 (2026-02-27)
FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
RUN apk add --no-cache ca-certificates
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
EXPOSE 8080
ENTRYPOINT ["netwatch-server"]
+6 -14
View File
@@ -44,11 +44,11 @@ halves, so the root `make check` fails if either one is broken. We provide:
linter in Docker
- `script/setup` — make a fresh clone ready for development: bootstrap plus the
git pre-commit hook
- `script/projectname` — print the project name (used for the Docker image tag)
- `script/projectname` — print the project name (used for the Docker image tags)
- `script/test` — run `script/frontend-test`, then the backend's Go tests, both
within one 30-second timeout
- `script/lint` — run `script/frontend-lint`, then golangci-lint in Docker, by
building the lint stage of `Dockerfile` without the cache
building the lint stage of `Dockerfile.backend` without the cache
- `script/fmt` — format all files (writes): prettier, then gofmt over `backend/`
- `script/fmt-check` — check formatting (read-only): prettier, then gofmt
- `script/check` — run test, lint, and fmt-check
@@ -63,9 +63,9 @@ halves, so the root `make check` fails if either one is broken. We provide:
frontend in a containerised headless Chrome (see
[test/viewport/README.md](test/viewport/README.md)). Not part of
`script/check`: it needs Docker and takes minutes.
- `script/docker` — build the image from `Dockerfile` without the build cache,
tagged `netwatch` via `script/projectname`
- `script/cibuild` — CI entrypoint: builds the image
- `script/docker` — build both images, tagged via `script/projectname`:
`netwatch` from `Dockerfile` and `netwatch-server` from `Dockerfile.backend`
- `script/cibuild` — CI entrypoint: builds both images
- `script/precommit` — run by the git pre-commit hook; runs `script/check`
- `script/install-precommit` — install the git pre-commit hook
@@ -178,21 +178,13 @@ After running `yarn build`, deploy the contents of the `dist/` directory to any
static file host (S3, GCS, Cloudflare Pages, Vercel, Netlify, GitHub Pages) or
use the Docker image behind a reverse proxy.
The Docker image, built from `Dockerfile`, is the whole service in one
container: nginx serves the built frontend and passes `/api/` and
`/.well-known/healthcheck` to the Go backend, `netwatch-server`, which listens
only inside the container, on `127.0.0.1:8081`. The image:
The Docker image:
- Listens on port 8080 by default (override with `PORT` env var)
- Trusts `X-Forwarded-For` from RFC1918 reverse proxies (10/8, 172.16/12,
192.168/16)
- Sends access logs to stdout
- Caches static assets with immutable headers
- Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data`
volume. The backend runs as user `netwatch` (uid 1000), so a directory
bind-mounted at `/data` must be writable by uid 1000
- Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx
or the backend exits on its own, so the platform restarts it
## Browser Compatibility
+7 -15
View File
@@ -25,21 +25,13 @@ latest run passes.
- 2026-09-29: bounded the report endpoint (issue #20): `POST /api/v1/reports`
still needs no credentials, but each client address, as resolved through
`TRUSTED_PROXIES`, may send `REPORTS_PER_MINUTE` (default 60) reports a
minute, counted by `go-chi/httprate`, and past that gets 429 with
`Retry-After`; the report files in `DATA_DIR`, counted from start with those
already there, may total at most `DATA_DIR_MAX_BYTES` (default 1 GiB), past
which reports get 507; and the wildcard CORS is gone: no CORS headers unless
`CORS_ALLOWED_ORIGINS` lists origins. Deleting report files frees room only at
the next start; pruning is issue #54
- 2026-09-28: one container image (issue #52): the root `Dockerfile` builds the
only image, and `Dockerfile.backend` is gone. nginx serves the frontend on
port 8080 and proxies `/api/` and `/.well-known/healthcheck` to the backend,
which listens on `127.0.0.1:8081` in the same container; the new
`BIND_ADDRESS` setting sets its listen address. `bin/entrypoint.sh` starts
both, passes TERM and INT on to both, and exits non-zero when either exits on
its own. The backend runs as user `netwatch` and stores reports on the `/data`
volume. `script/docker` is the org model again
`TRUSTED_PROXIES`, may send `REPORTS_PER_MINUTE` (default 60) reports a minute
and past that gets 429 with `Retry-After`; the report files in `DATA_DIR`,
counted from start with those already there, may total at most
`DATA_DIR_MAX_BYTES` (default 1 GiB), past which reports get 507; and the
wildcard CORS is gone: no CORS headers unless `CORS_ALLOWED_ORIGINS` lists
origins. Deleting report files frees room only at the next start; pruning is
issue #54
- 2026-09-28: unified the gate (issue #16): the root `make check` covers the Go
backend as well as the frontend, and the pre-commit hook with it; the backend
moved onto scripts-to-rule-them-all (`backend/script/*`, `backend/Makefile` as
+1 -1
View File
@@ -2,7 +2,7 @@
# Entrypoints section of README.md). There is no check, hooks or docker
# target here: the root Makefile's check covers this directory, its
# hooks target installs the repo's only pre-commit hook, and its docker
# target builds the one image, which contains this backend.
# target builds this image, whose build context is the repo root.
.PHONY: all build test lint fmt fmt-check run clean
+13 -27
View File
@@ -11,16 +11,15 @@ From this directory:
make run
```
From the repo root, whose `Dockerfile` builds the one image that ships this
backend behind nginx (see [Container image](#container-image)):
From the repo root, which is also the build context of `Dockerfile.backend`:
```bash
# Run tests, lint, and format check over the frontend and this backend
make check
# Build the image: nginx, the frontend and this backend
# Build both images, including netwatch-server
make docker
docker run -p 8080:8080 netwatch
docker run -p 8080:8080 netwatch-server
```
## Entrypoints
@@ -28,7 +27,7 @@ docker run -p 8080:8080 netwatch
This directory follows the same
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
pattern as the repo root: the targets in `backend/Makefile` are thin shims over
`backend/script/`. The root `Dockerfile` runs them, and the root scripts call
`backend/script/`. `Dockerfile.backend` runs them, and the root scripts call
`test`, `fmt` and `fmt-check`:
- `script/build` — compile the static `netwatch-server` binary with its version
@@ -38,8 +37,8 @@ pattern as the repo root: the targets in `backend/Makefile` are thin shims over
- `script/test` — run the Go tests under a 30-second timeout
- `script/lint` — check `.golangci.yml` against its pinned sha256, then run
golangci-lint. It runs inside the golangci-lint image of the lint stage of
the root `Dockerfile`; from a checkout, run `make lint` at the repo root,
which builds that stage
`Dockerfile.backend`; from a checkout, run `make lint` at the repo root, which
builds that stage
- `script/fmt` — format the Go sources (writes)
- `script/fmt-check` — check Go formatting (read-only)
- `script/run` — build and run the server locally
@@ -47,7 +46,7 @@ pattern as the repo root: the targets in `backend/Makefile` are thin shims over
There is no `check`, `hooks` or `docker` target here: the root `make check`
covers this directory, the root `make hooks` installs the repo's only pre-commit
hook, and the root `make docker` builds the image that contains this backend.
hook, and the root `make docker` builds this image.
## Rationale
@@ -77,7 +76,6 @@ Internal packages in `internal/` follow standard Go project layout:
| Variable | Default | Description |
| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface |
| `PORT` | `8080` | HTTP listen port |
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most the report files in `DATA_DIR` may total; see [Report limits](#report-limits) |
@@ -91,15 +89,6 @@ The loopback entries cover the reverse proxy that shares the container; the
RFC1918 ranges match `nginx.conf`. A request whose direct peer is outside this
set has its forwarded headers ignored, and the direct peer is logged instead.
### Container image
The root `Dockerfile` builds one image in which nginx listens on the public port
8080, serves the frontend, and proxies `/api/` and `/.well-known/healthcheck` to
this server. The image's entrypoint, `bin/entrypoint.sh`, starts the server as
user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and `PORT=8081`, so
only nginx reaches it. `DATA_DIR` is `/data/reports`, on the `/data` volume,
which `netwatch` owns.
### Report storage
Reports are written as `reports-<timestamp>.jsonl.zst` files in `DATA_DIR`.
@@ -113,15 +102,12 @@ credentials, so it is bounded instead. Both refusals below answer with the same
`{"status":"error"}` body as any other error.
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
`Retry-After: 60`. The minute slides: reports from the minute before still
count, fading out over the current one, so an address is sure never to be
refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60
seconds. The page sends one report a minute from each open tab, so the default
of 60 refuses nothing from up to 30 tabs behind one address, such as a
household or an office sharing it, however their reports bunch up. Report
responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and
`X-RateLimit-Reset` headers.
send `REPORTS_PER_MINUTE` reports a minute, all at once if it likes. Past that
it gets 429 with a `Retry-After` header until its allowance refills, at one
report every 60 / `REPORTS_PER_MINUTE` seconds. The page sends one report a
minute from each open tab, so the default of 60 refuses nothing from up to 60
tabs behind one address, such as a household or an office sharing it, even
when all their reports arrive together.
- **Size cap.** The report files in `DATA_DIR` may total at most
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
waiting in memory count at their uncompressed size until they are written, so
+2 -4
View File
@@ -5,17 +5,16 @@ go 1.25.5
require (
github.com/go-chi/chi/v5 v5.2.5
github.com/go-chi/cors v1.2.2
github.com/go-chi/httprate v0.16.0
github.com/joho/godotenv v1.5.1
github.com/klauspost/compress v1.18.4
github.com/spf13/viper v1.21.0
go.uber.org/fx v1.24.0
golang.org/x/time v0.15.0
)
require (
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
@@ -23,11 +22,10 @@ require (
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.26.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.28.0 // indirect
)
+4 -10
View File
@@ -8,8 +8,6 @@ github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -18,8 +16,6 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -46,10 +42,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
@@ -62,10 +54,12 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-4
View File
@@ -43,7 +43,6 @@ type Params struct {
// Config holds the resolved application configuration.
type Config struct {
BindAddress string
CORSAllowedOrigins []string
DataDir string
DataDirMaxBytes int64
@@ -79,8 +78,6 @@ func New(
viper.SetDefault("DATA_DIR", "./data/reports")
viper.SetDefault("DATA_DIR_MAX_BYTES", defaultDataDirMaxBytes)
viper.SetDefault("DEBUG", "false")
// An empty BIND_ADDRESS listens on every interface.
viper.SetDefault("BIND_ADDRESS", "")
viper.SetDefault("PORT", "8080")
viper.SetDefault("REPORTS_PER_MINUTE", defaultReportsPerMinute)
viper.SetDefault("SENTRY_DSN", "")
@@ -98,7 +95,6 @@ func New(
}
s := &Config{
BindAddress: viper.GetString("BIND_ADDRESS"),
CORSAllowedOrigins: splitList(viper.GetString("CORS_ALLOWED_ORIGINS")),
DataDir: viper.GetString("DATA_DIR"),
DataDirMaxBytes: viper.GetInt64("DATA_DIR_MAX_BYTES"),
+70 -12
View File
@@ -7,11 +7,14 @@ import (
"fmt"
"io"
"log/slog"
"math"
"net"
"net/http"
"net/netip"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
"sneak.berlin/go/netwatch/internal/config"
@@ -20,8 +23,8 @@ import (
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/cors"
"github.com/go-chi/httprate"
"go.uber.org/fx"
"golang.org/x/time/rate"
)
const corsMaxAgeSec = 300
@@ -343,21 +346,76 @@ func (s *Middleware) CORS(
}
// RateLimit returns middleware that allows each client address
// perMinute requests a minute and answers the rest with 429, the
// Retry-After header httprate sets, and the usual error body. The
// address is the one clientIP resolves, so clients behind the reverse
// proxy are limited one by one, not together as the proxy.
// perMinute requests a minute, all at once if it likes, and answers
// the rest with 429 and a Retry-After header. The address is the one
// clientIP resolves, so clients behind the reverse proxy are limited
// one by one, not together as the proxy.
func (s *Middleware) RateLimit(
perMinute int,
) func(http.Handler) http.Handler {
return httprate.LimitBy(perMinute, time.Minute,
func(r *http.Request) (string, error) {
return clientIP(r.RemoteAddr, r.Header, s.trustedProxies), nil
},
httprate.WithLimitHandler(
func(w http.ResponseWriter, _ *http.Request) {
// One request's allowance comes back every interval, so a
// refused client can always retry after it.
interval := time.Minute / time.Duration(perMinute)
retryAfter := strconv.Itoa(int(math.Ceil(interval.Seconds())))
limiter := &addressLimiter{
burst: perMinute,
byAddr: make(map[string]*rate.Limiter),
limit: rate.Every(interval),
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
addr := clientIP(r.RemoteAddr, r.Header, s.trustedProxies)
if !limiter.allow(addr, time.Now()) {
w.Header().Set("Retry-After", retryAfter)
writeJSONError(w, http.StatusTooManyRequests)
return
}
next.ServeHTTP(w, r)
},
),
)
}
}
// addressLimiter holds a token bucket for each client address that
// has made a request recently.
type addressLimiter struct {
burst int
byAddr map[string]*rate.Limiter
lastSweep time.Time
limit rate.Limit
mu sync.Mutex
}
// allow takes a token from addr's bucket, which starts full, and
// reports whether there was one. At most once a minute it first drops
// every bucket that has filled up again: a full bucket behaves exactly
// like the new one that would replace it, so this changes no answer,
// and the map holds only the addresses heard from recently.
func (a *addressLimiter) allow(addr string, now time.Time) bool {
a.mu.Lock()
defer a.mu.Unlock()
if now.Sub(a.lastSweep) >= time.Minute {
for key, bucket := range a.byAddr {
if bucket.TokensAt(now) >= float64(a.burst) {
delete(a.byAddr, key)
}
}
a.lastSweep = now
}
bucket, ok := a.byAddr[addr]
if !ok {
bucket = rate.NewLimiter(a.limit, a.burst)
a.byAddr[addr] = bucket
}
return bucket.AllowN(now, 1)
}
@@ -0,0 +1,46 @@
package middleware
import (
"testing"
"time"
"golang.org/x/time/rate"
)
// TestAddressLimiterDropsOnlyFullBuckets checks the sweep in allow:
// a minute after the last one, it drops an address whose bucket has
// filled up again, and keeps one still short of tokens, whose limit
// would otherwise start over.
func TestAddressLimiterDropsOnlyFullBuckets(t *testing.T) {
t.Parallel()
const (
refilled = "198.51.100.1"
drained = "198.51.100.2"
)
// Two a minute: one token back every 30 seconds.
limiter := &addressLimiter{
burst: 2,
byAddr: make(map[string]*rate.Limiter),
limit: rate.Every(30 * time.Second),
}
start := time.Now()
// The first call sweeps the empty map and takes one of two tokens.
limiter.allow(refilled, start)
limiter.allow(drained, start.Add(59*time.Second))
limiter.allow(drained, start.Add(59*time.Second))
// A minute after the first sweep, this call sweeps again.
limiter.allow("198.51.100.3", start.Add(time.Minute))
if _, ok := limiter.byAddr[refilled]; ok {
t.Error("address with a full bucket was kept")
}
if _, ok := limiter.byAddr[drained]; !ok {
t.Error("address short of tokens was dropped")
}
}
+10 -13
View File
@@ -310,10 +310,11 @@ func okHandler() http.Handler {
})
}
// TestRateLimitRefusesPastAllowanceThenResets checks one client
// TestRateLimitRefusesPastAllowanceUntilRetryAfter checks one client
// address: it may use its whole allowance at once, the next request
// is refused with 429, and later it may send again.
func TestRateLimitRefusesPastAllowanceThenResets(t *testing.T) {
// is refused with 429, and once Retry-After has passed it may send
// again.
func TestRateLimitRefusesPastAllowanceUntilRetryAfter(t *testing.T) {
t.Parallel()
// synctest runs this on a fake clock: time.Sleep returns at once,
@@ -349,20 +350,16 @@ func TestRateLimitRefusesPastAllowanceThenResets(t *testing.T) {
t.Errorf("body = %q, want %q", got, "{\"status\":\"error\"}\n")
}
if got := rec.Header().Get("Retry-After"); got != "60" {
t.Fatalf("Retry-After = %q, want %q", got, "60")
// Two a minute: one request's allowance comes back every 30s.
if got := rec.Header().Get("Retry-After"); got != "30" {
t.Fatalf("Retry-After = %q, want %q", got, "30")
}
// httprate also counts the previous minute's requests, fading
// them out over the current one, so two minutes on the whole
// allowance is back.
time.Sleep(2 * time.Minute)
time.Sleep(30 * time.Second)
for i := range perMinute {
if code := post().Code; code != http.StatusOK {
t.Fatalf("two minutes later, request %d: status = %d, want %d",
i+1, code, http.StatusOK)
}
t.Fatalf("after Retry-After: status = %d, want %d",
code, http.StatusOK)
}
})
}
-6
View File
@@ -3,9 +3,3 @@ package server
// MaxRequestBodyBytes exposes the router-wide body limit to the
// external tests.
const MaxRequestBodyBytes = maxRequestBodyBytes
// ListenAddr exposes the address the server listens on to the
// external tests.
func (s *Server) ListenAddr() string {
return s.newHTTPServer().Addr
}
+2 -6
View File
@@ -2,9 +2,8 @@ package server
import (
"errors"
"net"
"fmt"
"net/http"
"strconv"
"time"
"go.uber.org/fx"
@@ -28,10 +27,7 @@ const (
// newHTTPServer constructs the http.Server. It performs no I/O
// and does not start listening.
func (s *Server) newHTTPServer() *http.Server {
listenAddr := net.JoinHostPort(
s.params.Config.BindAddress,
strconv.Itoa(s.params.Config.Port),
)
listenAddr := fmt.Sprintf(":%d", s.params.Config.Port)
return &http.Server{
Addr: listenAddr,
-32
View File
@@ -1,32 +0,0 @@
package server_test
import "testing"
// TestListenAddress checks that the server listens on BIND_ADDRESS
// and PORT, and on port 8080 on every interface when neither is set.
// The container image sets both, to keep the backend on loopback
// behind nginx.
func TestListenAddress(t *testing.T) {
tests := []struct {
bindAddress string
port string
want string
}{
{bindAddress: "", port: "", want: ":8080"},
{bindAddress: "127.0.0.1", port: "8081", want: "127.0.0.1:8081"},
{bindAddress: "::1", port: "8081", want: "[::1]:8081"},
}
for _, tt := range tests {
t.Run(tt.want, func(t *testing.T) {
// t.Setenv rules out t.Parallel.
t.Setenv("BIND_ADDRESS", tt.bindAddress)
t.Setenv("PORT", tt.port)
got := newServer(t).ListenAddr()
if got != tt.want {
t.Errorf("listen address = %q, want %q", got, tt.want)
}
})
}
}
+4 -5
View File
@@ -19,9 +19,8 @@ import (
"go.uber.org/fx/fxtest"
)
// newServer builds a Server from the same constructors as main,
// configured from the environment. It is never started, so nothing
// listens.
// newServer builds the server from the same constructors as main,
// never started: SetupRoutes is called directly, so nothing listens.
func newServer(t *testing.T) *server.Server {
t.Helper()
@@ -46,6 +45,8 @@ func newServer(t *testing.T) *server.Server {
t.Fatalf("build server: %v", err)
}
srv.SetupRoutes()
return srv
}
@@ -55,7 +56,6 @@ func TestReportsAreRateLimited(t *testing.T) {
t.Setenv("REPORTS_PER_MINUTE", "2")
srv := newServer(t)
srv.SetupRoutes()
post := func() int {
rec := httptest.NewRecorder()
@@ -88,7 +88,6 @@ func TestHealthCheckRejectsOversizeBody(t *testing.T) {
t.Parallel()
srv := newServer(t)
srv.SetupRoutes()
rec := httptest.NewRecorder()
req := httptest.NewRequestWithContext(t.Context(),
+1 -1
View File
@@ -8,7 +8,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
# VERSION comes from the environment (the root Dockerfile passes its
# VERSION comes from the environment (Dockerfile.backend passes its
# ARG VERSION in). Unset or empty, it is git describe, or "dev" where
# there is no git or no repository history.
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
# script/lint: run golangci-lint over the backend. This runs inside the
# lint stage of the root Dockerfile, whose digest-pinned golangci-lint
# lint stage of Dockerfile.backend, whose digest-pinned golangci-lint
# image provides the linter; nothing installs golangci-lint on the host.
# From a checkout, run `make lint` at the repo root, which builds that
# stage.
-59
View File
@@ -1,59 +0,0 @@
#!/bin/sh
# The container's entrypoint: runs netwatch-server and nginx side by
# side. TERM or INT stops both, and the container exits 0 if both exit
# cleanly. If either exits on its own, the other is stopped too and the
# container exits non-zero, so the platform restarts it instead of
# leaving it half up.
#
# No set -e: kill and wait return non-zero here in normal operation.
set -u
# A stop signal is only noted here; the loop below acts on it.
stop_requested=""
trap 'stop_requested=yes' TERM INT
# netwatch-server runs as the netwatch user and listens on loopback
# only, on a port other than the public one; nginx.conf proxies to this
# address. The netwatch user has no login shell, hence -s /bin/sh.
# busybox su replaces itself with the command instead of staying on as
# its parent, so $! is the server's own PID.
BIND_ADDRESS=127.0.0.1 PORT=8081 \
su -s /bin/sh netwatch -c 'exec netwatch-server' &
backend=$!
# nginx starts through the nginx image's own entrypoint, which applies
# the image's start-up configuration and then replaces itself with
# nginx.
/docker-entrypoint.sh nginx -g 'daemon off;' &
nginx=$!
running() {
kill -0 "$1" 2>/dev/null
}
# POSIX sh cannot wait for whichever of two children exits first, so
# look once a second. The shell collects a child that has exited while
# it runs sleep, and running() is false for that child from then on.
while [ -z "$stop_requested" ] && running "$backend" && running "$nginx"; do
sleep 1
done
# Stop both, then wait until neither is left.
kill -TERM "$backend" "$nginx" 2>/dev/null
while running "$backend" || running "$nginx"; do
sleep 1
done
wait "$backend"
backend_status=$?
wait "$nginx"
nginx_status=$?
echo "entrypoint: netwatch-server exited $backend_status," \
"nginx exited $nginx_status"
# Success is a requested stop that both processes exited cleanly from.
if [ -n "$stop_requested" ] && [ "$backend_status" -eq 0 ] &&
[ "$nginx_status" -eq 0 ]; then
exit 0
fi
exit 1
-19
View File
@@ -25,23 +25,4 @@ server {
expires 1y;
add_header Cache-Control "public, immutable";
}
# netwatch-server, the Go backend, runs in the same container and
# listens on loopback only: bin/entrypoint.sh starts it on
# 127.0.0.1:8081. These headers go with every request passed to it.
# X-Forwarded-For carries only the client address, as resolved by
# the real IP settings above, and not the chain the request came
# with: the backend takes the first entry, which a client can write.
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
location /api/ {
proxy_pass http://127.0.0.1:8081;
}
location = /.well-known/healthcheck {
proxy_pass http://127.0.0.1:8081;
}
}
+2 -2
View File
@@ -27,8 +27,8 @@ NVM_VERSION="0.40.3"
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
YARN_VERSION="1.22.22"
# The Go inside the golang:1.25-alpine image Dockerfile builds the
# backend with, 2026-08-09. The archive hashes are in ensure_go.
# The Go inside the golang:1.25-alpine image Dockerfile.backend builds
# with, 2026-08-09. The archive hashes are in ensure_go.
GO_VERSION="1.25.7"
BIN_DIR="$HOME/.local/bin"
+6 -4
View File
@@ -1,8 +1,9 @@
#!/bin/sh
# script/cibuild: run the CI build: build the one image from Dockerfile,
# whose stages run the checks as build steps (the backend's fmt-check,
# lint and tests, and the frontend's test, lint and fmt-check). This is
# the only build step the Gitea workflow runs.
# script/cibuild: run the CI build. It builds both images: the frontend
# from Dockerfile and the backend from Dockerfile.backend. Each runs its
# half of the checks as build steps, so a successful cibuild implies the
# whole repo is green. This is the only build step the Gitea workflow
# runs.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -10,6 +11,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
timeout 300 docker build .
timeout 300 docker build -f Dockerfile.backend .
}
main "$@"
+6 -14
View File
@@ -1,8 +1,7 @@
#!/bin/sh
# script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname.
# --no-cache because the gate phases the final stage depends on are RUN
# steps, and a cached one is a check that did not run.
# script/docker: build both Docker images, tagged with the project name
# from script/projectname: the frontend as <name>, from Dockerfile, and
# the backend as <name>-server, from Dockerfile.backend.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -10,16 +9,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
# Own line: a failing command substitution inside an argument does
# not trip `set -e`, so the inline form degrades silently to an
# empty constant. VERSION is computed here because .dockerignore
# excludes .git, so `git describe` in a build stage yields an empty
# version without failing.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$("$SCRIPT_DIR/projectname")" .
name="$("$SCRIPT_DIR/projectname")"
timeout 300 docker build -t "$name" .
timeout 300 docker build -t "$name-server" -f Dockerfile.backend .
}
main "$@"
+4 -4
View File
@@ -1,9 +1,9 @@
#!/bin/sh
# script/frontend-check: run the frontend half of the checks only (test,
# lint, fmt-check). This exists for the frontend stage of Dockerfile, a
# node image with neither Go nor Docker; the Dockerfile's lint and
# backend build stages gate the backend half. Everywhere else, use
# script/check, which covers the whole repo. Must not modify any files.
# lint, fmt-check). This exists for the frontend Dockerfile, whose build
# stage is a node image with neither Go nor Docker; the backend half is
# gated by Dockerfile.backend. Everywhere else, use script/check, which
# covers the whole repo. Must not modify any files.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+3 -3
View File
@@ -3,8 +3,8 @@
# Go linter over backend/.
#
# The Go linter runs only in Docker: this builds the lint stage of
# Dockerfile, the digest-pinned golangci-lint image, which runs the
# backend's fmt-check and lint targets. --no-cache makes the linter
# Dockerfile.backend, the digest-pinned golangci-lint image, which runs
# the backend's fmt-check and lint targets. --no-cache makes the linter
# really run every time rather than reuse an earlier result, and the
# stage is built for its checks alone, so no image is kept.
set -eu
@@ -15,7 +15,7 @@ main() {
cd "$ROOT"
"$ROOT/script/frontend-lint"
timeout 300 docker build --no-cache --target lint \
--output type=cacheonly .
--output type=cacheonly -f Dockerfile.backend .
}
main "$@"