Compare commits
2
Commits
2bf52ba7ff
...
9f4663cedb
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9f4663cedb | ||
|
|
bbcc7d921d |
@@ -6,5 +6,5 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
# actions/checkout v4.2.2, 2026-02-22
|
# actions/checkout v4.2.2, 2026-02-22
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||||
# script/cibuild builds both images.
|
# script/cibuild builds the image, whose stages run every check.
|
||||||
- run: script/cibuild
|
- run: script/cibuild
|
||||||
|
|||||||
+68
-5
@@ -1,5 +1,51 @@
|
|||||||
|
# The one image netwatch ships: nginx serves the built frontend and
|
||||||
|
# passes /api/ and /.well-known/healthcheck to netwatch-server, the Go
|
||||||
|
# backend, which runs in the same container on loopback only.
|
||||||
|
# bin/entrypoint.sh starts and watches both.
|
||||||
|
|
||||||
|
# Lint stage — fast feedback on formatting and lint issues. The
|
||||||
|
# golangci/golangci-lint image ships Go, gofmt, make and the linter, so
|
||||||
|
# nothing is installed here. The root make lint builds this stage alone.
|
||||||
|
# golangci/golangci-lint:v2.12.2 (2026-08-10)
|
||||||
|
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
COPY backend/go.mod backend/go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY backend/ .
|
||||||
|
RUN make fmt-check
|
||||||
|
RUN make lint
|
||||||
|
|
||||||
|
# Backend build stage
|
||||||
|
# golang:1.25-alpine (2026-02-27)
|
||||||
|
FROM golang:1.25-alpine@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
|
||||||
|
|
||||||
|
RUN apk add --no-cache make
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Force BuildKit to run the lint stage before proceeding. BuildKit runs
|
||||||
|
# stages in parallel by default; without this no-op copy a lint failure
|
||||||
|
# would not gate compilation.
|
||||||
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
|
COPY backend/go.mod backend/go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY backend/ .
|
||||||
|
|
||||||
|
RUN make test
|
||||||
|
|
||||||
|
# make build is a shim around backend/script/build, the one definition
|
||||||
|
# of the build command:
|
||||||
|
# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=... -X main.Buildarch=..."
|
||||||
|
# That script reads VERSION from the environment, so it is handed over
|
||||||
|
# there rather than as a make variable.
|
||||||
|
ARG VERSION=dev
|
||||||
|
RUN VERSION="${VERSION}" make build
|
||||||
|
|
||||||
|
# Frontend stage
|
||||||
# node:22-alpine as of 2026-02-22
|
# node:22-alpine as of 2026-02-22
|
||||||
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS build
|
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY package.json yarn.lock ./
|
COPY package.json yarn.lock ./
|
||||||
RUN yarn install --frozen-lockfile
|
RUN yarn install --frozen-lockfile
|
||||||
@@ -8,16 +54,33 @@ COPY . .
|
|||||||
# make frontend-check is the frontend half of make check (test + lint +
|
# make frontend-check is the frontend half of make check (test + lint +
|
||||||
# fmt-check); its test step is the production yarn build, so this both
|
# fmt-check); its test step is the production yarn build, so this both
|
||||||
# produces dist/ and gates the image on lint/fmt-check/test regressions.
|
# produces dist/ and gates the image on lint/fmt-check/test regressions.
|
||||||
# This node stage has neither Go nor Docker for the other half, which
|
# This node stage has neither Go nor Docker; the lint and builder stages
|
||||||
# Dockerfile.backend gates; script/cibuild builds both images.
|
# above gate the backend half.
|
||||||
RUN make frontend-check
|
RUN make frontend-check
|
||||||
|
|
||||||
|
# Runtime stage
|
||||||
# nginx:stable-alpine as of 2026-02-22
|
# nginx:stable-alpine as of 2026-02-22
|
||||||
FROM nginx@sha256:15e96e59aa3b0aada3a121296e3bce117721f42d88f5f64217ef4b18f458c6ab
|
FROM nginx@sha256:15e96e59aa3b0aada3a121296e3bce117721f42d88f5f64217ef4b18f458c6ab
|
||||||
|
|
||||||
|
# netwatch-server runs as this user, which owns the report directory.
|
||||||
|
# nginx keeps the image's own arrangement: its main process runs as
|
||||||
|
# root, its worker processes as the nginx user.
|
||||||
|
RUN addgroup -g 1000 -S netwatch && \
|
||||||
|
adduser -u 1000 -S netwatch -G netwatch
|
||||||
|
|
||||||
RUN rm /etc/nginx/conf.d/default.conf
|
RUN rm /etc/nginx/conf.d/default.conf
|
||||||
COPY nginx.conf /etc/nginx/conf.d/netwatch.conf
|
COPY nginx.conf /etc/nginx/conf.d/netwatch.conf
|
||||||
COPY --from=build /app/dist /usr/share/nginx/html
|
COPY --from=frontend /app/dist /usr/share/nginx/html
|
||||||
|
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
|
||||||
|
COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||||
|
|
||||||
|
ENV DATA_DIR=/data/reports
|
||||||
|
RUN mkdir -p /data/reports && chown -R netwatch:netwatch /data
|
||||||
|
VOLUME /data
|
||||||
|
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
|
|
||||||
CMD ["nginx", "-g", "daemon off;"]
|
# The nginx image stops its container with SIGQUIT; the entrypoint
|
||||||
|
# acts on TERM and INT.
|
||||||
|
STOPSIGNAL SIGTERM
|
||||||
|
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||||
|
|||||||
@@ -1,49 +0,0 @@
|
|||||||
# Lint stage — fast feedback on formatting and lint issues. The
|
|
||||||
# golangci/golangci-lint image ships Go, gofmt, make and the linter, so
|
|
||||||
# nothing is installed here.
|
|
||||||
# golangci/golangci-lint:v2.12.2 (2026-08-10)
|
|
||||||
FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
COPY backend/go.mod backend/go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY backend/ .
|
|
||||||
RUN make fmt-check
|
|
||||||
RUN make lint
|
|
||||||
|
|
||||||
# Build stage
|
|
||||||
# golang:1.25-alpine (2026-02-27)
|
|
||||||
FROM golang:1.25-alpine@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
|
|
||||||
|
|
||||||
RUN apk add --no-cache make
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Force BuildKit to run the lint stage before proceeding. BuildKit runs
|
|
||||||
# stages in parallel by default; without this no-op copy a lint failure
|
|
||||||
# would not gate compilation.
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
|
|
||||||
COPY backend/go.mod backend/go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY backend/ .
|
|
||||||
|
|
||||||
RUN make test
|
|
||||||
|
|
||||||
# make build is a shim around backend/script/build, the one definition
|
|
||||||
# of the build command:
|
|
||||||
# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=... -X main.Buildarch=..."
|
|
||||||
# That script reads VERSION from the environment, so it is handed over
|
|
||||||
# there rather than as a make variable.
|
|
||||||
ARG VERSION=dev
|
|
||||||
RUN VERSION="${VERSION}" make build
|
|
||||||
|
|
||||||
# Runtime stage
|
|
||||||
# alpine:3.23 (2026-02-27)
|
|
||||||
FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
|
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates
|
|
||||||
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
|
|
||||||
|
|
||||||
EXPOSE 8080
|
|
||||||
ENTRYPOINT ["netwatch-server"]
|
|
||||||
@@ -44,11 +44,11 @@ halves, so the root `make check` fails if either one is broken. We provide:
|
|||||||
linter in Docker
|
linter in Docker
|
||||||
- `script/setup` — make a fresh clone ready for development: bootstrap plus the
|
- `script/setup` — make a fresh clone ready for development: bootstrap plus the
|
||||||
git pre-commit hook
|
git pre-commit hook
|
||||||
- `script/projectname` — print the project name (used for the Docker image tags)
|
- `script/projectname` — print the project name (used for the Docker image tag)
|
||||||
- `script/test` — run `script/frontend-test`, then the backend's Go tests, both
|
- `script/test` — run `script/frontend-test`, then the backend's Go tests, both
|
||||||
within one 30-second timeout
|
within one 30-second timeout
|
||||||
- `script/lint` — run `script/frontend-lint`, then golangci-lint in Docker, by
|
- `script/lint` — run `script/frontend-lint`, then golangci-lint in Docker, by
|
||||||
building the lint stage of `Dockerfile.backend` without the cache
|
building the lint stage of `Dockerfile` without the cache
|
||||||
- `script/fmt` — format all files (writes): prettier, then gofmt over `backend/`
|
- `script/fmt` — format all files (writes): prettier, then gofmt over `backend/`
|
||||||
- `script/fmt-check` — check formatting (read-only): prettier, then gofmt
|
- `script/fmt-check` — check formatting (read-only): prettier, then gofmt
|
||||||
- `script/check` — run test, lint, and fmt-check
|
- `script/check` — run test, lint, and fmt-check
|
||||||
@@ -63,9 +63,9 @@ halves, so the root `make check` fails if either one is broken. We provide:
|
|||||||
frontend in a containerised headless Chrome (see
|
frontend in a containerised headless Chrome (see
|
||||||
[test/viewport/README.md](test/viewport/README.md)). Not part of
|
[test/viewport/README.md](test/viewport/README.md)). Not part of
|
||||||
`script/check`: it needs Docker and takes minutes.
|
`script/check`: it needs Docker and takes minutes.
|
||||||
- `script/docker` — build both images, tagged via `script/projectname`:
|
- `script/docker` — build the image from `Dockerfile` without the build cache,
|
||||||
`netwatch` from `Dockerfile` and `netwatch-server` from `Dockerfile.backend`
|
tagged `netwatch` via `script/projectname`
|
||||||
- `script/cibuild` — CI entrypoint: builds both images
|
- `script/cibuild` — CI entrypoint: builds the image
|
||||||
- `script/precommit` — run by the git pre-commit hook; runs `script/check`
|
- `script/precommit` — run by the git pre-commit hook; runs `script/check`
|
||||||
- `script/install-precommit` — install the git pre-commit hook
|
- `script/install-precommit` — install the git pre-commit hook
|
||||||
|
|
||||||
@@ -178,13 +178,21 @@ After running `yarn build`, deploy the contents of the `dist/` directory to any
|
|||||||
static file host (S3, GCS, Cloudflare Pages, Vercel, Netlify, GitHub Pages) or
|
static file host (S3, GCS, Cloudflare Pages, Vercel, Netlify, GitHub Pages) or
|
||||||
use the Docker image behind a reverse proxy.
|
use the Docker image behind a reverse proxy.
|
||||||
|
|
||||||
The Docker image:
|
The Docker image, built from `Dockerfile`, is the whole service in one
|
||||||
|
container: nginx serves the built frontend and passes `/api/` and
|
||||||
|
`/.well-known/healthcheck` to the Go backend, `netwatch-server`, which listens
|
||||||
|
only inside the container, on `127.0.0.1:8081`. The image:
|
||||||
|
|
||||||
- Listens on port 8080 by default (override with `PORT` env var)
|
- Listens on port 8080 by default (override with `PORT` env var)
|
||||||
- Trusts `X-Forwarded-For` from RFC1918 reverse proxies (10/8, 172.16/12,
|
- Trusts `X-Forwarded-For` from RFC1918 reverse proxies (10/8, 172.16/12,
|
||||||
192.168/16)
|
192.168/16)
|
||||||
- Sends access logs to stdout
|
- Sends access logs to stdout
|
||||||
- Caches static assets with immutable headers
|
- Caches static assets with immutable headers
|
||||||
|
- Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data`
|
||||||
|
volume. The backend runs as user `netwatch` (uid 1000), so a directory
|
||||||
|
bind-mounted at `/data` must be writable by uid 1000
|
||||||
|
- Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx
|
||||||
|
or the backend exits on its own, so the platform restarts it
|
||||||
|
|
||||||
## Browser Compatibility
|
## Browser Compatibility
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,23 @@ latest run passes.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-09-29: bounded the report endpoint (issue #20): `POST /api/v1/reports`
|
||||||
|
still needs no credentials, but each client address, as resolved through
|
||||||
|
`TRUSTED_PROXIES`, may send `REPORTS_PER_MINUTE` (default 60) reports a
|
||||||
|
minute, counted by `go-chi/httprate`, and past that gets 429 with
|
||||||
|
`Retry-After`; the report files in `DATA_DIR`, counted from start with those
|
||||||
|
already there, may total at most `DATA_DIR_MAX_BYTES` (default 1 GiB), past
|
||||||
|
which reports get 507; and the wildcard CORS is gone: no CORS headers unless
|
||||||
|
`CORS_ALLOWED_ORIGINS` lists origins. Deleting report files frees room only at
|
||||||
|
the next start; pruning is issue #54
|
||||||
|
- 2026-09-28: one container image (issue #52): the root `Dockerfile` builds the
|
||||||
|
only image, and `Dockerfile.backend` is gone. nginx serves the frontend on
|
||||||
|
port 8080 and proxies `/api/` and `/.well-known/healthcheck` to the backend,
|
||||||
|
which listens on `127.0.0.1:8081` in the same container; the new
|
||||||
|
`BIND_ADDRESS` setting sets its listen address. `bin/entrypoint.sh` starts
|
||||||
|
both, passes TERM and INT on to both, and exits non-zero when either exits on
|
||||||
|
its own. The backend runs as user `netwatch` and stores reports on the `/data`
|
||||||
|
volume. `script/docker` is the org model again
|
||||||
- 2026-09-28: unified the gate (issue #16): the root `make check` covers the Go
|
- 2026-09-28: unified the gate (issue #16): the root `make check` covers the Go
|
||||||
backend as well as the frontend, and the pre-commit hook with it; the backend
|
backend as well as the frontend, and the pre-commit hook with it; the backend
|
||||||
moved onto scripts-to-rule-them-all (`backend/script/*`, `backend/Makefile` as
|
moved onto scripts-to-rule-them-all (`backend/script/*`, `backend/Makefile` as
|
||||||
|
|||||||
+1
-1
@@ -2,7 +2,7 @@
|
|||||||
# Entrypoints section of README.md). There is no check, hooks or docker
|
# Entrypoints section of README.md). There is no check, hooks or docker
|
||||||
# target here: the root Makefile's check covers this directory, its
|
# target here: the root Makefile's check covers this directory, its
|
||||||
# hooks target installs the repo's only pre-commit hook, and its docker
|
# hooks target installs the repo's only pre-commit hook, and its docker
|
||||||
# target builds this image, whose build context is the repo root.
|
# target builds the one image, which contains this backend.
|
||||||
|
|
||||||
.PHONY: all build test lint fmt fmt-check run clean
|
.PHONY: all build test lint fmt fmt-check run clean
|
||||||
|
|
||||||
|
|||||||
+59
-13
@@ -11,15 +11,16 @@ From this directory:
|
|||||||
make run
|
make run
|
||||||
```
|
```
|
||||||
|
|
||||||
From the repo root, which is also the build context of `Dockerfile.backend`:
|
From the repo root, whose `Dockerfile` builds the one image that ships this
|
||||||
|
backend behind nginx (see [Container image](#container-image)):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Run tests, lint, and format check over the frontend and this backend
|
# Run tests, lint, and format check over the frontend and this backend
|
||||||
make check
|
make check
|
||||||
|
|
||||||
# Build both images, including netwatch-server
|
# Build the image: nginx, the frontend and this backend
|
||||||
make docker
|
make docker
|
||||||
docker run -p 8080:8080 netwatch-server
|
docker run -p 8080:8080 netwatch
|
||||||
```
|
```
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
@@ -27,7 +28,7 @@ docker run -p 8080:8080 netwatch-server
|
|||||||
This directory follows the same
|
This directory follows the same
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
pattern as the repo root: the targets in `backend/Makefile` are thin shims over
|
pattern as the repo root: the targets in `backend/Makefile` are thin shims over
|
||||||
`backend/script/`. `Dockerfile.backend` runs them, and the root scripts call
|
`backend/script/`. The root `Dockerfile` runs them, and the root scripts call
|
||||||
`test`, `fmt` and `fmt-check`:
|
`test`, `fmt` and `fmt-check`:
|
||||||
|
|
||||||
- `script/build` — compile the static `netwatch-server` binary with its version
|
- `script/build` — compile the static `netwatch-server` binary with its version
|
||||||
@@ -37,8 +38,8 @@ pattern as the repo root: the targets in `backend/Makefile` are thin shims over
|
|||||||
- `script/test` — run the Go tests under a 30-second timeout
|
- `script/test` — run the Go tests under a 30-second timeout
|
||||||
- `script/lint` — check `.golangci.yml` against its pinned sha256, then run
|
- `script/lint` — check `.golangci.yml` against its pinned sha256, then run
|
||||||
golangci-lint. It runs inside the golangci-lint image of the lint stage of
|
golangci-lint. It runs inside the golangci-lint image of the lint stage of
|
||||||
`Dockerfile.backend`; from a checkout, run `make lint` at the repo root, which
|
the root `Dockerfile`; from a checkout, run `make lint` at the repo root,
|
||||||
builds that stage
|
which builds that stage
|
||||||
- `script/fmt` — format the Go sources (writes)
|
- `script/fmt` — format the Go sources (writes)
|
||||||
- `script/fmt-check` — check Go formatting (read-only)
|
- `script/fmt-check` — check Go formatting (read-only)
|
||||||
- `script/run` — build and run the server locally
|
- `script/run` — build and run the server locally
|
||||||
@@ -46,7 +47,7 @@ pattern as the repo root: the targets in `backend/Makefile` are thin shims over
|
|||||||
|
|
||||||
There is no `check`, `hooks` or `docker` target here: the root `make check`
|
There is no `check`, `hooks` or `docker` target here: the root `make check`
|
||||||
covers this directory, the root `make hooks` installs the repo's only pre-commit
|
covers this directory, the root `make hooks` installs the repo's only pre-commit
|
||||||
hook, and the root `make docker` builds this image.
|
hook, and the root `make docker` builds the image that contains this backend.
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
@@ -74,24 +75,69 @@ Internal packages in `internal/` follow standard Go project layout:
|
|||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
| Variable | Default | Description |
|
| Variable | Default | Description |
|
||||||
| ----------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
|
| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||||
| `PORT` | `8080` | HTTP listen port |
|
| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface |
|
||||||
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
| `PORT` | `8080` | HTTP listen port |
|
||||||
| `DEBUG` | `false` | Enable debug logging |
|
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
||||||
| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most the report files in `DATA_DIR` may total; see [Report limits](#report-limits) |
|
||||||
|
| `DEBUG` | `false` | Enable debug logging |
|
||||||
|
| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
||||||
|
| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) |
|
||||||
|
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
||||||
|
|
||||||
`TRUSTED_PROXIES` defaults to `127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`.
|
`TRUSTED_PROXIES` defaults to `127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`.
|
||||||
The loopback entries cover the reverse proxy that shares the container; the
|
The loopback entries cover the reverse proxy that shares the container; the
|
||||||
RFC1918 ranges match `nginx.conf`. A request whose direct peer is outside this
|
RFC1918 ranges match `nginx.conf`. A request whose direct peer is outside this
|
||||||
set has its forwarded headers ignored, and the direct peer is logged instead.
|
set has its forwarded headers ignored, and the direct peer is logged instead.
|
||||||
|
|
||||||
|
### Container image
|
||||||
|
|
||||||
|
The root `Dockerfile` builds one image in which nginx listens on the public port
|
||||||
|
8080, serves the frontend, and proxies `/api/` and `/.well-known/healthcheck` to
|
||||||
|
this server. The image's entrypoint, `bin/entrypoint.sh`, starts the server as
|
||||||
|
user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and `PORT=8081`, so
|
||||||
|
only nginx reaches it. `DATA_DIR` is `/data/reports`, on the `/data` volume,
|
||||||
|
which `netwatch` owns.
|
||||||
|
|
||||||
### Report storage
|
### Report storage
|
||||||
|
|
||||||
Reports are written as `reports-<timestamp>.jsonl.zst` files in `DATA_DIR`.
|
Reports are written as `reports-<timestamp>.jsonl.zst` files in `DATA_DIR`.
|
||||||
Each file contains one JSON object per line, compressed with zstd. Files are
|
Each file contains one JSON object per line, compressed with zstd. Files are
|
||||||
created with `O_EXCL` to prevent overwrites.
|
created with `O_EXCL` to prevent overwrites.
|
||||||
|
|
||||||
|
### Report limits
|
||||||
|
|
||||||
|
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
||||||
|
credentials, so it is bounded instead. Both refusals below answer with the same
|
||||||
|
`{"status":"error"}` body as any other error.
|
||||||
|
|
||||||
|
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
||||||
|
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
||||||
|
`Retry-After: 60`. The minute slides: reports from the minute before still
|
||||||
|
count, fading out over the current one, so an address is sure never to be
|
||||||
|
refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60
|
||||||
|
seconds. The page sends one report a minute from each open tab, so the default
|
||||||
|
of 60 refuses nothing from up to 30 tabs behind one address, such as a
|
||||||
|
household or an office sharing it, however their reports bunch up. Report
|
||||||
|
responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and
|
||||||
|
`X-RateLimit-Reset` headers.
|
||||||
|
- **Size cap.** The report files in `DATA_DIR` may total at most
|
||||||
|
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
|
||||||
|
waiting in memory count at their uncompressed size until they are written, so
|
||||||
|
a report that would take the total past the cap is refused with 507, and
|
||||||
|
nothing of it is stored. Deleting report files frees room only at the next
|
||||||
|
start, when the files are counted again. The default of 1 GiB is small enough
|
||||||
|
for any host; set it to the space you can give `DATA_DIR`.
|
||||||
|
|
||||||
|
### CORS
|
||||||
|
|
||||||
|
The page calls the API from the origin it is served from, so by default the
|
||||||
|
server sends no CORS headers, and browsers let no other origin's pages call it.
|
||||||
|
To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS`
|
||||||
|
(for example `https://netwatch.example.com`); pages from a listed origin may
|
||||||
|
`GET` and `POST` with a `Content-Type` header.
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
- Add integration test that POSTs a report and verifies the compressed output
|
- Add integration test that POSTs a report and verifies the compressed output
|
||||||
|
|||||||
+4
-1
@@ -5,6 +5,7 @@ go 1.25.5
|
|||||||
require (
|
require (
|
||||||
github.com/go-chi/chi/v5 v5.2.5
|
github.com/go-chi/chi/v5 v5.2.5
|
||||||
github.com/go-chi/cors v1.2.2
|
github.com/go-chi/cors v1.2.2
|
||||||
|
github.com/go-chi/httprate v0.16.0
|
||||||
github.com/joho/godotenv v1.5.1
|
github.com/joho/godotenv v1.5.1
|
||||||
github.com/klauspost/compress v1.18.4
|
github.com/klauspost/compress v1.18.4
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
@@ -14,6 +15,7 @@ require (
|
|||||||
require (
|
require (
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||||
@@ -21,10 +23,11 @@ require (
|
|||||||
github.com/spf13/cast v1.10.0 // indirect
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
github.com/spf13/pflag v1.0.10 // indirect
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
github.com/subosito/gotenv v1.6.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
|
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||||
go.uber.org/dig v1.19.0 // indirect
|
go.uber.org/dig v1.19.0 // indirect
|
||||||
go.uber.org/multierr v1.10.0 // indirect
|
go.uber.org/multierr v1.10.0 // indirect
|
||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
golang.org/x/sys v0.29.0 // indirect
|
golang.org/x/sys v0.30.0 // indirect
|
||||||
golang.org/x/text v0.28.0 // indirect
|
golang.org/x/text v0.28.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
+10
-2
@@ -8,6 +8,8 @@ github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
|||||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||||
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||||
|
github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
|
||||||
|
github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
@@ -16,6 +18,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
|||||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
@@ -42,6 +46,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
|||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||||
|
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||||
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
|
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||||
|
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||||
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
||||||
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
||||||
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
||||||
@@ -54,8 +62,8 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
|||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -23,6 +24,15 @@ import (
|
|||||||
const defaultTrustedProxies = "127.0.0.1/32,::1/128," +
|
const defaultTrustedProxies = "127.0.0.1/32,::1/128," +
|
||||||
"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||||
|
|
||||||
|
// Default limits on stored reports; backend/README.md gives the
|
||||||
|
// reasons for these values.
|
||||||
|
const (
|
||||||
|
defaultReportsPerMinute = 60
|
||||||
|
defaultDataDirMaxBytes = 1 << 30 // 1 GiB
|
||||||
|
)
|
||||||
|
|
||||||
|
var errNotPositive = errors.New("must be a positive whole number")
|
||||||
|
|
||||||
// Params defines the dependencies for Config.
|
// Params defines the dependencies for Config.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -33,15 +43,19 @@ type Params struct {
|
|||||||
|
|
||||||
// Config holds the resolved application configuration.
|
// Config holds the resolved application configuration.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
DataDir string
|
BindAddress string
|
||||||
Debug bool
|
CORSAllowedOrigins []string
|
||||||
MetricsPassword string
|
DataDir string
|
||||||
MetricsUsername string
|
DataDirMaxBytes int64
|
||||||
Port int
|
Debug bool
|
||||||
SentryDSN string
|
MetricsPassword string
|
||||||
TrustedProxies []string
|
MetricsUsername string
|
||||||
log *slog.Logger
|
Port int
|
||||||
params *Params
|
ReportsPerMinute int
|
||||||
|
SentryDSN string
|
||||||
|
TrustedProxies []string
|
||||||
|
log *slog.Logger
|
||||||
|
params *Params
|
||||||
}
|
}
|
||||||
|
|
||||||
// New loads configuration from env, .env files, and config
|
// New loads configuration from env, .env files, and config
|
||||||
@@ -60,9 +74,15 @@ func New(
|
|||||||
|
|
||||||
viper.AutomaticEnv()
|
viper.AutomaticEnv()
|
||||||
|
|
||||||
|
// An empty CORS_ALLOWED_ORIGINS allows no other origin.
|
||||||
|
viper.SetDefault("CORS_ALLOWED_ORIGINS", "")
|
||||||
viper.SetDefault("DATA_DIR", "./data/reports")
|
viper.SetDefault("DATA_DIR", "./data/reports")
|
||||||
|
viper.SetDefault("DATA_DIR_MAX_BYTES", defaultDataDirMaxBytes)
|
||||||
viper.SetDefault("DEBUG", "false")
|
viper.SetDefault("DEBUG", "false")
|
||||||
|
// An empty BIND_ADDRESS listens on every interface.
|
||||||
|
viper.SetDefault("BIND_ADDRESS", "")
|
||||||
viper.SetDefault("PORT", "8080")
|
viper.SetDefault("PORT", "8080")
|
||||||
|
viper.SetDefault("REPORTS_PER_MINUTE", defaultReportsPerMinute)
|
||||||
viper.SetDefault("SENTRY_DSN", "")
|
viper.SetDefault("SENTRY_DSN", "")
|
||||||
viper.SetDefault("METRICS_USERNAME", "")
|
viper.SetDefault("METRICS_USERNAME", "")
|
||||||
viper.SetDefault("METRICS_PASSWORD", "")
|
viper.SetDefault("METRICS_PASSWORD", "")
|
||||||
@@ -78,15 +98,31 @@ func New(
|
|||||||
}
|
}
|
||||||
|
|
||||||
s := &Config{
|
s := &Config{
|
||||||
DataDir: viper.GetString("DATA_DIR"),
|
BindAddress: viper.GetString("BIND_ADDRESS"),
|
||||||
Debug: viper.GetBool("DEBUG"),
|
CORSAllowedOrigins: splitList(viper.GetString("CORS_ALLOWED_ORIGINS")),
|
||||||
MetricsPassword: viper.GetString("METRICS_PASSWORD"),
|
DataDir: viper.GetString("DATA_DIR"),
|
||||||
MetricsUsername: viper.GetString("METRICS_USERNAME"),
|
DataDirMaxBytes: viper.GetInt64("DATA_DIR_MAX_BYTES"),
|
||||||
Port: viper.GetInt("PORT"),
|
Debug: viper.GetBool("DEBUG"),
|
||||||
SentryDSN: viper.GetString("SENTRY_DSN"),
|
MetricsPassword: viper.GetString("METRICS_PASSWORD"),
|
||||||
TrustedProxies: splitList(viper.GetString("TRUSTED_PROXIES")),
|
MetricsUsername: viper.GetString("METRICS_USERNAME"),
|
||||||
log: log,
|
Port: viper.GetInt("PORT"),
|
||||||
params: ¶ms,
|
ReportsPerMinute: viper.GetInt("REPORTS_PER_MINUTE"),
|
||||||
|
SentryDSN: viper.GetString("SENTRY_DSN"),
|
||||||
|
TrustedProxies: splitList(viper.GetString("TRUSTED_PROXIES")),
|
||||||
|
log: log,
|
||||||
|
params: ¶ms,
|
||||||
|
}
|
||||||
|
|
||||||
|
// viper reads a value that is not a number as 0, so this also
|
||||||
|
// catches a mistyped setting.
|
||||||
|
if s.ReportsPerMinute <= 0 {
|
||||||
|
return nil, fmt.Errorf("REPORTS_PER_MINUTE %q: %w",
|
||||||
|
viper.GetString("REPORTS_PER_MINUTE"), errNotPositive)
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.DataDirMaxBytes <= 0 {
|
||||||
|
return nil, fmt.Errorf("DATA_DIR_MAX_BYTES %q: %w",
|
||||||
|
viper.GetString("DATA_DIR_MAX_BYTES"), errNotPositive)
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.Debug {
|
if s.Debug {
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package config_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/netwatch/internal/config"
|
||||||
|
"sneak.berlin/go/netwatch/internal/globals"
|
||||||
|
"sneak.berlin/go/netwatch/internal/logger"
|
||||||
|
|
||||||
|
"go.uber.org/fx"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requireConfigError builds the config as main does and fails the
|
||||||
|
// test unless that fails with an error naming setting. It uses
|
||||||
|
// fx.New, because fxtest.New fails the test itself on an error.
|
||||||
|
func requireConfigError(t *testing.T, setting string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
app := fx.New(
|
||||||
|
fx.NopLogger,
|
||||||
|
fx.Provide(globals.New, logger.New, config.New),
|
||||||
|
fx.Invoke(func(*config.Config) {}),
|
||||||
|
)
|
||||||
|
|
||||||
|
err := app.Err()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), setting) {
|
||||||
|
t.Fatalf("config error = %v, want one naming %s", err, setting)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReportsPerMinuteMustBePositive: unchecked, zero would panic
|
||||||
|
// when the routes are built, and a negative rate would lift the
|
||||||
|
// limit.
|
||||||
|
func TestReportsPerMinuteMustBePositive(t *testing.T) {
|
||||||
|
t.Setenv("REPORTS_PER_MINUTE", "0")
|
||||||
|
|
||||||
|
requireConfigError(t, "REPORTS_PER_MINUTE")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDataDirMaxBytesMustBeANumber: viper reads a value that is not
|
||||||
|
// a number, such as "1GB", as 0, which would refuse every report.
|
||||||
|
func TestDataDirMaxBytesMustBeANumber(t *testing.T) {
|
||||||
|
t.Setenv("DATA_DIR_MAX_BYTES", "1GB")
|
||||||
|
|
||||||
|
requireConfigError(t, "DATA_DIR_MAX_BYTES")
|
||||||
|
}
|
||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"sneak.berlin/go/netwatch/internal/reportbuf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// maxLoggedFieldBytes bounds untrusted text (string fields,
|
// maxLoggedFieldBytes bounds untrusted text (string fields,
|
||||||
@@ -55,10 +57,9 @@ func (s *Handlers) HandleReport() http.HandlerFunc {
|
|||||||
|
|
||||||
err = s.buf.Append(rpt)
|
err = s.buf.Append(rpt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.log.Error("failed to buffer report", "error", err)
|
|
||||||
s.respondJSON(w, r,
|
s.respondJSON(w, r,
|
||||||
&response{Status: "error"},
|
&response{Status: "error"},
|
||||||
http.StatusInternalServerError,
|
s.appendErrorStatus(err),
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -88,6 +89,21 @@ func (s *Handlers) decodeErrorStatus(err error) int {
|
|||||||
return http.StatusBadRequest
|
return http.StatusBadRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// appendErrorStatus logs a failure to store a report and returns
|
||||||
|
// the status to send: 507 when the report files are at their size
|
||||||
|
// cap, otherwise 500.
|
||||||
|
func (s *Handlers) appendErrorStatus(err error) int {
|
||||||
|
if errors.Is(err, reportbuf.ErrFull) {
|
||||||
|
s.log.Warn("report refused: report files at their size cap")
|
||||||
|
|
||||||
|
return http.StatusInsufficientStorage
|
||||||
|
}
|
||||||
|
|
||||||
|
s.log.Error("failed to buffer report", "error", err)
|
||||||
|
|
||||||
|
return http.StatusInternalServerError
|
||||||
|
}
|
||||||
|
|
||||||
// logReportReceived logs an accepted report. Untrusted fields are
|
// logReportReceived logs an accepted report. Untrusted fields are
|
||||||
// bounded (client_id, timestamp) or reduced to a length
|
// bounded (client_id, timestamp) or reduced to a length
|
||||||
// (geo_bytes) so the raw attacker-controlled body never reaches
|
// (geo_bytes) so the raw attacker-controlled body never reaches
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/netwatch/internal/handlers"
|
"sneak.berlin/go/netwatch/internal/handlers"
|
||||||
"sneak.berlin/go/netwatch/internal/middleware"
|
"sneak.berlin/go/netwatch/internal/middleware"
|
||||||
|
"sneak.berlin/go/netwatch/internal/reportbuf"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errStorageFailed = errors.New("storage failed")
|
var errStorageFailed = errors.New("storage failed")
|
||||||
@@ -66,6 +67,32 @@ func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleReportFullIs507 checks the answer when the report files
|
||||||
|
// are at their size cap: 507 and the usual error body, which tells
|
||||||
|
// the client nothing more.
|
||||||
|
func TestHandleReportFullIs507(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h := newTestHandlers(stubAppender{err: reportbuf.ErrFull}, io.Discard)
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(),
|
||||||
|
http.MethodPost, "/api/v1/reports",
|
||||||
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
|
)
|
||||||
|
|
||||||
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
if rec.Code != http.StatusInsufficientStorage {
|
||||||
|
t.Fatalf("status = %d, want %d",
|
||||||
|
rec.Code, http.StatusInsufficientStorage)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := rec.Body.String(); got != "{\"status\":\"error\"}\n" {
|
||||||
|
t.Errorf("body = %q, want %q", got, "{\"status\":\"error\"}\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandleReportMalformedJSONIs400(t *testing.T) {
|
func TestHandleReportMalformedJSONIs400(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,13 @@ func NewWithLogger(log *slog.Logger) *Middleware {
|
|||||||
return &Middleware{log: log}
|
return &Middleware{log: log}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewWithTrustedProxies builds a Middleware that honours forwarded
|
||||||
|
// headers from the given networks, for tests of the client address
|
||||||
|
// paths without the fx graph.
|
||||||
|
func NewWithTrustedProxies(trusted []netip.Prefix) *Middleware {
|
||||||
|
return &Middleware{trustedProxies: trusted}
|
||||||
|
}
|
||||||
|
|
||||||
func ClientIP(
|
func ClientIP(
|
||||||
remoteAddr string,
|
remoteAddr string,
|
||||||
header http.Header,
|
header http.Header,
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
|
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"github.com/go-chi/cors"
|
"github.com/go-chi/cors"
|
||||||
|
"github.com/go-chi/httprate"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -320,21 +321,43 @@ func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CORS returns middleware that adds permissive CORS headers.
|
// CORS returns middleware that lets pages served from the given
|
||||||
func (s *Middleware) CORS() func(http.Handler) http.Handler {
|
// origins call the API. With no origins it adds no CORS headers at
|
||||||
|
// all, so only same-origin pages can use the API. That case must not
|
||||||
|
// reach cors.Handler, which treats an empty origin list as "allow
|
||||||
|
// every origin".
|
||||||
|
func (s *Middleware) CORS(
|
||||||
|
origins []string,
|
||||||
|
) func(http.Handler) http.Handler {
|
||||||
|
if len(origins) == 0 {
|
||||||
|
return func(next http.Handler) http.Handler { return next }
|
||||||
|
}
|
||||||
|
|
||||||
return cors.Handler(cors.Options{
|
return cors.Handler(cors.Options{
|
||||||
AllowedOrigins: []string{"*"},
|
AllowedOrigins: origins,
|
||||||
AllowedMethods: []string{
|
AllowedMethods: []string{http.MethodGet, http.MethodPost},
|
||||||
"GET", "POST", "PUT", "DELETE", "OPTIONS",
|
AllowedHeaders: []string{"Content-Type"},
|
||||||
},
|
|
||||||
AllowedHeaders: []string{
|
|
||||||
"Accept",
|
|
||||||
"Authorization",
|
|
||||||
"Content-Type",
|
|
||||||
"X-CSRF-Token",
|
|
||||||
},
|
|
||||||
ExposedHeaders: []string{"Link"},
|
|
||||||
AllowCredentials: false,
|
AllowCredentials: false,
|
||||||
MaxAge: corsMaxAgeSec,
|
MaxAge: corsMaxAgeSec,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RateLimit returns middleware that allows each client address
|
||||||
|
// perMinute requests a minute and answers the rest with 429, the
|
||||||
|
// Retry-After header httprate sets, and the usual error body. The
|
||||||
|
// address is the one clientIP resolves, so clients behind the reverse
|
||||||
|
// proxy are limited one by one, not together as the proxy.
|
||||||
|
func (s *Middleware) RateLimit(
|
||||||
|
perMinute int,
|
||||||
|
) func(http.Handler) http.Handler {
|
||||||
|
return httprate.LimitBy(perMinute, time.Minute,
|
||||||
|
func(r *http.Request) (string, error) {
|
||||||
|
return clientIP(r.RemoteAddr, r.Header, s.trustedProxies), nil
|
||||||
|
},
|
||||||
|
httprate.WithLimitHandler(
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
writeJSONError(w, http.StatusTooManyRequests)
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/netwatch/internal/middleware"
|
"sneak.berlin/go/netwatch/internal/middleware"
|
||||||
)
|
)
|
||||||
@@ -300,3 +302,170 @@ func TestRecovererRepanicsOnAbortHandler(t *testing.T) {
|
|||||||
t.Errorf("abort was logged: %q", logbuf.String())
|
t.Errorf("abort was logged: %q", logbuf.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// okHandler stands in for the route a middleware guards.
|
||||||
|
func okHandler() http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRateLimitRefusesPastAllowanceThenResets checks one client
|
||||||
|
// address: it may use its whole allowance at once, the next request
|
||||||
|
// is refused with 429, and later it may send again.
|
||||||
|
func TestRateLimitRefusesPastAllowanceThenResets(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// synctest runs this on a fake clock: time.Sleep returns at once,
|
||||||
|
// with the clock moved on.
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
const perMinute = 2
|
||||||
|
|
||||||
|
handler := (&middleware.Middleware{}).RateLimit(perMinute)(okHandler())
|
||||||
|
|
||||||
|
post := func() *httptest.ResponseRecorder {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(),
|
||||||
|
http.MethodPost, "/api/v1/reports", http.NoBody)
|
||||||
|
handler.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range perMinute {
|
||||||
|
if code := post().Code; code != http.StatusOK {
|
||||||
|
t.Fatalf("request %d: status = %d, want %d",
|
||||||
|
i+1, code, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := post()
|
||||||
|
if rec.Code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("request past the allowance: status = %d, want %d",
|
||||||
|
rec.Code, http.StatusTooManyRequests)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := rec.Body.String(); got != "{\"status\":\"error\"}\n" {
|
||||||
|
t.Errorf("body = %q, want %q", got, "{\"status\":\"error\"}\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := rec.Header().Get("Retry-After"); got != "60" {
|
||||||
|
t.Fatalf("Retry-After = %q, want %q", got, "60")
|
||||||
|
}
|
||||||
|
|
||||||
|
// httprate also counts the previous minute's requests, fading
|
||||||
|
// them out over the current one, so two minutes on the whole
|
||||||
|
// allowance is back.
|
||||||
|
time.Sleep(2 * time.Minute)
|
||||||
|
|
||||||
|
for i := range perMinute {
|
||||||
|
if code := post().Code; code != http.StatusOK {
|
||||||
|
t.Fatalf("two minutes later, request %d: status = %d, want %d",
|
||||||
|
i+1, code, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRateLimitIsPerForwardedClient checks that clients behind a
|
||||||
|
// trusted proxy each get their own allowance: the limit is keyed on
|
||||||
|
// the client address clientIP resolves, not on the proxy's.
|
||||||
|
func TestRateLimitIsPerForwardedClient(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const otherClient = "203.0.113.8"
|
||||||
|
|
||||||
|
mw := middleware.NewWithTrustedProxies(mustPrefixes(t, "127.0.0.1/32"))
|
||||||
|
handler := mw.RateLimit(1)(okHandler())
|
||||||
|
|
||||||
|
post := func(client string) int {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(),
|
||||||
|
http.MethodPost, "/api/v1/reports", http.NoBody)
|
||||||
|
req.RemoteAddr = loopbackPeer
|
||||||
|
req.Header.Set("X-Forwarded-For", client)
|
||||||
|
handler.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
return rec.Code
|
||||||
|
}
|
||||||
|
|
||||||
|
if code := post(forwardedIP); code != http.StatusOK {
|
||||||
|
t.Fatalf("first request: status = %d, want %d", code, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
if code := post(forwardedIP); code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("same client again: status = %d, want %d",
|
||||||
|
code, http.StatusTooManyRequests)
|
||||||
|
}
|
||||||
|
|
||||||
|
if code := post(otherClient); code != http.StatusOK {
|
||||||
|
t.Fatalf("other client behind the same proxy: status = %d, want %d",
|
||||||
|
code, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// preflight sends cors the preflight request a browser makes before
|
||||||
|
// it POSTs JSON from origin.
|
||||||
|
func preflight(
|
||||||
|
t *testing.T,
|
||||||
|
cors func(http.Handler) http.Handler,
|
||||||
|
origin string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(),
|
||||||
|
http.MethodOptions, "/api/v1/reports", http.NoBody)
|
||||||
|
req.Header.Set("Origin", origin)
|
||||||
|
req.Header.Set("Access-Control-Request-Method", http.MethodPost)
|
||||||
|
req.Header.Set("Access-Control-Request-Headers", "content-type")
|
||||||
|
cors(okHandler()).ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCORSWithoutOriginsAddsNoHeaders checks the default: with no
|
||||||
|
// origins configured, no origin is given any CORS header.
|
||||||
|
func TestCORSWithoutOriginsAddsNoHeaders(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := preflight(t,
|
||||||
|
(&middleware.Middleware{}).CORS(nil), "https://elsewhere.example")
|
||||||
|
|
||||||
|
for name := range rec.Header() {
|
||||||
|
if strings.HasPrefix(name, "Access-Control-") {
|
||||||
|
t.Errorf("CORS header %s set with no origins configured", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCORSAllowsOnlyListedOrigins(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const listed = "https://netwatch.example"
|
||||||
|
|
||||||
|
cors := (&middleware.Middleware{}).CORS([]string{listed})
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
origin string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "listed origin allowed", origin: listed, want: listed},
|
||||||
|
{name: "other origin refused", origin: "https://elsewhere.example"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rec := preflight(t, cors, tc.origin)
|
||||||
|
|
||||||
|
got := rec.Header().Get("Access-Control-Allow-Origin")
|
||||||
|
if got != tc.want {
|
||||||
|
t.Errorf("Access-Control-Allow-Origin = %q, want %q",
|
||||||
|
got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package reportbuf
|
||||||
|
|
||||||
|
// Flush writes the buffered reports to a file now, as the periodic
|
||||||
|
// flush does, so tests need not wait a minute for it.
|
||||||
|
func (b *Buffer) Flush() error {
|
||||||
|
return b.flushLocked()
|
||||||
|
}
|
||||||
@@ -6,11 +6,13 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -27,8 +29,16 @@ const (
|
|||||||
defaultDataDir = "./data/reports"
|
defaultDataDir = "./data/reports"
|
||||||
dirPerms fs.FileMode = 0o750
|
dirPerms fs.FileMode = 0o750
|
||||||
filePerms fs.FileMode = 0o640
|
filePerms fs.FileMode = 0o640
|
||||||
|
|
||||||
|
// Report files are named filePrefix + timestamp + fileSuffix.
|
||||||
|
filePrefix = "reports-"
|
||||||
|
fileSuffix = ".jsonl.zst"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrFull is returned by Append when storing the report would
|
||||||
|
// take the report files past the configured maximum size.
|
||||||
|
var ErrFull = errors.New("report files at their size cap")
|
||||||
|
|
||||||
// Params defines the dependencies for Buffer.
|
// Params defines the dependencies for Buffer.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -44,8 +54,13 @@ type Buffer struct {
|
|||||||
dataDir string
|
dataDir string
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
|
maxBytes int64
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
stopOnce sync.Once
|
stopOnce sync.Once
|
||||||
|
// usedBytes is what Append checks against maxBytes: the size
|
||||||
|
// of the report files in dataDir, plus the reports not yet
|
||||||
|
// written to one at their uncompressed size.
|
||||||
|
usedBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a Buffer and registers lifecycle hooks to
|
// New creates a Buffer and registers lifecycle hooks to
|
||||||
@@ -60,9 +75,10 @@ func New(
|
|||||||
}
|
}
|
||||||
|
|
||||||
b := &Buffer{
|
b := &Buffer{
|
||||||
dataDir: dir,
|
dataDir: dir,
|
||||||
done: make(chan struct{}),
|
done: make(chan struct{}),
|
||||||
log: params.Logger.Get(),
|
log: params.Logger.Get(),
|
||||||
|
maxBytes: params.Config.DataDirMaxBytes,
|
||||||
}
|
}
|
||||||
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
@@ -72,6 +88,12 @@ func New(
|
|||||||
return fmt.Errorf("create data dir: %w", err)
|
return fmt.Errorf("create data dir: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Report files left by earlier runs count too.
|
||||||
|
b.usedBytes, err = reportFilesSize(b.dataDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
go b.flushLoop()
|
go b.flushLoop()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -97,15 +119,27 @@ func New(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Append marshals v as a single JSON line and appends it to
|
// Append marshals v as a single JSON line and appends it to
|
||||||
// the buffer. If the buffer reaches the size threshold, it is
|
// the buffer. It stores nothing and returns ErrFull if the line
|
||||||
// drained and written to disk asynchronously.
|
// would take usedBytes past maxBytes. If the buffer reaches the
|
||||||
|
// size threshold, it is drained and written to disk
|
||||||
|
// asynchronously.
|
||||||
func (b *Buffer) Append(v any) error {
|
func (b *Buffer) Append(v any) error {
|
||||||
line, err := json.Marshal(v)
|
line, err := json.Marshal(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshal report: %w", err)
|
return fmt.Errorf("marshal report: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lineBytes := int64(len(line)) + 1 // with its newline
|
||||||
|
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
|
|
||||||
|
if b.usedBytes+lineBytes > b.maxBytes {
|
||||||
|
b.mu.Unlock()
|
||||||
|
|
||||||
|
return ErrFull
|
||||||
|
}
|
||||||
|
|
||||||
|
b.usedBytes += lineBytes
|
||||||
b.buf.Write(line)
|
b.buf.Write(line)
|
||||||
b.buf.WriteByte('\n')
|
b.buf.WriteByte('\n')
|
||||||
|
|
||||||
@@ -178,8 +212,7 @@ func (b *Buffer) drainBuf() []byte {
|
|||||||
// in the data directory.
|
// in the data directory.
|
||||||
func (b *Buffer) writeFile(data []byte) error {
|
func (b *Buffer) writeFile(data []byte) error {
|
||||||
ts := time.Now().UTC().Format("2006-01-02T15-04-05.000Z")
|
ts := time.Now().UTC().Format("2006-01-02T15-04-05.000Z")
|
||||||
name := fmt.Sprintf("reports-%s.jsonl.zst", ts)
|
path := filepath.Join(b.dataDir, filePrefix+ts+fileSuffix)
|
||||||
path := filepath.Join(b.dataDir, name)
|
|
||||||
|
|
||||||
// path is built from the operator-supplied dataDir plus a
|
// path is built from the operator-supplied dataDir plus a
|
||||||
// generated timestamp, so it carries no external input.
|
// generated timestamp, so it carries no external input.
|
||||||
@@ -214,10 +247,51 @@ func (b *Buffer) writeFile(data []byte) error {
|
|||||||
return fmt.Errorf("close zstd encoder: %w", err)
|
return fmt.Errorf("close zstd encoder: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
info, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("stat report file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
err = f.Close()
|
err = f.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("close report file: %w", err)
|
return fmt.Errorf("close report file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The reports counted at their uncompressed size while they
|
||||||
|
// waited; now they count as the file. After a failed write they
|
||||||
|
// stay counted as they were, which errs toward refusing reports
|
||||||
|
// early rather than letting the files pass the cap.
|
||||||
|
b.mu.Lock()
|
||||||
|
b.usedBytes += info.Size() - int64(len(data))
|
||||||
|
b.mu.Unlock()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportFilesSize returns the total size of the report files in
|
||||||
|
// dir.
|
||||||
|
func reportFilesSize(dir string) (int64, error) {
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("read data dir: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
name := entry.Name()
|
||||||
|
if !strings.HasPrefix(name, filePrefix) ||
|
||||||
|
!strings.HasSuffix(name, fileSuffix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("stat report file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
total += info.Size()
|
||||||
|
}
|
||||||
|
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
package reportbuf_test
|
package reportbuf_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -94,6 +97,131 @@ func TestFailedFinalFlushFailsStop(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startBuffer starts a Buffer through fx, as main does, with the
|
||||||
|
// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
|
||||||
|
func startBuffer(t *testing.T) *reportbuf.Buffer {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var buf *reportbuf.Buffer
|
||||||
|
|
||||||
|
app := fxtest.New(t,
|
||||||
|
fx.Provide(
|
||||||
|
globals.New,
|
||||||
|
logger.New,
|
||||||
|
config.New,
|
||||||
|
reportbuf.New,
|
||||||
|
),
|
||||||
|
fx.Populate(&buf),
|
||||||
|
)
|
||||||
|
|
||||||
|
app.RequireStart()
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
return buf
|
||||||
|
}
|
||||||
|
|
||||||
|
// lineBytes is what one report takes in the buffer: its JSON and a
|
||||||
|
// newline.
|
||||||
|
func lineBytes(t *testing.T, report any) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
line, err := json.Marshal(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal report: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return len(line) + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppendPastCapIsRefused(t *testing.T) {
|
||||||
|
report := map[string]string{"id": "cap"}
|
||||||
|
|
||||||
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
|
t.Setenv("DATA_DIR_MAX_BYTES", strconv.Itoa(lineBytes(t, report)))
|
||||||
|
|
||||||
|
buf := startBuffer(t)
|
||||||
|
|
||||||
|
err := buf.Append(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("report that fills the cap exactly: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = buf.Append(report)
|
||||||
|
if !errors.Is(err, reportbuf.ErrFull) {
|
||||||
|
t.Fatalf("report past the cap: error = %v, want ErrFull", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCapCountsReportFilesAlreadyInDataDir starts on a data
|
||||||
|
// directory holding a report file from an earlier run, and a file
|
||||||
|
// that is not a report, which must not count.
|
||||||
|
func TestCapCountsReportFilesAlreadyInDataDir(t *testing.T) {
|
||||||
|
const earlierBytes = 100
|
||||||
|
|
||||||
|
report := map[string]string{"id": "cap"}
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
writeBytes(t, filepath.Join(dir, "reports-2026-01-01T00-00-00.000Z.jsonl.zst"),
|
||||||
|
earlierBytes)
|
||||||
|
writeBytes(t, filepath.Join(dir, "notes.txt"), 10*earlierBytes)
|
||||||
|
|
||||||
|
t.Setenv("DATA_DIR", dir)
|
||||||
|
t.Setenv("DATA_DIR_MAX_BYTES",
|
||||||
|
strconv.Itoa(earlierBytes+lineBytes(t, report)))
|
||||||
|
|
||||||
|
buf := startBuffer(t)
|
||||||
|
|
||||||
|
err := buf.Append(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("report that fills the cap exactly: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = buf.Append(report)
|
||||||
|
if !errors.Is(err, reportbuf.ErrFull) {
|
||||||
|
t.Fatalf("report past the cap: error = %v, want ErrFull", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWrittenReportsCountAtFileSize checks that once reports are
|
||||||
|
// written, they count as their compressed file, not their
|
||||||
|
// uncompressed size, which frees room under the cap.
|
||||||
|
func TestWrittenReportsCountAtFileSize(t *testing.T) {
|
||||||
|
// Repetitive, so its file is far smaller than its JSON.
|
||||||
|
report := map[string]string{"id": strings.Repeat("a", 1000)}
|
||||||
|
size := lineBytes(t, report)
|
||||||
|
|
||||||
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
|
// Room for the report twice over only if the first one counts
|
||||||
|
// at its file's size by the time the second arrives.
|
||||||
|
t.Setenv("DATA_DIR_MAX_BYTES", strconv.Itoa(2*size-1))
|
||||||
|
|
||||||
|
buf := startBuffer(t)
|
||||||
|
|
||||||
|
err := buf.Append(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first report: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = buf.Flush()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("flush: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = buf.Append(report)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second report, after the first was written: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeBytes(t *testing.T, path string, n int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
err := os.WriteFile(path, make([]byte, n), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func hasReportFile(t *testing.T, dir string) bool {
|
func hasReportFile(t *testing.T, dir string) bool {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -3,3 +3,9 @@ package server
|
|||||||
// MaxRequestBodyBytes exposes the router-wide body limit to the
|
// MaxRequestBodyBytes exposes the router-wide body limit to the
|
||||||
// external tests.
|
// external tests.
|
||||||
const MaxRequestBodyBytes = maxRequestBodyBytes
|
const MaxRequestBodyBytes = maxRequestBodyBytes
|
||||||
|
|
||||||
|
// ListenAddr exposes the address the server listens on to the
|
||||||
|
// external tests.
|
||||||
|
func (s *Server) ListenAddr() string {
|
||||||
|
return s.newHTTPServer().Addr
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,8 +2,9 @@ package server
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
@@ -27,7 +28,10 @@ const (
|
|||||||
// newHTTPServer constructs the http.Server. It performs no I/O
|
// newHTTPServer constructs the http.Server. It performs no I/O
|
||||||
// and does not start listening.
|
// and does not start listening.
|
||||||
func (s *Server) newHTTPServer() *http.Server {
|
func (s *Server) newHTTPServer() *http.Server {
|
||||||
listenAddr := fmt.Sprintf(":%d", s.params.Config.Port)
|
listenAddr := net.JoinHostPort(
|
||||||
|
s.params.Config.BindAddress,
|
||||||
|
strconv.Itoa(s.params.Config.Port),
|
||||||
|
)
|
||||||
|
|
||||||
return &http.Server{
|
return &http.Server{
|
||||||
Addr: listenAddr,
|
Addr: listenAddr,
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package server_test
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TestListenAddress checks that the server listens on BIND_ADDRESS
|
||||||
|
// and PORT, and on port 8080 on every interface when neither is set.
|
||||||
|
// The container image sets both, to keep the backend on loopback
|
||||||
|
// behind nginx.
|
||||||
|
func TestListenAddress(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
bindAddress string
|
||||||
|
port string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{bindAddress: "", port: "", want: ":8080"},
|
||||||
|
{bindAddress: "127.0.0.1", port: "8081", want: "127.0.0.1:8081"},
|
||||||
|
{bindAddress: "::1", port: "8081", want: "[::1]:8081"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.want, func(t *testing.T) {
|
||||||
|
// t.Setenv rules out t.Parallel.
|
||||||
|
t.Setenv("BIND_ADDRESS", tt.bindAddress)
|
||||||
|
t.Setenv("PORT", tt.port)
|
||||||
|
|
||||||
|
got := newServer(t).ListenAddr()
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("listen address = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,7 +25,7 @@ func (s *Server) SetupRoutes() {
|
|||||||
s.router.Use(middleware.RequestID)
|
s.router.Use(middleware.RequestID)
|
||||||
s.router.Use(s.mw.Logging())
|
s.router.Use(s.mw.Logging())
|
||||||
s.router.Use(s.mw.SecurityHeaders())
|
s.router.Use(s.mw.SecurityHeaders())
|
||||||
s.router.Use(s.mw.CORS())
|
s.router.Use(s.mw.CORS(s.params.Config.CORSAllowedOrigins))
|
||||||
s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
|
s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
|
||||||
s.router.Use(middleware.Timeout(requestTimeout))
|
s.router.Use(middleware.Timeout(requestTimeout))
|
||||||
|
|
||||||
@@ -35,6 +35,7 @@ func (s *Server) SetupRoutes() {
|
|||||||
)
|
)
|
||||||
|
|
||||||
s.router.Route("/api/v1", func(r chi.Router) {
|
s.router.Route("/api/v1", func(r chi.Router) {
|
||||||
r.Post("/reports", s.h.HandleReport())
|
r.With(s.mw.RateLimit(s.params.Config.ReportsPerMinute)).
|
||||||
|
Post("/reports", s.h.HandleReport())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,16 +19,14 @@ import (
|
|||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestHealthCheckRejectsOversizeBody sends the health check, which
|
// newServer builds a Server from the same constructors as main,
|
||||||
// never reads its body, a body one byte over the limit. Only the
|
// configured from the environment. It is never started, so nothing
|
||||||
// router-wide body limit can reject it.
|
// listens.
|
||||||
func TestHealthCheckRejectsOversizeBody(t *testing.T) {
|
func newServer(t *testing.T) *server.Server {
|
||||||
t.Parallel()
|
t.Helper()
|
||||||
|
|
||||||
var srv *server.Server
|
var srv *server.Server
|
||||||
|
|
||||||
// The same constructors as main, never started: SetupRoutes is
|
|
||||||
// called directly, so nothing listens.
|
|
||||||
app := fxtest.New(t,
|
app := fxtest.New(t,
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
config.New,
|
config.New,
|
||||||
@@ -48,6 +46,48 @@ func TestHealthCheckRejectsOversizeBody(t *testing.T) {
|
|||||||
t.Fatalf("build server: %v", err)
|
t.Fatalf("build server: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReportsAreRateLimited checks that POST /api/v1/reports is
|
||||||
|
// behind the per-address rate limit, set here to two a minute.
|
||||||
|
func TestReportsAreRateLimited(t *testing.T) {
|
||||||
|
t.Setenv("REPORTS_PER_MINUTE", "2")
|
||||||
|
|
||||||
|
srv := newServer(t)
|
||||||
|
srv.SetupRoutes()
|
||||||
|
|
||||||
|
post := func() int {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(),
|
||||||
|
http.MethodPost, "/api/v1/reports",
|
||||||
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
|
)
|
||||||
|
srv.ServeHTTP(rec, req)
|
||||||
|
|
||||||
|
return rec.Code
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range 2 {
|
||||||
|
if code := post(); code != http.StatusOK {
|
||||||
|
t.Fatalf("report %d: status = %d, want %d",
|
||||||
|
i+1, code, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if code := post(); code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("third report in a minute: status = %d, want %d",
|
||||||
|
code, http.StatusTooManyRequests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHealthCheckRejectsOversizeBody sends the health check, which
|
||||||
|
// never reads its body, a body one byte over the limit. Only the
|
||||||
|
// router-wide body limit can reject it.
|
||||||
|
func TestHealthCheckRejectsOversizeBody(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
srv := newServer(t)
|
||||||
srv.SetupRoutes()
|
srv.SetupRoutes()
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
# VERSION comes from the environment (Dockerfile.backend passes its
|
# VERSION comes from the environment (the root Dockerfile passes its
|
||||||
# ARG VERSION in). Unset or empty, it is git describe, or "dev" where
|
# ARG VERSION in). Unset or empty, it is git describe, or "dev" where
|
||||||
# there is no git or no repository history.
|
# there is no git or no repository history.
|
||||||
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
|
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/lint: run golangci-lint over the backend. This runs inside the
|
# script/lint: run golangci-lint over the backend. This runs inside the
|
||||||
# lint stage of Dockerfile.backend, whose digest-pinned golangci-lint
|
# lint stage of the root Dockerfile, whose digest-pinned golangci-lint
|
||||||
# image provides the linter; nothing installs golangci-lint on the host.
|
# image provides the linter; nothing installs golangci-lint on the host.
|
||||||
# From a checkout, run `make lint` at the repo root, which builds that
|
# From a checkout, run `make lint` at the repo root, which builds that
|
||||||
# stage.
|
# stage.
|
||||||
|
|||||||
Executable
+59
@@ -0,0 +1,59 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# The container's entrypoint: runs netwatch-server and nginx side by
|
||||||
|
# side. TERM or INT stops both, and the container exits 0 if both exit
|
||||||
|
# cleanly. If either exits on its own, the other is stopped too and the
|
||||||
|
# container exits non-zero, so the platform restarts it instead of
|
||||||
|
# leaving it half up.
|
||||||
|
#
|
||||||
|
# No set -e: kill and wait return non-zero here in normal operation.
|
||||||
|
set -u
|
||||||
|
|
||||||
|
# A stop signal is only noted here; the loop below acts on it.
|
||||||
|
stop_requested=""
|
||||||
|
trap 'stop_requested=yes' TERM INT
|
||||||
|
|
||||||
|
# netwatch-server runs as the netwatch user and listens on loopback
|
||||||
|
# only, on a port other than the public one; nginx.conf proxies to this
|
||||||
|
# address. The netwatch user has no login shell, hence -s /bin/sh.
|
||||||
|
# busybox su replaces itself with the command instead of staying on as
|
||||||
|
# its parent, so $! is the server's own PID.
|
||||||
|
BIND_ADDRESS=127.0.0.1 PORT=8081 \
|
||||||
|
su -s /bin/sh netwatch -c 'exec netwatch-server' &
|
||||||
|
backend=$!
|
||||||
|
|
||||||
|
# nginx starts through the nginx image's own entrypoint, which applies
|
||||||
|
# the image's start-up configuration and then replaces itself with
|
||||||
|
# nginx.
|
||||||
|
/docker-entrypoint.sh nginx -g 'daemon off;' &
|
||||||
|
nginx=$!
|
||||||
|
|
||||||
|
running() {
|
||||||
|
kill -0 "$1" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# POSIX sh cannot wait for whichever of two children exits first, so
|
||||||
|
# look once a second. The shell collects a child that has exited while
|
||||||
|
# it runs sleep, and running() is false for that child from then on.
|
||||||
|
while [ -z "$stop_requested" ] && running "$backend" && running "$nginx"; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
# Stop both, then wait until neither is left.
|
||||||
|
kill -TERM "$backend" "$nginx" 2>/dev/null
|
||||||
|
while running "$backend" || running "$nginx"; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
wait "$backend"
|
||||||
|
backend_status=$?
|
||||||
|
wait "$nginx"
|
||||||
|
nginx_status=$?
|
||||||
|
echo "entrypoint: netwatch-server exited $backend_status," \
|
||||||
|
"nginx exited $nginx_status"
|
||||||
|
|
||||||
|
# Success is a requested stop that both processes exited cleanly from.
|
||||||
|
if [ -n "$stop_requested" ] && [ "$backend_status" -eq 0 ] &&
|
||||||
|
[ "$nginx_status" -eq 0 ]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
+19
@@ -25,4 +25,23 @@ server {
|
|||||||
expires 1y;
|
expires 1y;
|
||||||
add_header Cache-Control "public, immutable";
|
add_header Cache-Control "public, immutable";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# netwatch-server, the Go backend, runs in the same container and
|
||||||
|
# listens on loopback only: bin/entrypoint.sh starts it on
|
||||||
|
# 127.0.0.1:8081. These headers go with every request passed to it.
|
||||||
|
# X-Forwarded-For carries only the client address, as resolved by
|
||||||
|
# the real IP settings above, and not the chain the request came
|
||||||
|
# with: the backend takes the first entry, which a client can write.
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
location /api/ {
|
||||||
|
proxy_pass http://127.0.0.1:8081;
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /.well-known/healthcheck {
|
||||||
|
proxy_pass http://127.0.0.1:8081;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -27,8 +27,8 @@ NVM_VERSION="0.40.3"
|
|||||||
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
||||||
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
||||||
YARN_VERSION="1.22.22"
|
YARN_VERSION="1.22.22"
|
||||||
# The Go inside the golang:1.25-alpine image Dockerfile.backend builds
|
# The Go inside the golang:1.25-alpine image Dockerfile builds the
|
||||||
# with, 2026-08-09. The archive hashes are in ensure_go.
|
# backend with, 2026-08-09. The archive hashes are in ensure_go.
|
||||||
GO_VERSION="1.25.7"
|
GO_VERSION="1.25.7"
|
||||||
|
|
||||||
BIN_DIR="$HOME/.local/bin"
|
BIN_DIR="$HOME/.local/bin"
|
||||||
|
|||||||
+4
-6
@@ -1,9 +1,8 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/cibuild: run the CI build. It builds both images: the frontend
|
# script/cibuild: run the CI build: build the one image from Dockerfile,
|
||||||
# from Dockerfile and the backend from Dockerfile.backend. Each runs its
|
# whose stages run the checks as build steps (the backend's fmt-check,
|
||||||
# half of the checks as build steps, so a successful cibuild implies the
|
# lint and tests, and the frontend's test, lint and fmt-check). This is
|
||||||
# whole repo is green. This is the only build step the Gitea workflow
|
# the only build step the Gitea workflow runs.
|
||||||
# runs.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -11,7 +10,6 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
timeout 300 docker build .
|
timeout 300 docker build .
|
||||||
timeout 300 docker build -f Dockerfile.backend .
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+14
-6
@@ -1,7 +1,8 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/docker: build both Docker images, tagged with the project name
|
# script/docker: build the Docker image tagged with the project name.
|
||||||
# from script/projectname: the frontend as <name>, from Dockerfile, and
|
# Identical in all repos; the tag comes from script/projectname.
|
||||||
# the backend as <name>-server, from Dockerfile.backend.
|
# --no-cache because the gate phases the final stage depends on are RUN
|
||||||
|
# steps, and a cached one is a check that did not run.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
@@ -9,9 +10,16 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
name="$("$SCRIPT_DIR/projectname")"
|
# Own line: a failing command substitution inside an argument does
|
||||||
timeout 300 docker build -t "$name" .
|
# not trip `set -e`, so the inline form degrades silently to an
|
||||||
timeout 300 docker build -t "$name-server" -f Dockerfile.backend .
|
# empty constant. VERSION is computed here because .dockerignore
|
||||||
|
# excludes .git, so `git describe` in a build stage yields an empty
|
||||||
|
# version without failing.
|
||||||
|
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||||
|
[ -n "$version" ] || version="unknown"
|
||||||
|
docker build --no-cache \
|
||||||
|
--build-arg VERSION="$version" \
|
||||||
|
-t "$("$SCRIPT_DIR/projectname")" .
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/frontend-check: run the frontend half of the checks only (test,
|
# script/frontend-check: run the frontend half of the checks only (test,
|
||||||
# lint, fmt-check). This exists for the frontend Dockerfile, whose build
|
# lint, fmt-check). This exists for the frontend stage of Dockerfile, a
|
||||||
# stage is a node image with neither Go nor Docker; the backend half is
|
# node image with neither Go nor Docker; the Dockerfile's lint and
|
||||||
# gated by Dockerfile.backend. Everywhere else, use script/check, which
|
# backend build stages gate the backend half. Everywhere else, use
|
||||||
# covers the whole repo. Must not modify any files.
|
# script/check, which covers the whole repo. Must not modify any files.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|||||||
+3
-3
@@ -3,8 +3,8 @@
|
|||||||
# Go linter over backend/.
|
# Go linter over backend/.
|
||||||
#
|
#
|
||||||
# The Go linter runs only in Docker: this builds the lint stage of
|
# The Go linter runs only in Docker: this builds the lint stage of
|
||||||
# Dockerfile.backend, the digest-pinned golangci-lint image, which runs
|
# Dockerfile, the digest-pinned golangci-lint image, which runs the
|
||||||
# the backend's fmt-check and lint targets. --no-cache makes the linter
|
# backend's fmt-check and lint targets. --no-cache makes the linter
|
||||||
# really run every time rather than reuse an earlier result, and the
|
# really run every time rather than reuse an earlier result, and the
|
||||||
# stage is built for its checks alone, so no image is kept.
|
# stage is built for its checks alone, so no image is kept.
|
||||||
set -eu
|
set -eu
|
||||||
@@ -15,7 +15,7 @@ main() {
|
|||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
"$ROOT/script/frontend-lint"
|
"$ROOT/script/frontend-lint"
|
||||||
timeout 300 docker build --no-cache --target lint \
|
timeout 300 docker build --no-cache --target lint \
|
||||||
--output type=cacheonly -f Dockerfile.backend .
|
--output type=cacheonly .
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user