Compare commits
2
Commits
521cdcb343
...
d3d3b47cf4
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
d3d3b47cf4 | ||
 clawbot
|
406e073aee |
+4
-4
@@ -72,10 +72,10 @@
|
||||
**/*.sublime-*
|
||||
|
||||
# This repository's own entries, after the shared content above: the
|
||||
# frontend build, the viewport test's output, what `make build` and
|
||||
# `make run` in backend/ leave behind, and log files at the root.
|
||||
# frontend build, the viewport test's output, the binary `make build`
|
||||
# writes and the reports `make run` stores, and log files at the root.
|
||||
/dist
|
||||
/tmp
|
||||
/backend/netwatch-server
|
||||
/backend/data
|
||||
/netwatch-server
|
||||
/data
|
||||
/*.log
|
||||
|
||||
+4
-1
@@ -51,7 +51,10 @@ node_modules/
|
||||
# Build output
|
||||
dist/
|
||||
tmp/
|
||||
/backend/netwatch-server
|
||||
/netwatch-server
|
||||
|
||||
# The reports make run stores, in netwatch-server's default DATA_DIR
|
||||
/data/
|
||||
|
||||
# Go test binaries and coverage output
|
||||
*.test
|
||||
|
||||
+14
-13
@@ -21,15 +21,15 @@ RUN yarn install --frozen-lockfile
|
||||
COPY . .
|
||||
RUN yarn eslint .
|
||||
|
||||
# Lint phase: golangci-lint over the backend with backend/.golangci.yml,
|
||||
# Lint phase: golangci-lint over the Go packages with .golangci.yml,
|
||||
# and eslint through the copy from frontend-lint at the end. The
|
||||
# golangci/golangci-lint image ships Go and the linter.
|
||||
# golangci/golangci-lint:v2.14.0, 2026-09-24
|
||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||
WORKDIR /src
|
||||
COPY backend/go.mod backend/go.sum ./
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY backend/ .
|
||||
COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
# Nothing is wanted from frontend-lint; the copy is what makes this
|
||||
# phase run it.
|
||||
@@ -69,7 +69,7 @@ RUN yarn build
|
||||
# golang:1.25.7-trixie, 2026-10-07
|
||||
FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test
|
||||
WORKDIR /src
|
||||
COPY backend/ .
|
||||
COPY . .
|
||||
RUN --mount=type=tmpfs,target=/go/pkg/mod \
|
||||
--mount=type=tmpfs,target=/root/.cache/go-build \
|
||||
go test -timeout 90s -race -cover ./... || \
|
||||
@@ -90,13 +90,12 @@ RUN apk add --no-cache git
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
WORKDIR /src
|
||||
COPY backend/go.mod backend/go.sum backend/
|
||||
RUN cd backend && go mod download
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
|
||||
# backend/script/build is the one definition of the build command:
|
||||
# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=..."
|
||||
# It reads VERSION from the environment.
|
||||
# The same go build as in script/build, which builds the binary on the
|
||||
# host.
|
||||
#
|
||||
# The version is the VERSION build argument when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context: the
|
||||
@@ -112,7 +111,9 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
VERSION="$version" backend/script/build
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w -X main.Version=$version" \
|
||||
-o netwatch-server ./cmd/netwatch-server/
|
||||
|
||||
# Runtime stage, and the last one: a plain `docker build .` builds it
|
||||
# and the stages it copies from, the two phases included.
|
||||
@@ -128,10 +129,10 @@ RUN addgroup -g 1000 -S netwatch && \
|
||||
# At start-up the nginx image renders every template here into
|
||||
# conf.d; bin/entrypoint.sh says how.
|
||||
RUN rm /etc/nginx/conf.d/default.conf
|
||||
COPY nginx.conf /etc/nginx/templates/netwatch.conf.template
|
||||
COPY security-headers.conf /etc/nginx/security-headers.conf
|
||||
COPY configs/nginx.conf /etc/nginx/templates/netwatch.conf.template
|
||||
COPY configs/security-headers.conf /etc/nginx/security-headers.conf
|
||||
COPY --from=frontend /app/dist /usr/share/nginx/html
|
||||
COPY --from=builder /src/backend/netwatch-server /usr/local/bin/netwatch-server
|
||||
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
|
||||
COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
|
||||
# bin/entrypoint.sh creates DATA_DIR at start and gives it and /data to
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
.PHONY: bootstrap setup dev build test lint fmt fmt-check check \
|
||||
.PHONY: bootstrap setup dev build run clean test lint fmt fmt-check check \
|
||||
add-dependency tidy frontend-viewport-test docker hooks
|
||||
|
||||
# Standard targets are thin shims; the implementations live in script/
|
||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||
# of README.md). test, lint, fmt, fmt-check and check cover the whole
|
||||
# repo: the frontend here and the Go backend in backend/.
|
||||
# repo: the frontend in web/ and the Go backend in cmd/ and internal/.
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
@@ -15,10 +15,17 @@ setup:
|
||||
dev:
|
||||
@script/dev
|
||||
|
||||
# The frontend only; backend/Makefile's build target builds the Go server.
|
||||
# The frontend into dist/ and the netwatch-server binary.
|
||||
build:
|
||||
@script/build
|
||||
|
||||
# Builds, then runs netwatch-server on port 8080.
|
||||
run:
|
||||
@script/run
|
||||
|
||||
clean:
|
||||
@script/clean
|
||||
|
||||
test:
|
||||
@script/test
|
||||
|
||||
|
||||
@@ -13,10 +13,13 @@ make setup
|
||||
# Run the page on the Vite dev server
|
||||
make dev
|
||||
|
||||
# Build netwatch-server and run it on port 8080
|
||||
make run
|
||||
|
||||
# Run the tests, both linters and the format check
|
||||
make check
|
||||
|
||||
# Build the page into dist/
|
||||
# Build the page into dist/ and netwatch-server into the repo root
|
||||
make build
|
||||
|
||||
# Build the image and run it
|
||||
@@ -25,9 +28,8 @@ docker run -p 8080:8080 netwatch
|
||||
```
|
||||
|
||||
`make check` and `make docker` need Docker. `make dev` passes `/api` to
|
||||
`http://127.0.0.1:8080`, where `make run` in `backend/` starts `netwatch-server`
|
||||
with its defaults, so the reports the page posts are stored in
|
||||
`backend/data/reports`.
|
||||
`http://127.0.0.1:8080`, where `make run` starts `netwatch-server` with its
|
||||
defaults, so the reports the page posts are stored in `data/reports`.
|
||||
|
||||
## Entrypoints
|
||||
|
||||
@@ -35,49 +37,51 @@ This repository adheres to the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
standard: normalized scripts in `script/` are the entrypoints for the
|
||||
development workflow, and the Makefile targets are thin shims that call them.
|
||||
The Go backend in `backend/` has its own `script/` directory and shim Makefile
|
||||
(see [backend/README.md](backend/README.md)). The root scripts cover both
|
||||
halves, so the root `make check` fails if either one is broken. We provide:
|
||||
The scripts cover both the frontend and the Go backend, so `make check` fails if
|
||||
either one is broken. We provide:
|
||||
|
||||
- `script/bootstrap` — install all dependencies (the pinned node via nvm unless
|
||||
one new enough for the frontend's dependencies is installed, yarn via
|
||||
corepack, `yarn install --frozen-lockfile`, Go `GO_VERSION` unless the `go` on
|
||||
`PATH` is exactly that version, the Go modules, and gcc with the C library
|
||||
headers unless gcc is installed, for the race detector in `make test` in
|
||||
`backend/`), linking what it installs itself into `~/.local/bin`, which has to
|
||||
be on `PATH`. It installs no Go linter and not Docker: `make test` and
|
||||
`make lint` run in Docker
|
||||
`PATH` is exactly that version, and the Go modules), linking what it installs
|
||||
itself into `~/.local/bin`, which has to be on `PATH`. It installs no Go
|
||||
linter and not Docker: `make test` and `make lint` run in Docker
|
||||
- `script/setup` — make a fresh clone ready for development: bootstrap plus the
|
||||
git pre-commit hook
|
||||
- `script/dev` — run the Vite dev server, which proxies `/api` to a locally
|
||||
running `netwatch-server`
|
||||
- `script/build` — build the frontend for production into `dist/`;
|
||||
`backend/script/build` builds the Go server
|
||||
- `script/build` — build the frontend for production into `dist/`, then the
|
||||
static `netwatch-server` binary into the repo root with its version stamped
|
||||
in: `VERSION` from the environment, or when that is unset or empty,
|
||||
`git describe` inside a git checkout, then `dev`
|
||||
- `script/run` — build with `script/build`, then run `netwatch-server` with its
|
||||
defaults (see [Settings](#settings))
|
||||
- `script/clean` — remove what `script/build` writes
|
||||
- `script/projectname` — print the project name (used for the Docker image tag)
|
||||
- `script/test` — build the `test` phase of `Dockerfile` without the cache: the
|
||||
frontend's unit tests and production build in its `frontend` stage, and the
|
||||
backend's Go tests with the race detector and coverage
|
||||
- `script/lint` — build the `lint` phase of `Dockerfile` without the cache:
|
||||
eslint in its `frontend-lint` stage, and golangci-lint over `backend/`
|
||||
eslint in its `frontend-lint` stage, and golangci-lint over the Go packages
|
||||
- `script/fmt` — format all files (writes): prettier over the JavaScript, CSS,
|
||||
HTML and Markdown, then gofmt over `backend/`. It runs on the host, as
|
||||
`script/fmt-check` does, with `~/.local/bin` put on `PATH`
|
||||
HTML and Markdown, then gofmt over `cmd/` and `internal/`. It runs on the
|
||||
host, as `script/fmt-check` does, with `~/.local/bin` put on `PATH`
|
||||
- `script/fmt-check` — check formatting (read-only): prettier, then gofmt
|
||||
- `script/check` — run test, lint, and fmt-check
|
||||
- `script/add-dependency` — add a frontend package, or move one to another
|
||||
version: `make add-dependency PACKAGE=<name>@<version>` runs `yarn add --dev`,
|
||||
which changes `package.json` and `yarn.lock` together, then
|
||||
`yarn install --frozen-lockfile`
|
||||
- `script/tidy` — run `go mod tidy` in `backend/`: to add a Go module, import it
|
||||
and run `make tidy`; to move one to another version, edit its `require` line
|
||||
in `backend/go.mod`, then run `make tidy`
|
||||
- `script/tidy` — run `go mod tidy`: to add a Go module, import it and run
|
||||
`make tidy`; to move one to another version, edit its `require` line in
|
||||
`go.mod`, then run `make tidy`
|
||||
- `script/frontend-test` — run the unit tests in `test/unit/` on the host with
|
||||
Node's built-in test runner, through the `test` script in `package.json`, and
|
||||
if any fails, run them again listing every test, and fail; then the production
|
||||
build. Each test run has a 90-second timeout. `make test` runs the same in
|
||||
Docker
|
||||
- `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with
|
||||
prettier (writes), the markdown in `backend/` included
|
||||
prettier (writes)
|
||||
- `script/frontend-fmt-check` — check prettier formatting (read-only)
|
||||
- `script/frontend-viewport-test` — responsive-layout verification of the built
|
||||
frontend in a containerised headless Chrome (see
|
||||
@@ -93,11 +97,11 @@ halves, so the root `make check` fails if either one is broken. We provide:
|
||||
## Responsive layout
|
||||
|
||||
The narrow-viewport layout lives in the `max-width: 768px` media block in
|
||||
`src/styles.css`. It is verified automatically by `make frontend-viewport-test`,
|
||||
which drives a digest-pinned headless Chrome against the built `dist/` and
|
||||
asserts on computed layout at widths derived from that CSS — on every breakpoint
|
||||
it declares and one pixel either side of it, plus a 320px floor, a desktop
|
||||
baseline and two landscape sizes. See
|
||||
`web/src/styles.css`. It is verified automatically by
|
||||
`make frontend-viewport-test`, which drives a digest-pinned headless Chrome
|
||||
against the built `dist/` and asserts on computed layout at widths derived from
|
||||
that CSS — on every breakpoint it declares and one pixel either side of it, plus
|
||||
a 320px floor, a desktop baseline and two landscape sizes. See
|
||||
[test/viewport/README.md](test/viewport/README.md) for what it covers and what
|
||||
it genuinely cannot.
|
||||
|
||||
@@ -112,8 +116,11 @@ the same and nothing is stored.
|
||||
|
||||
## Design
|
||||
|
||||
The page's source is in `web/`, the Go backend's in `cmd/` and `internal/`, and
|
||||
nginx's configuration in `configs/`.
|
||||
|
||||
The page is built with Vite and Tailwind CSS v4. Its code is all in
|
||||
`src/main.js`, with a class-based architecture:
|
||||
`web/src/main.js`, with a class-based architecture:
|
||||
|
||||
- **`CONFIG`**: Configuration object (update interval, timeouts, axis ticks,
|
||||
etc.). The interval menu sets `updateInterval`, the one value the page writes
|
||||
@@ -148,10 +155,10 @@ probing. The report-building step is a pure function of host state.
|
||||
|
||||
### Backend
|
||||
|
||||
`netwatch-server`, in `backend/`, is a small Go HTTP server that stores the
|
||||
reports the page posts. It keeps them in memory and writes them to `DATA_DIR` as
|
||||
zstd-compressed files of JSON lines: every minute, whenever 10 MiB are waiting,
|
||||
and when it stops. Its routes:
|
||||
`netwatch-server` is a small Go HTTP server that stores the reports the page
|
||||
posts. It keeps them in memory and writes them to `DATA_DIR` as zstd-compressed
|
||||
files of JSON lines: every minute, whenever 10 MiB are waiting, and when it
|
||||
stops. Its routes:
|
||||
|
||||
- `POST /api/v1/reports` — takes a report, without credentials; each client
|
||||
address may send a limited number a minute, and the report files are capped in
|
||||
@@ -162,11 +169,139 @@ and when it stops. Its routes:
|
||||
`METRICS_USERNAME` and `METRICS_PASSWORD` are set; each client address may
|
||||
make a limited number of requests to it a minute
|
||||
|
||||
It is an `fx`-wired Go application: `cmd/netwatch-server/` starts it, and its
|
||||
packages are in `internal/`:
|
||||
|
||||
- **`config`**: Loads configuration from environment variables and config files
|
||||
via Viper.
|
||||
- **`handlers`**: HTTP request handlers for the API (health check, report
|
||||
ingestion).
|
||||
- **`reportbuf`**: The in-memory buffer of reports and the files it writes them
|
||||
to.
|
||||
- **`server`**: Chi-based HTTP server with middleware wiring and route
|
||||
registration.
|
||||
- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting
|
||||
infrastructure.
|
||||
|
||||
In the image, the `test` phase of `Dockerfile` tests it, the `builder` stage
|
||||
builds it with `backend/script/build`, and `bin/entrypoint.sh` runs it as user
|
||||
`netwatch` on `127.0.0.1:8081`, behind nginx. Outside the image, `make run` in
|
||||
`backend/` builds it and runs it on port 8080. Its settings, report storage and
|
||||
limits are in [backend/README.md](backend/README.md).
|
||||
builds it, and `bin/entrypoint.sh` runs it as user `netwatch` on
|
||||
`127.0.0.1:8081`, behind nginx (see [Deployment](#deployment)). Outside the
|
||||
image, `make run` builds it and runs it on port 8080.
|
||||
|
||||
#### Settings
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||
| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface |
|
||||
| `PORT` | `8080` | HTTP listen port |
|
||||
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
||||
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) |
|
||||
| `DEBUG` | `false` | Enable debug logging |
|
||||
| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
||||
| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) |
|
||||
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
||||
| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) |
|
||||
| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) |
|
||||
| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) |
|
||||
|
||||
`TRUSTED_PROXIES` defaults to
|
||||
`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback
|
||||
entries cover a reverse proxy on the same host. A request whose direct peer is
|
||||
outside this set has its forwarded headers ignored, and the direct peer is
|
||||
logged and rate-limited instead. The container image does not use this default;
|
||||
see [Deployment](#deployment).
|
||||
|
||||
A variable set to a value the server cannot use, such as `PORT=abc`,
|
||||
`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from
|
||||
starting, with an error naming the variable. An empty variable counts as unset.
|
||||
|
||||
#### Report storage
|
||||
|
||||
Reports are written as `reports-<timestamp>-<number>.jsonl.zst` files in
|
||||
`DATA_DIR`. The timestamp is in UTC to the millisecond, so the names sort by
|
||||
time. The number starts at 1 when the server starts and goes up by one for each
|
||||
file the server starts to write, so two files written in the same millisecond
|
||||
still get different names. A failed write uses up its number and leaves a gap in
|
||||
the numbers: its file, if it was created, is removed. The file stays, counted
|
||||
toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too.
|
||||
Files are created with `O_EXCL` to prevent overwrites.
|
||||
|
||||
#### Report limits
|
||||
|
||||
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
||||
credentials, so it is bounded instead. Each refusal below answers with the same
|
||||
`{"status":"error"}` body as any other error.
|
||||
|
||||
- **JSON only.** A report must be sent with `Content-Type: application/json`,
|
||||
parameters such as `charset` allowed, as the page sends it; any other is
|
||||
refused with 415, and nothing of it is stored. A page on another origin can
|
||||
send a `text/plain` POST, or a form, without a CORS preflight; refusing them
|
||||
keeps any site from making its visitors' browsers store reports (see
|
||||
[CORS](#cors)).
|
||||
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
||||
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
||||
`Retry-After: 60`. The minute slides: reports from the minute before still
|
||||
count, fading out over the current one, so an address is sure never to be
|
||||
refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60
|
||||
seconds. The page sends one report a minute from each open tab, so the default
|
||||
of 60 refuses nothing from up to 30 tabs behind one address, such as a
|
||||
household or an office sharing it, however their reports bunch up. Report
|
||||
responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and
|
||||
`X-RateLimit-Reset` headers.
|
||||
- **Size cap.** The report files in `DATA_DIR` may total at most
|
||||
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
|
||||
waiting in memory count at their uncompressed size until they are written;
|
||||
those lost to a failed write stop counting, and the part of its file written
|
||||
is removed. When a report would take the total past the cap, the oldest report
|
||||
files are deleted to make room, and each deletion is logged with the file's
|
||||
name and size; a file still being written is never deleted. A report is
|
||||
refused with 507, and nothing of it is stored, only when the reports waiting
|
||||
to be written fill the cap on their own, and then no file is deleted. At
|
||||
start, report files past the cap, as after lowering it, are deleted the same
|
||||
way. So the cap is how much of the newest reports is kept: the default of 1
|
||||
GiB is small enough for any host; set it to the space you can give `DATA_DIR`.
|
||||
|
||||
#### CORS
|
||||
|
||||
The page calls the API from the origin it is served from, so by default the
|
||||
server sends no CORS headers, and browsers let no other origin's pages call it.
|
||||
To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS`
|
||||
(for example `https://netwatch.example.com`); pages from a listed origin may
|
||||
`GET` and `POST` with a `Content-Type` header. Each entry must be a plain
|
||||
origin, `scheme://host` with an optional `:port`, as browsers send it: no path,
|
||||
not even a trailing `/`, and no `*`. Any other entry stops the server from
|
||||
starting, with an error naming `CORS_ALLOWED_ORIGINS`.
|
||||
|
||||
#### Metrics
|
||||
|
||||
With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves
|
||||
Prometheus metrics at `GET /metrics` to requests with those as their basic auth
|
||||
credentials, and answers any other with 401. For each request that reaches the
|
||||
health check or `POST /api/v1/reports`, those the rate limit refuses included,
|
||||
the metrics record its duration and response size, labelled with its path,
|
||||
method and status; they also count those requests in progress, and include Go's
|
||||
runtime and process metrics. No other request is recorded: not those to
|
||||
`/metrics` itself, and not those answered before they reach either route, such
|
||||
as a CORS preflight, or a request refused with 404 for a path no route has, 405
|
||||
for a method its route does not take, or 413 for declaring a body length over
|
||||
the 1 MiB limit. A report whose body goes over the limit without declaring its
|
||||
length reaches the route, is answered 413 there, and is recorded with that
|
||||
status. Clients can make up any number of paths and methods, and each would add
|
||||
labels to the metrics for as long as the server runs.
|
||||
|
||||
`/metrics` is rate limited, so that its password cannot be guessed quickly: each
|
||||
client address, resolved through `TRUSTED_PROXIES`, may make 60 requests to it a
|
||||
minute, whatever their credentials. Past that it gets 429 with
|
||||
`Retry-After: 60`, and its credentials are not checked. The minute slides as it
|
||||
does for reports (see [Report limits](#report-limits)), so a scraper polling
|
||||
every 2 seconds or less often is never refused. This allowance is apart from the
|
||||
one for reports.
|
||||
|
||||
#### Sentry
|
||||
|
||||
With `SENTRY_DSN` set, each panic in a handler is reported to that Sentry
|
||||
project under the release `netwatch-server-` followed by the server's version.
|
||||
On shutdown the server waits up to 2 seconds for Sentry to finish sending.
|
||||
|
||||
### Monitoring targets
|
||||
|
||||
@@ -268,20 +403,25 @@ behind a reverse proxy.
|
||||
|
||||
The Docker image, built from `Dockerfile` by `make docker`, is the whole service
|
||||
in one container: nginx serves the built frontend and passes `/api/`,
|
||||
`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`,
|
||||
which listens only inside the container, on `127.0.0.1:8081`. The image:
|
||||
`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`.
|
||||
`bin/entrypoint.sh` starts it listening only inside the container, on
|
||||
`127.0.0.1:8081`, with `TRUSTED_PROXIES=127.0.0.1/32`, so it takes the client
|
||||
address nginx passes on and no other. The image:
|
||||
|
||||
- Listens on port 8080 by default (override with `PORT` env var)
|
||||
- Takes the client address from `X-Forwarded-For` only on requests from the
|
||||
reverse proxies named in `TRUSTED_PROXIES`, and by default from none
|
||||
reverse proxies named in `TRUSTED_PROXIES`, and by default from none.
|
||||
`bin/entrypoint.sh` checks each entry with `netwatch-server check-cidr`, which
|
||||
parses it as the backend parses its own `TRUSTED_PROXIES`
|
||||
- Sends access logs to stdout
|
||||
- Caches static assets with immutable headers
|
||||
- Sends the security headers `REPO_POLICIES.md` requires on every response, as
|
||||
`security-headers.conf` sets them, in place of the backend's own
|
||||
`configs/security-headers.conf` sets them, in place of the backend's own
|
||||
- Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data`
|
||||
volume. Before the backend starts, the image creates `DATA_DIR` and gives it
|
||||
and `/data` to user `netwatch` (uid 1000), which the backend runs as, so a
|
||||
host directory bind-mounted at `/data` ends up owned by uid 1000
|
||||
and `/data` to user `netwatch` (uid 1000), which the backend runs as, with
|
||||
`netwatch-server prepare-data-dir`, which acts on nothing outside `/data`, so
|
||||
a host directory bind-mounted at `/data` ends up owned by uid 1000
|
||||
- Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx
|
||||
or the backend exits on its own, so the platform restarts it
|
||||
|
||||
|
||||
@@ -16,12 +16,38 @@ Docker image, and the Gitea workflow `.gitea/workflows/check.yml` runs
|
||||
|
||||
# Next Step
|
||||
|
||||
Decide whether the repo moves to the layout `REPO_POLICIES.md` gives, with
|
||||
`backend/` no longer repeating files from the root
|
||||
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)).
|
||||
Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
||||
`make check`, as it needs Docker and takes minutes.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-14: `POST /api/v1/reports` takes only a report sent with
|
||||
`Content-Type: application/json`, parameters such as `charset` allowed, and
|
||||
refuses any other with 415 and the usual error body, storing nothing
|
||||
([#126](https://git.eeqj.de/sneak/netwatch/issues/126)). A page on another
|
||||
origin can send a `text/plain` POST without a CORS preflight, so any site
|
||||
could make its visitors' browsers store reports; a JSON POST waits for the
|
||||
preflight, which only an origin in `CORS_ALLOWED_ORIGINS` passes. The page
|
||||
already sent `application/json`. A test posts a report as `text/plain`, as
|
||||
`application/x-www-form-urlencoded`, as `multipart/form-data`, with no
|
||||
`Content-Type`, and as `application/json` with and without a charset, and
|
||||
checks which are stored
|
||||
- 2026-10-14: the repo has the layout `REPO_POLICIES.md` gives
|
||||
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). The Go module is at the
|
||||
root (`go.mod`, `go.sum`, `.golangci.yml`, `cmd/netwatch-server/`,
|
||||
`internal/`), so its path `sneak.berlin/go/netwatch` names the repository.
|
||||
`go.mod` ignores `node_modules/`, where npm packages can ship `.go` files that
|
||||
`make tidy` would otherwise read as this module's packages. The page's source
|
||||
is in `web/` (`index.html`, `src/`), which `vite.config.js` makes Vite's root;
|
||||
the build still goes to `dist/`. `nginx.conf` and `security-headers.conf` are
|
||||
in `configs/`. `backend/` is gone: its `LICENSE` and `.dockerignore` are
|
||||
deleted, its `.gitignore` entries are in the root one, its `README.md` is
|
||||
folded into the root `README.md`, and its scripts into the root `script/`:
|
||||
`script/build` builds the server binary after the frontend, `script/run` and
|
||||
`script/clean` came from `backend/script/`, and `make test`, `make lint`,
|
||||
`make fmt` and `make fmt-check` do what its `test`, `lint`, `fmt` and
|
||||
`fmt-check` did. `script/bootstrap` no longer installs gcc, which only the
|
||||
host Go tests needed. The image and the page are unchanged
|
||||
- 2026-10-07: the files shared from `sneak/prompts` are its copies at commit
|
||||
`dd4027b` ([#113](https://git.eeqj.de/sneak/netwatch/issues/113)), with this
|
||||
repository's own entries after the shared content in `.gitignore`,
|
||||
@@ -416,8 +442,6 @@ Decide whether the repo moves to the layout `REPO_POLICIES.md` gives, with
|
||||
|
||||
# Future Steps
|
||||
|
||||
- Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
||||
`make check`, as it needs Docker and takes minutes
|
||||
- A backend test that posts a report to `POST /api/v1/reports` and checks the
|
||||
compressed file it is written to
|
||||
- A backend route that decompresses the stored reports and answers queries on
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
node_modules
|
||||
.DS_Store
|
||||
@@ -1,6 +0,0 @@
|
||||
/netwatch-server
|
||||
*.log
|
||||
*.out
|
||||
*.test
|
||||
.env
|
||||
data/
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 @sneak (https://sneak.berlin)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,30 +0,0 @@
|
||||
# Thin shims; the implementations live in backend/script/ (see the
|
||||
# Entrypoints section of README.md). There is no check, hooks or docker
|
||||
# target here: the root Makefile's check covers this directory, its
|
||||
# hooks target installs the repo's only pre-commit hook, and its docker
|
||||
# target builds the one image, which contains this backend.
|
||||
|
||||
.PHONY: all build test lint fmt fmt-check run clean
|
||||
|
||||
all: build
|
||||
|
||||
build:
|
||||
@script/build
|
||||
|
||||
test:
|
||||
@script/test
|
||||
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
fmt-check:
|
||||
@script/fmt-check
|
||||
|
||||
run:
|
||||
@script/run
|
||||
|
||||
clean:
|
||||
@script/clean
|
||||
@@ -1,232 +0,0 @@
|
||||
netwatch-server is an MIT-licensed Go HTTP backend by
|
||||
[@sneak](https://sneak.berlin) that receives telemetry reports from the NetWatch
|
||||
SPA and persists them as zstd-compressed JSONL files on disk.
|
||||
|
||||
## Getting Started
|
||||
|
||||
From this directory:
|
||||
|
||||
```bash
|
||||
# Build and run locally
|
||||
make run
|
||||
```
|
||||
|
||||
From the repo root, whose `Dockerfile` builds the one image that ships this
|
||||
backend behind nginx (see [Container image](#container-image)):
|
||||
|
||||
```bash
|
||||
# Run tests, lint, and format check over the frontend and this backend
|
||||
make check
|
||||
|
||||
# Build the image: nginx, the frontend and this backend
|
||||
make docker
|
||||
docker run -p 8080:8080 netwatch
|
||||
```
|
||||
|
||||
## Entrypoints
|
||||
|
||||
This directory follows the same
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
pattern as the repo root: the targets in `backend/Makefile` are thin shims over
|
||||
`backend/script/`. The root `Dockerfile` runs `build`, and the root scripts call
|
||||
`fmt` and `fmt-check`:
|
||||
|
||||
- `script/build` — compile the static `netwatch-server` binary with its version
|
||||
stamped in. The version is `VERSION` from the environment; when that is unset
|
||||
or empty, it falls back to `git describe` inside a git checkout, then to `dev`
|
||||
- `script/test` — run the Go tests on the host with the race detector and
|
||||
coverage; the root `make test` runs them in the `test` phase of the root
|
||||
`Dockerfile`. Go's `-timeout 90s` bounds the tests, not their compile, and
|
||||
`-count=1` keeps Go from reporting a stored pass. If they fail, they run again
|
||||
with `-v` for the details, and the script fails. The race detector needs a C
|
||||
compiler
|
||||
- `script/lint` — run the root `script/lint`, which builds the `lint` phase of
|
||||
the root `Dockerfile`: golangci-lint over this directory, and eslint over the
|
||||
frontend. golangci-lint never runs on the host
|
||||
- `script/fmt` — format the Go sources (writes)
|
||||
- `script/fmt-check` — check Go formatting (read-only)
|
||||
- `script/run` — build and run the server locally
|
||||
- `script/clean` — remove build artifacts
|
||||
|
||||
There is no `check`, `hooks` or `docker` target here: the root `make check`
|
||||
covers this directory, the root `make hooks` installs the repo's only pre-commit
|
||||
hook, and the root `make docker` builds the image that contains this backend.
|
||||
|
||||
## Rationale
|
||||
|
||||
The NetWatch frontend collects latency measurements from the browser but has no
|
||||
way to persist or aggregate them. This backend provides a minimal
|
||||
`POST /api/v1/reports` endpoint that buffers incoming reports in memory and
|
||||
flushes them to compressed files on disk for later analysis.
|
||||
|
||||
## Design
|
||||
|
||||
The server is structured as an `fx`-wired Go application under
|
||||
`cmd/netwatch-server/`. Internal packages in `internal/` follow standard Go
|
||||
project layout:
|
||||
|
||||
- **`config`**: Loads configuration from environment variables and config files
|
||||
via Viper.
|
||||
- **`handlers`**: HTTP request handlers for the API (health check, report
|
||||
ingestion).
|
||||
- **`reportbuf`**: In-memory buffer that accumulates JSONL report lines and
|
||||
flushes to zstd-compressed files when the buffer reaches 10 MiB or every 60
|
||||
seconds.
|
||||
- **`server`**: Chi-based HTTP server with middleware wiring and route
|
||||
registration.
|
||||
- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting
|
||||
infrastructure.
|
||||
|
||||
### Configuration
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||
| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface |
|
||||
| `PORT` | `8080` | HTTP listen port |
|
||||
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
||||
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) |
|
||||
| `DEBUG` | `false` | Enable debug logging |
|
||||
| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
||||
| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) |
|
||||
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
||||
| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) |
|
||||
| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) |
|
||||
| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) |
|
||||
|
||||
`TRUSTED_PROXIES` defaults to
|
||||
`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback
|
||||
entries cover a reverse proxy on the same host. A request whose direct peer is
|
||||
outside this set has its forwarded headers ignored, and the direct peer is
|
||||
logged and rate-limited instead. The container image does not use this default;
|
||||
see [Container image](#container-image).
|
||||
|
||||
A variable set to a value the server cannot use, such as `PORT=abc`,
|
||||
`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from
|
||||
starting, with an error naming the variable. An empty variable counts as unset.
|
||||
|
||||
### Container image
|
||||
|
||||
The root `Dockerfile` builds one image in which nginx listens on the public port
|
||||
8080, serves the frontend, and proxies `/api/`, `/.well-known/healthcheck` and
|
||||
`/metrics` to this server. The image's entrypoint, `bin/entrypoint.sh`, starts
|
||||
the server as user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and
|
||||
`PORT=8081`, so only nginx reaches it, and with `TRUSTED_PROXIES=127.0.0.1/32`,
|
||||
so it takes the client address nginx passes on and no other. `DATA_DIR` is
|
||||
`/data/reports`, on the `/data` volume; before starting the server, the
|
||||
entrypoint creates it and gives it and `/data` to `netwatch` with
|
||||
`netwatch-server prepare-data-dir`, which acts on nothing outside `/data`. nginx
|
||||
replaces the security headers this server sets with those in the root
|
||||
`security-headers.conf`, so those are what clients of the image see.
|
||||
|
||||
The container's own `TRUSTED_PROXIES` goes to nginx instead: IP addresses or
|
||||
CIDRs, separated by commas, of the reverse proxies in front of the container.
|
||||
nginx takes the client address from `X-Forwarded-For` only on a request from one
|
||||
of them. Unset or empty, nginx trusts no proxy, and the client address is the
|
||||
one each request comes from, so every client behind a proxy shares one rate
|
||||
limit. An entry that is not an IP address or CIDR, such as a hostname or
|
||||
`1.2.3`, stops the container at start with an error naming `TRUSTED_PROXIES`:
|
||||
the entrypoint checks each entry with `netwatch-server check-cidr`, which parses
|
||||
it as this server parses its own `TRUSTED_PROXIES`.
|
||||
|
||||
### Report storage
|
||||
|
||||
Reports are written as `reports-<timestamp>-<number>.jsonl.zst` files in
|
||||
`DATA_DIR`. The timestamp is in UTC to the millisecond, so the names sort by
|
||||
time. The number starts at 1 when the server starts and goes up by one for each
|
||||
file the server starts to write, so two files written in the same millisecond
|
||||
still get different names. A failed write uses up its number and leaves a gap in
|
||||
the numbers: its file, if it was created, is removed. The file stays, counted
|
||||
toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too.
|
||||
Each file contains one JSON object per line, compressed with zstd. Files are
|
||||
created with `O_EXCL` to prevent overwrites.
|
||||
|
||||
### Report limits
|
||||
|
||||
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
||||
credentials, so it is bounded instead. Both refusals below answer with the same
|
||||
`{"status":"error"}` body as any other error.
|
||||
|
||||
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
||||
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
||||
`Retry-After: 60`. The minute slides: reports from the minute before still
|
||||
count, fading out over the current one, so an address is sure never to be
|
||||
refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60
|
||||
seconds. The page sends one report a minute from each open tab, so the default
|
||||
of 60 refuses nothing from up to 30 tabs behind one address, such as a
|
||||
household or an office sharing it, however their reports bunch up. Report
|
||||
responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and
|
||||
`X-RateLimit-Reset` headers.
|
||||
- **Size cap.** The report files in `DATA_DIR` may total at most
|
||||
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
|
||||
waiting in memory count at their uncompressed size until they are written;
|
||||
those lost to a failed write stop counting, and the part of its file written
|
||||
is removed. When a report would take the total past the cap, the oldest report
|
||||
files are deleted to make room, and each deletion is logged with the file's
|
||||
name and size; a file still being written is never deleted. A report is
|
||||
refused with 507, and nothing of it is stored, only when the reports waiting
|
||||
to be written fill the cap on their own, and then no file is deleted. At
|
||||
start, report files past the cap, as after lowering it, are deleted the same
|
||||
way. So the cap is how much of the newest reports is kept: the default of 1
|
||||
GiB is small enough for any host; set it to the space you can give `DATA_DIR`.
|
||||
|
||||
### CORS
|
||||
|
||||
The page calls the API from the origin it is served from, so by default the
|
||||
server sends no CORS headers, and browsers let no other origin's pages call it.
|
||||
To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS`
|
||||
(for example `https://netwatch.example.com`); pages from a listed origin may
|
||||
`GET` and `POST` with a `Content-Type` header. Each entry must be a plain
|
||||
origin, `scheme://host` with an optional `:port`, as browsers send it: no path,
|
||||
not even a trailing `/`, and no `*`. Any other entry stops the server from
|
||||
starting, with an error naming `CORS_ALLOWED_ORIGINS`.
|
||||
|
||||
### Metrics
|
||||
|
||||
With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves
|
||||
Prometheus metrics at `GET /metrics` to requests with those as their basic auth
|
||||
credentials, and answers any other with 401. For each request that reaches the
|
||||
health check or `POST /api/v1/reports`, those the rate limit refuses included,
|
||||
the metrics record its duration and response size, labelled with its path,
|
||||
method and status; they also count those requests in progress, and include Go's
|
||||
runtime and process metrics. No other request is recorded: not those to
|
||||
`/metrics` itself, and not those answered before they reach either route, such
|
||||
as a CORS preflight, or a request refused with 404 for a path no route has, 405
|
||||
for a method its route does not take, or 413 for declaring a body length over
|
||||
the 1 MiB limit. A report whose body goes over the limit without declaring its
|
||||
length reaches the route, is answered 413 there, and is recorded with that
|
||||
status. Clients can make up any number of paths and methods, and each would add
|
||||
labels to the metrics for as long as the server runs. With neither set, nothing
|
||||
is recorded and `/metrics` answers 404. One without the other stops the server
|
||||
from starting, with an error naming both; so does a `METRICS_USERNAME`
|
||||
containing `:`, which basic auth cannot carry, with an error naming it.
|
||||
|
||||
`/metrics` is rate limited, so that its password cannot be guessed quickly: each
|
||||
client address, resolved through `TRUSTED_PROXIES`, may make 60 requests to it a
|
||||
minute, whatever their credentials. Past that it gets 429 with
|
||||
`Retry-After: 60`, and its credentials are not checked. The minute slides as it
|
||||
does for reports (see [Report limits](#report-limits)), so a scraper polling
|
||||
every 2 seconds or less often is never refused. This allowance is apart from the
|
||||
one for reports.
|
||||
|
||||
### Sentry
|
||||
|
||||
With `SENTRY_DSN` set, the server sends its errors to that Sentry project: each
|
||||
panic in a handler is reported there, under the release `netwatch-server-`
|
||||
followed by the server's version, and the request still gets 500 from the
|
||||
server's panic recovery. On shutdown the server waits up to 2 seconds for Sentry
|
||||
to finish sending. A DSN Sentry refuses stops the server from starting, with an
|
||||
error naming `SENTRY_DSN`. With it empty, Sentry is not set up, and nothing is
|
||||
sent to it.
|
||||
|
||||
## TODO
|
||||
|
||||
The to-do list, this backend's open work included, is [TODO.md](../TODO.md) at
|
||||
the repo root.
|
||||
|
||||
## License
|
||||
|
||||
MIT. See [LICENSE](LICENSE).
|
||||
|
||||
## Author
|
||||
|
||||
[@sneak](https://sneak.berlin)
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/build: compile the static netwatch-server binary into the
|
||||
# backend project root, with its version stamped in.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
# VERSION comes from the environment (the root Dockerfile passes its
|
||||
# ARG VERSION in). Unset or empty, it is git describe, or "dev" where
|
||||
# there is no git or no repository history.
|
||||
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
|
||||
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w -X main.Version=$version" \
|
||||
-o netwatch-server ./cmd/netwatch-server/
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/clean: remove build artifacts.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
rm -f netwatch-server
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/fmt: format the Go sources (writes).
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
go fmt ./...
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/fmt-check: check Go formatting (read-only). Same scope as
|
||||
# script/fmt, but fails instead of writing.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
unformatted="$(gofmt -l .)"
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "Files not formatted:" >&2
|
||||
echo "$unformatted" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/lint: lint the whole repo as the root make lint does, by
|
||||
# building the lint phase of the root Dockerfile. golangci-lint never
|
||||
# runs on the host (REPO_POLICIES.md).
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
exec "$ROOT/script/lint"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/test: run the backend test suite on the host with the race
|
||||
# detector and coverage. The root make test runs the same in the test
|
||||
# phase of the root Dockerfile. Go's own -timeout bounds the tests and
|
||||
# not their compile, so a cold build cache cannot fail it. -count=1
|
||||
# keeps Go's test result cache out of both runs, so neither can report
|
||||
# a stored pass. The race detector needs cgo, and so a C compiler. If
|
||||
# the tests fail, they run again with -v for the details, and the
|
||||
# script fails even if that run passes.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
go test -count=1 -timeout 90s -race -cover ./... || {
|
||||
echo "--- Rerunning with -v for details ---"
|
||||
go test -count=1 -timeout 90s -race -v ./...
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+8
-8
@@ -35,8 +35,8 @@ fi
|
||||
# TRUSTED_PROXIES names the reverse proxies in front of the container,
|
||||
# as IP addresses or CIDRs separated by commas. nginx takes the client
|
||||
# address from X-Forwarded-For only on a request from one of them, so
|
||||
# unset or empty, it trusts no one. nginx.conf includes the file written
|
||||
# here, one set_real_ip_from line per entry.
|
||||
# unset or empty, it trusts no one. configs/nginx.conf includes the file
|
||||
# written here, one set_real_ip_from line per entry.
|
||||
#
|
||||
# nginx looks up an entry it cannot read as an address as a hostname,
|
||||
# and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry
|
||||
@@ -76,10 +76,10 @@ stop_requested=""
|
||||
trap 'stop_requested=yes' TERM INT
|
||||
|
||||
# netwatch-server runs as the netwatch user and listens on loopback
|
||||
# only, on a port other than the public one; nginx.conf proxies to this
|
||||
# address. Its only client is nginx, so it takes the client address
|
||||
# nginx passes on from 127.0.0.1 alone, whatever TRUSTED_PROXIES the
|
||||
# container has. The netwatch user has no login shell, hence -s
|
||||
# only, on a port other than the public one; configs/nginx.conf proxies
|
||||
# to this address. Its only client is nginx, so it takes the client
|
||||
# address nginx passes on from 127.0.0.1 alone, whatever TRUSTED_PROXIES
|
||||
# the container has. The netwatch user has no login shell, hence -s
|
||||
# /bin/sh. busybox su replaces itself with the command instead of
|
||||
# staying on as its parent, so $! is the server's own PID.
|
||||
BIND_ADDRESS=127.0.0.1 PORT=8081 TRUSTED_PROXIES=127.0.0.1/32 \
|
||||
@@ -88,8 +88,8 @@ backend=$!
|
||||
|
||||
# nginx starts through the nginx image's own entrypoint, which applies
|
||||
# the image's start-up configuration and then replaces itself with
|
||||
# nginx. Part of that start-up configuration renders nginx.conf into
|
||||
# conf.d with nginx listening on PORT. NGINX_ENVSUBST_FILTER limits
|
||||
# nginx. Part of that start-up configuration renders configs/nginx.conf
|
||||
# into conf.d with nginx listening on PORT. NGINX_ENVSUBST_FILTER limits
|
||||
# that rendering to PORT: a variable nginx itself uses, such as $uri,
|
||||
# would otherwise be replaced by an environment variable of the same
|
||||
# name.
|
||||
|
||||
@@ -53,7 +53,7 @@ server {
|
||||
|
||||
# netwatch-server sets the same security headers on its own
|
||||
# responses. Its copies are dropped so that each header goes out
|
||||
# once, as security-headers.conf sets it.
|
||||
# once, as configs/security-headers.conf sets it.
|
||||
proxy_hide_header Strict-Transport-Security;
|
||||
proxy_hide_header Content-Security-Policy;
|
||||
proxy_hide_header X-Frame-Options;
|
||||
@@ -1,5 +1,5 @@
|
||||
# The security headers REPO_POLICIES.md requires on every response.
|
||||
# nginx.conf includes this file, which Dockerfile copies to
|
||||
# configs/nginx.conf includes this file, which Dockerfile copies to
|
||||
# /etc/nginx/security-headers.conf. always sends each header on error
|
||||
# responses too.
|
||||
|
||||
@@ -8,11 +8,11 @@ add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" alway
|
||||
# Scripts and styles load only from the page's own origin. Inline ones
|
||||
# are blocked, style attributes in markup included, so style elements
|
||||
# through classes or element.style. data: images are for the favicon
|
||||
# in index.html. connect-src is * because the browser checks each probe in
|
||||
# src/main.js against it, and also every redirect the probe follows,
|
||||
# and several of those hosts redirect to others; a list of hosts here
|
||||
# would block those probes. It also covers the reports the page sends
|
||||
# to its own origin.
|
||||
# in web/index.html. connect-src is * because the browser checks each
|
||||
# probe in web/src/main.js against it, and also every redirect the probe
|
||||
# follows, and several of those hosts redirect to others; a list of
|
||||
# hosts here would block those probes. It also covers the reports the
|
||||
# page sends to its own origin.
|
||||
add_header Content-Security-Policy "default-src 'self'; connect-src *; img-src 'self' data:; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" always;
|
||||
|
||||
add_header X-Frame-Options DENY always;
|
||||
+1
-1
@@ -8,7 +8,7 @@ export default defineConfig([
|
||||
js.configs.recommended,
|
||||
{
|
||||
// The page, and facts.js, which the viewport test runs inside it.
|
||||
files: ["src/**/*.js", "test/viewport/facts.js"],
|
||||
files: ["web/src/**/*.js", "test/viewport/facts.js"],
|
||||
languageOptions: {
|
||||
globals: {
|
||||
...globals.browser,
|
||||
|
||||
@@ -2,6 +2,9 @@ module sneak.berlin/go/netwatch
|
||||
|
||||
go 1.25.5
|
||||
|
||||
// npm packages can ship .go files, which ./... and all would include.
|
||||
ignore ./node_modules
|
||||
|
||||
require (
|
||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||
github.com/getsentry/sentry-go v0.49.0
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
const defaultTrustedProxies = "127.0.0.1/32,::1/128," +
|
||||
"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||
|
||||
// Default limits on stored reports; backend/README.md gives the
|
||||
// reasons for these values.
|
||||
// Default limits on stored reports; README.md gives the reasons for
|
||||
// these values.
|
||||
const (
|
||||
defaultReportsPerMinute = 60
|
||||
defaultDataDirMaxBytes = 1 << 30 // 1 GiB
|
||||
@@ -5,7 +5,9 @@ package handlers
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"mime"
|
||||
"net/http"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
@@ -18,6 +20,10 @@ import (
|
||||
|
||||
const jsonContentType = "application/json; charset=utf-8"
|
||||
|
||||
// errNotJSON is what decodeJSON returns for a request whose
|
||||
// Content-Type is not application/json.
|
||||
var errNotJSON = errors.New("content type is not application/json")
|
||||
|
||||
// reportAppender is the subset of the report buffer the handlers
|
||||
// depend on. Defining it here keeps the storage failure path
|
||||
// exercisable with a stub in tests.
|
||||
@@ -80,14 +86,25 @@ func (s *Handlers) respondJSON(
|
||||
}
|
||||
}
|
||||
|
||||
// decodeJSON decodes the request body into v. The body is
|
||||
// expected to already be bounded by the body-size middleware, so
|
||||
// a caller can distinguish an over-limit body from malformed
|
||||
// JSON by testing the returned error for *http.MaxBytesError.
|
||||
// decodeJSON decodes the request body into v. It returns errNotJSON,
|
||||
// reading nothing, unless the request's Content-Type is
|
||||
// application/json, with or without parameters such as charset. The
|
||||
// body is expected to already be bounded by the body-size middleware,
|
||||
// so a caller can distinguish an over-limit body from malformed JSON
|
||||
// by testing the returned error for *http.MaxBytesError.
|
||||
func (s *Handlers) decodeJSON(
|
||||
_ http.ResponseWriter,
|
||||
r *http.Request,
|
||||
v any,
|
||||
) error {
|
||||
// A page on another origin can POST text/plain or a form without
|
||||
// a CORS preflight, so its visitors' browsers would send it
|
||||
// whatever the CORS settings. A JSON POST waits for the
|
||||
// preflight, so only JSON is taken.
|
||||
mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil || mediaType != "application/json" {
|
||||
return errNotJSON
|
||||
}
|
||||
|
||||
return json.NewDecoder(r.Body).Decode(v)
|
||||
}
|
||||
@@ -30,7 +30,7 @@ type report struct {
|
||||
}
|
||||
|
||||
// HandleReport returns a handler that accepts telemetry
|
||||
// reports from NetWatch clients.
|
||||
// reports from NetWatch clients, sent as application/json.
|
||||
func (s *Handlers) HandleReport() http.HandlerFunc {
|
||||
type response struct {
|
||||
Status string `json:"status"`
|
||||
@@ -66,9 +66,15 @@ func (s *Handlers) HandleReport() http.HandlerFunc {
|
||||
}
|
||||
|
||||
// decodeErrorStatus logs a report decode failure and returns the
|
||||
// status to send: 413 when the body exceeded the size limit,
|
||||
// otherwise 400 for malformed JSON.
|
||||
// status to send: 415 when the report was not sent as JSON, 413 when
|
||||
// the body exceeded the size limit, otherwise 400 for malformed JSON.
|
||||
func (s *Handlers) decodeErrorStatus(err error) int {
|
||||
if errors.Is(err, errNotJSON) {
|
||||
s.log.Warn("report refused: Content-Type is not application/json")
|
||||
|
||||
return http.StatusUnsupportedMediaType
|
||||
}
|
||||
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
s.log.Warn("report body too large", "limit_bytes", tooLarge.Limit)
|
||||
@@ -27,6 +27,17 @@ type stubAppender struct {
|
||||
|
||||
func (s stubAppender) Append(any) error { return s.err }
|
||||
|
||||
// recordingAppender keeps every report the handler stores.
|
||||
type recordingAppender struct {
|
||||
reports []any
|
||||
}
|
||||
|
||||
func (a *recordingAppender) Append(v any) error {
|
||||
a.reports = append(a.reports, v)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTestHandlers(buf stubAppender, out io.Writer) *handlers.Handlers {
|
||||
return handlers.NewForTest(buf, slog.New(slog.NewJSONHandler(out, nil)))
|
||||
}
|
||||
@@ -83,6 +94,7 @@ func TestHandleReportAcceptsValidReports(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(tt.body),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -103,6 +115,89 @@ func TestHandleReportAcceptsValidReports(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleReportStoresOnlyJSON checks that a report is stored only
|
||||
// when sent as application/json, as the page sends it. One sent as
|
||||
// text/plain or as a form, which a page on another origin can send
|
||||
// without a CORS preflight, is refused with 415.
|
||||
func TestHandleReportStoresOnlyJSON(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
contentType string
|
||||
wantStatus int
|
||||
wantStored int
|
||||
}{
|
||||
{
|
||||
name: "text/plain",
|
||||
contentType: "text/plain;charset=UTF-8",
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
wantStored: 0,
|
||||
},
|
||||
{
|
||||
name: "a urlencoded form",
|
||||
contentType: "application/x-www-form-urlencoded",
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
wantStored: 0,
|
||||
},
|
||||
{
|
||||
name: "a multipart form",
|
||||
contentType: "multipart/form-data; boundary=x",
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
wantStored: 0,
|
||||
},
|
||||
{
|
||||
name: "no Content-Type",
|
||||
contentType: "",
|
||||
wantStatus: http.StatusUnsupportedMediaType,
|
||||
wantStored: 0,
|
||||
},
|
||||
{
|
||||
name: "application/json",
|
||||
contentType: "application/json",
|
||||
wantStatus: http.StatusOK,
|
||||
wantStored: 1,
|
||||
},
|
||||
{
|
||||
name: "application/json with a charset",
|
||||
contentType: "application/json;charset=UTF-8",
|
||||
wantStatus: http.StatusOK,
|
||||
wantStored: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var stored recordingAppender
|
||||
|
||||
h := handlers.NewForTest(&stored, slog.New(slog.DiscardHandler))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequestWithContext(t.Context(),
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||
)
|
||||
|
||||
if tt.contentType != "" {
|
||||
req.Header.Set("Content-Type", tt.contentType)
|
||||
}
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != tt.wantStatus {
|
||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||
}
|
||||
|
||||
if len(stored.reports) != tt.wantStored {
|
||||
t.Errorf("reports stored = %d, want %d",
|
||||
len(stored.reports), tt.wantStored)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -113,6 +208,7 @@ func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -138,6 +234,7 @@ func TestHandleReportFullIs507(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -161,6 +258,7 @@ func TestHandleReportMalformedJSONIs400(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{not json`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -185,6 +283,8 @@ func TestHandleReportOversizeIs413(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"clientId":"`+strings.Repeat("x", 200)+`"}`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
// No declared length, so only the middleware's read cap can
|
||||
// stop this body.
|
||||
req.ContentLength = -1
|
||||
@@ -213,6 +313,7 @@ func TestHandleReportDoesNotLogRawGeo(t *testing.T) {
|
||||
`{"clientId":"c1","geo":{"raw":"`+sentinel+`"},"hosts":[]}`,
|
||||
),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -245,6 +346,7 @@ func TestHandleReportLogsClientIDCutToBound(t *testing.T) {
|
||||
`{"clientId":"`+long+`","timestamp":"`+long+`","hosts":[]}`,
|
||||
),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -284,6 +386,7 @@ func TestHandleReportDecodeErrorLogIsBounded(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"hosts":[{"history":[{"t":`+huge+`}]}]}`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
@@ -66,6 +66,7 @@ func TestReportsAreRateLimited(t *testing.T) {
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
return rec.Code
|
||||
+2
-8
@@ -17,7 +17,7 @@
|
||||
# golangci-lint is not installed: make lint runs it in Docker, as make
|
||||
# test runs the tests, and this script does not install Docker either.
|
||||
#
|
||||
# Unlike the org model: Go and gcc for backend/, a newer node for eslint.
|
||||
# Unlike the org model: Go for the backend, a newer node for eslint.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
@@ -272,19 +272,13 @@ main() {
|
||||
|
||||
if missing make; then pkg_install gnumake make make make; fi
|
||||
if missing git; then pkg_install git git git git; fi
|
||||
# The race detector in backend/'s make test needs cgo, which Go turns
|
||||
# on only when it finds its C compiler, gcc on Linux. apt and apk
|
||||
# ship the C library headers apart from gcc.
|
||||
if missing gcc; then
|
||||
pkg_install gcc "gcc libc6-dev" gcc "gcc musl-dev"
|
||||
fi
|
||||
|
||||
ensure_node
|
||||
ensure_yarn
|
||||
yarn install --frozen-lockfile
|
||||
|
||||
ensure_go
|
||||
(cd "$ROOT/backend" && go mod download)
|
||||
go mod download
|
||||
|
||||
if missing docker; then
|
||||
echo "bootstrap: docker not found; make test and make lint, and so" >&2
|
||||
|
||||
+11
-2
@@ -1,6 +1,7 @@
|
||||
#!/bin/sh
|
||||
# script/build: build the frontend for production into dist/. The Go
|
||||
# backend is built by backend/script/build.
|
||||
# script/build: build the frontend for production into dist/, then the
|
||||
# static netwatch-server binary into the repo root, with its version
|
||||
# stamped in.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
@@ -8,6 +9,14 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
yarn build
|
||||
|
||||
# VERSION comes from the environment. Unset or empty, it is git
|
||||
# describe, or "dev" where there is no git or no repository history.
|
||||
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
|
||||
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w -X main.Version=$version" \
|
||||
-o netwatch-server ./cmd/netwatch-server/
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
# script/clean: remove what script/build writes: dist/ and the
|
||||
# netwatch-server binary.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
rm -rf dist netwatch-server
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+5
-3
@@ -1,7 +1,9 @@
|
||||
#!/bin/sh
|
||||
# script/fmt: format the whole repo (writes): prettier over the
|
||||
# JavaScript, CSS, HTML and Markdown, then gofmt over the Go backend.
|
||||
# The org model formats only markdown; this repo also has JS and Go.
|
||||
# JavaScript, CSS, HTML and Markdown, then gofmt over the Go sources in
|
||||
# cmd/ and internal/; not over the whole root, as node_modules/ holds
|
||||
# .go files of its own. The org model formats only markdown; this repo
|
||||
# also has JS and Go.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
@@ -12,7 +14,7 @@ main() {
|
||||
# ~/.local/bin, which the shell that called it may not have on PATH.
|
||||
PATH="$HOME/.local/bin:$PATH"
|
||||
"$ROOT/script/frontend-fmt"
|
||||
"$ROOT/backend/script/fmt"
|
||||
gofmt -l -w cmd internal
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+6
-1
@@ -12,7 +12,12 @@ main() {
|
||||
# ~/.local/bin, which the shell that called it may not have on PATH.
|
||||
PATH="$HOME/.local/bin:$PATH"
|
||||
"$ROOT/script/frontend-fmt-check"
|
||||
"$ROOT/backend/script/fmt-check"
|
||||
unformatted="$(gofmt -l cmd internal)"
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "Go files not formatted:" >&2
|
||||
echo "$unformatted" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+4
-4
@@ -1,9 +1,9 @@
|
||||
#!/bin/sh
|
||||
# script/frontend-fmt: format the JavaScript, CSS, HTML and Markdown,
|
||||
# repo-wide (writes), the markdown in backend/ included. Those are the
|
||||
# languages REPO_POLICIES.md gives prettier; the YAML is left alone, as
|
||||
# backend/.golangci.yml must stay the org standard byte for byte.
|
||||
# Prettier does not read Go; backend/script/fmt formats the Go sources.
|
||||
# repo-wide (writes). Those are the languages REPO_POLICIES.md gives
|
||||
# prettier; the YAML is left alone, as .golangci.yml must stay the org
|
||||
# standard byte for byte. Prettier does not read Go; script/fmt runs
|
||||
# gofmt over the Go sources.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# script/frontend-viewport-test: verify the responsive layout of the built
|
||||
# frontend in a real browser engine.
|
||||
#
|
||||
# Builds dist/, serves it with the same nginx image and the same nginx.conf
|
||||
# the shipping container uses, points a containerised headless Chrome at it
|
||||
# over CDP, and asserts on computed layout at every viewport width derived
|
||||
# from the app's own CSS. See test/viewport/README.md for what this covers
|
||||
# and what it cannot.
|
||||
# Builds dist/, serves it with the same nginx image and the same
|
||||
# configs/nginx.conf the shipping container uses, points a containerised
|
||||
# headless Chrome at it over CDP, and asserts on computed layout at every
|
||||
# viewport width derived from the app's own CSS. See
|
||||
# test/viewport/README.md for what this covers and what it cannot.
|
||||
#
|
||||
# Deliberately not part of script/check: it needs Docker and takes far
|
||||
# longer than the 20s budget make test has to stay inside.
|
||||
@@ -61,18 +61,18 @@ main() {
|
||||
# host.
|
||||
docker network create --internal "$NETWORK" > /dev/null
|
||||
|
||||
# nginx.conf is a template: the image renders it over its own
|
||||
# configs/nginx.conf is a template: the image renders it over its own
|
||||
# default.conf, with the same port and limit bin/entrypoint.sh uses.
|
||||
# The empty file it includes trusts no proxy, as bin/entrypoint.sh
|
||||
# writes it when TRUSTED_PROXIES is unset. nginx.conf also includes
|
||||
# the security headers, so the page runs under the shipped policy.
|
||||
# writes it when TRUSTED_PROXIES is unset. It also includes the
|
||||
# security headers, so the page runs under the shipped policy.
|
||||
docker run -d --rm --name "$SERVER" \
|
||||
--network "$NETWORK" --network-alias netwatch \
|
||||
-e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \
|
||||
-v "$ROOT/dist:/usr/share/nginx/html:ro" \
|
||||
-v "$ROOT/nginx.conf:/etc/nginx/templates/default.conf.template:ro" \
|
||||
-v "$ROOT/configs/nginx.conf:/etc/nginx/templates/default.conf.template:ro" \
|
||||
-v /dev/null:/etc/nginx/trusted-proxies.conf:ro \
|
||||
-v "$ROOT/security-headers.conf:/etc/nginx/security-headers.conf:ro" \
|
||||
-v "$ROOT/configs/security-headers.conf:/etc/nginx/security-headers.conf:ro" \
|
||||
"$SERVER_IMAGE" > /dev/null
|
||||
|
||||
# The image's own entrypoint already exposes CDP on 9222 and passes
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/bin/sh
|
||||
# script/run: build and run netwatch-server locally.
|
||||
# script/run: build with script/build, then run netwatch-server locally
|
||||
# with its defaults: port 8080, reports stored in data/reports.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
+3
-3
@@ -1,12 +1,12 @@
|
||||
#!/bin/sh
|
||||
# script/tidy: run go mod tidy in backend/, which adds the modules the
|
||||
# Go sources import, drops those they no longer do, and updates go.sum.
|
||||
# script/tidy: run go mod tidy, which adds the modules the Go sources
|
||||
# import, drops those they no longer do, and updates go.sum.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT/backend"
|
||||
cd "$ROOT"
|
||||
go mod tidy
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Unit tests for src/main.js, run with Node's built-in test runner by the
|
||||
// Unit tests for web/src/main.js, run with Node's built-in test runner by the
|
||||
// test script in package.json. Importing the module does not start the
|
||||
// page.
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
latencyHex,
|
||||
measureLatency,
|
||||
tick,
|
||||
} from "../../src/main.js";
|
||||
} from "../../web/src/main.js";
|
||||
|
||||
// There is no page here, so the tests stand in for it. The debug log looks
|
||||
// for its panel by id and finds none. Each element of a host's row that
|
||||
|
||||
+10
-10
@@ -7,11 +7,11 @@ make frontend-viewport-test
|
||||
```
|
||||
|
||||
It builds `dist/`, serves it from the same digest-pinned `nginx` image and the
|
||||
same `nginx.conf` the shipping container uses, drives a digest-pinned headless
|
||||
Chrome against it over CDP, and asserts on computed layout at every viewport
|
||||
width derived from the app's own CSS. Screenshots land in `tmp/viewport/`
|
||||
alongside a `results.json`; they are artifacts for a human to look at when
|
||||
something fails, not the evidence. The assertions are the evidence.
|
||||
same `configs/nginx.conf` the shipping container uses, drives a digest-pinned
|
||||
headless Chrome against it over CDP, and asserts on computed layout at every
|
||||
viewport width derived from the app's own CSS. Screenshots land in
|
||||
`tmp/viewport/` alongside a `results.json`; they are artifacts for a human to
|
||||
look at when something fails, not the evidence. The assertions are the evidence.
|
||||
|
||||
The target is deliberately outside `make check`: it takes minutes, and
|
||||
`make test` has to stay under 60 seconds.
|
||||
@@ -19,10 +19,10 @@ The target is deliberately outside `make check`: it takes minutes, and
|
||||
## How the widths are chosen
|
||||
|
||||
Not from a list of phone models. `viewports.js` parses the `@media` conditions
|
||||
out of `src/styles.css` and scans `src/main.js` and `index.html` for Tailwind
|
||||
responsive prefixes, then tests every breakpoint it finds at one pixel below it,
|
||||
exactly on it, and one pixel above it. A generic 375px "phone" test sails
|
||||
straight past an off-by-one at a media query boundary; `max-width: 768px`
|
||||
out of `web/src/styles.css` and scans `web/src/main.js` and `web/index.html` for
|
||||
Tailwind responsive prefixes, then tests every breakpoint it finds at one pixel
|
||||
below it, exactly on it, and one pixel above it. A generic 375px "phone" test
|
||||
sails straight past an off-by-one at a media query boundary; `max-width: 768px`
|
||||
matches _at_ 768, and the sweep pins down which side of that line each layout is
|
||||
on.
|
||||
|
||||
@@ -108,7 +108,7 @@ function of viewport width and CSS, and is covered above.
|
||||
|
||||
Complementary layers, not two stacks. The unit tests in `test/unit/`, which
|
||||
`make test` runs with Node's built-in test runner, exercise the functions
|
||||
`src/main.js` exports in-process with no browser. This harness exercises
|
||||
`web/src/main.js` exports in-process with no browser. This harness exercises
|
||||
rendered layout in a real engine and is the only thing here that can see a media
|
||||
query. Neither replaces the other; assertions about computed styles and element
|
||||
geometry belong here, assertions about functions belong in `test/unit/`.
|
||||
|
||||
@@ -51,7 +51,8 @@ const PROBE_DELAYS_MS = [2, 45, 123, 456, 780];
|
||||
// offline row rendering is exercised too.
|
||||
const UNREACHABLE_MODULUS = 7;
|
||||
// The gateway candidate that "answers", so gateway detection succeeds and
|
||||
// the Local Gateway row renders. Matches GATEWAY_CANDIDATES in src/main.js.
|
||||
// the Local Gateway row renders. Matches GATEWAY_CANDIDATES in
|
||||
// web/src/main.js.
|
||||
const RESPONSIVE_GATEWAY = "http://192.168.1.1";
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
@@ -161,26 +161,27 @@ const ANCHOR_VIEWPORTS = [
|
||||
export function deriveViewports(root) {
|
||||
const conditions = [
|
||||
...mediaConditionsFromCss(
|
||||
readFileSync(join(root, "src/styles.css"), "utf8"),
|
||||
"src/styles.css",
|
||||
readFileSync(join(root, "web/src/styles.css"), "utf8"),
|
||||
"web/src/styles.css",
|
||||
),
|
||||
...mediaConditionsFromMarkup([
|
||||
{
|
||||
path: "src/main.js",
|
||||
text: readFileSync(join(root, "src/main.js"), "utf8"),
|
||||
path: "web/src/main.js",
|
||||
text: readFileSync(join(root, "web/src/main.js"), "utf8"),
|
||||
},
|
||||
{
|
||||
path: "index.html",
|
||||
text: readFileSync(join(root, "index.html"), "utf8"),
|
||||
path: "web/index.html",
|
||||
text: readFileSync(join(root, "web/index.html"), "utf8"),
|
||||
},
|
||||
]),
|
||||
];
|
||||
|
||||
if (conditions.length === 0) {
|
||||
throw new Error(
|
||||
"no responsive breakpoints found in src/styles.css, src/main.js or " +
|
||||
"index.html — either the responsive layout was deleted or this " +
|
||||
"derivation has stopped matching the sources",
|
||||
"no responsive breakpoints found in web/src/styles.css, " +
|
||||
"web/src/main.js or web/index.html — either the responsive " +
|
||||
"layout was deleted or this derivation has stopped matching " +
|
||||
"the sources",
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,10 @@ const commitHash = execSync("git rev-parse --short HEAD").toString().trim();
|
||||
const commitFull = execSync("git rev-parse HEAD").toString().trim();
|
||||
|
||||
export default defineConfig({
|
||||
// The page's source is in web/; the build still goes to dist/ at the
|
||||
// repo root, which Dockerfile and the viewport test serve. Vite
|
||||
// empties only an outDir inside root unless told to.
|
||||
root: "web",
|
||||
plugins: [tailwindcss()],
|
||||
server: {
|
||||
// Proxy /api to a locally running netwatch-server so `yarn dev`
|
||||
@@ -19,6 +23,8 @@ export default defineConfig({
|
||||
__COMMIT_FULL__: JSON.stringify(commitFull),
|
||||
},
|
||||
build: {
|
||||
outDir: "../dist",
|
||||
emptyOutDir: true,
|
||||
target: "esnext",
|
||||
minify: "esbuild",
|
||||
cssMinify: true,
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
type="image/svg+xml"
|
||||
href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>📡</text></svg>"
|
||||
/>
|
||||
<!-- Linked here, not imported by src/main.js, so the unit tests can
|
||||
<!-- Linked here, not imported by web/src/main.js, so the unit tests can
|
||||
import that module in Node, which cannot import CSS. -->
|
||||
<link rel="stylesheet" href="/src/styles.css" />
|
||||
</head>
|
||||
Reference in New Issue
Block a user