Compare commits
1
Commits
next
..
c6f325cff5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6f325cff5 |
+4
-4
@@ -72,10 +72,10 @@
|
|||||||
**/*.sublime-*
|
**/*.sublime-*
|
||||||
|
|
||||||
# This repository's own entries, after the shared content above: the
|
# This repository's own entries, after the shared content above: the
|
||||||
# frontend build, the viewport test's output, the binary `make build`
|
# frontend build, the viewport test's output, what `make build` and
|
||||||
# writes and the reports `make run` stores, and log files at the root.
|
# `make run` in backend/ leave behind, and log files at the root.
|
||||||
/dist
|
/dist
|
||||||
/tmp
|
/tmp
|
||||||
/netwatch-server
|
/backend/netwatch-server
|
||||||
/data
|
/backend/data
|
||||||
/*.log
|
/*.log
|
||||||
|
|||||||
+1
-4
@@ -51,10 +51,7 @@ node_modules/
|
|||||||
# Build output
|
# Build output
|
||||||
dist/
|
dist/
|
||||||
tmp/
|
tmp/
|
||||||
/netwatch-server
|
/backend/netwatch-server
|
||||||
|
|
||||||
# The reports make run stores, in netwatch-server's default DATA_DIR
|
|
||||||
/data/
|
|
||||||
|
|
||||||
# Go test binaries and coverage output
|
# Go test binaries and coverage output
|
||||||
*.test
|
*.test
|
||||||
|
|||||||
+34
-43
@@ -5,11 +5,11 @@
|
|||||||
#
|
#
|
||||||
# The lint and test phases are the gates: `make lint` (script/lint)
|
# The lint and test phases are the gates: `make lint` (script/lint)
|
||||||
# builds the lint stage alone and `make test` (script/test) the test
|
# builds the lint stage alone and `make test` (script/test) the test
|
||||||
# stage alone, and the two build stages, frontend-build and builder,
|
# stage alone, and the builder stage depends on both, so the image
|
||||||
# depend on both, so the image cannot be built unless they pass. Each
|
# cannot be built unless they pass. Each covers the frontend as well,
|
||||||
# phase covers the frontend as well, through a copy from a node stage.
|
# through a copy from a node stage. Inside them each tool is invoked
|
||||||
# Inside them each tool is invoked directly, never through make or
|
# directly, never through make or script/, whose lint and test are
|
||||||
# script/, whose lint and test are themselves docker builds.
|
# themselves docker builds.
|
||||||
|
|
||||||
# Frontend lint stage: eslint with the rules in eslint.config.js. The
|
# Frontend lint stage: eslint with the rules in eslint.config.js. The
|
||||||
# lint phase below runs it.
|
# lint phase below runs it.
|
||||||
@@ -21,40 +21,44 @@ RUN yarn install --frozen-lockfile
|
|||||||
COPY . .
|
COPY . .
|
||||||
RUN yarn eslint .
|
RUN yarn eslint .
|
||||||
|
|
||||||
# Lint phase: golangci-lint over the Go packages with .golangci.yml,
|
# Lint phase: golangci-lint over the backend with backend/.golangci.yml,
|
||||||
# and eslint through the copy from frontend-lint at the end. The
|
# and eslint through the copy from frontend-lint at the end. The
|
||||||
# golangci/golangci-lint image ships Go and the linter.
|
# golangci/golangci-lint image ships Go and the linter.
|
||||||
# golangci/golangci-lint:v2.14.0, 2026-09-24
|
# golangci/golangci-lint:v2.14.0, 2026-09-24
|
||||||
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY backend/go.mod backend/go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY backend/ .
|
||||||
RUN golangci-lint run --config .golangci.yml ./...
|
RUN golangci-lint run --config .golangci.yml ./...
|
||||||
# Nothing is wanted from frontend-lint; the copy is what makes this
|
# Nothing is wanted from frontend-lint; the copy is what makes this
|
||||||
# phase run it.
|
# phase run it.
|
||||||
COPY --from=frontend-lint /app/yarn.lock /dev/null
|
COPY --from=frontend-lint /app/yarn.lock /dev/null
|
||||||
|
|
||||||
# Frontend test stage: the unit tests in test/unit/. The test phase
|
# Frontend stage: the unit tests in test/unit/, then the production
|
||||||
# below runs it, and the frontend-build stage starts from it. The tests
|
# build into dist/, which the runtime stage serves. The test phase below
|
||||||
# print a dot each; if any fails, they run again with every test listed,
|
# runs it. The tests print a dot each; if any fails, they run again with
|
||||||
# and the step fails even if that run passes. NODE_OPTIONS chooses the
|
# every test listed, and the step fails even if that run passes.
|
||||||
# reporter because yarn adds its arguments after the test files, where
|
# NODE_OPTIONS chooses the reporter because yarn adds its arguments
|
||||||
# node would take a reporter option for one more file.
|
# after the test files, where node would take a reporter option for one
|
||||||
|
# more file.
|
||||||
# node:22-alpine as of 2026-02-22
|
# node:22-alpine as of 2026-02-22
|
||||||
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend-test
|
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY package.json yarn.lock ./
|
COPY package.json yarn.lock ./
|
||||||
RUN yarn install --frozen-lockfile
|
RUN yarn install --frozen-lockfile
|
||||||
|
# vite.config.js reads the commit for the page's footer with git.
|
||||||
|
RUN apk add --no-cache git
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \
|
RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \
|
||||||
{ echo "--- Rerunning with every test listed for details ---"; \
|
{ echo "--- Rerunning with every test listed for details ---"; \
|
||||||
NODE_OPTIONS=--test-reporter=spec timeout 90 yarn --silent run test; \
|
NODE_OPTIONS=--test-reporter=spec timeout 90 yarn --silent run test; \
|
||||||
exit 1; }
|
exit 1; }
|
||||||
|
RUN yarn build
|
||||||
|
|
||||||
# Test phase: the backend's tests with the race detector and coverage,
|
# Test phase: the backend's tests with the race detector and coverage,
|
||||||
# and the frontend's through the copy from the frontend-test stage at
|
# and the frontend's through the copy from the frontend stage at the
|
||||||
# the end. -race needs cgo and so a C compiler, which the Debian Go image
|
# end. -race needs cgo and so a C compiler, which the Debian Go image
|
||||||
# ships and the alpine one does not. -timeout 90s is a backstop above
|
# ships and the alpine one does not. -timeout 90s is a backstop above
|
||||||
# the 60-second cap on the suite. The rerun with -v only shows details:
|
# the 60-second cap on the suite. The rerun with -v only shows details:
|
||||||
# the step fails however it ends, because the first run already failed.
|
# the step fails however it ends, because the first run already failed.
|
||||||
@@ -65,27 +69,15 @@ RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \
|
|||||||
# golang:1.25.7-trixie, 2026-10-07
|
# golang:1.25.7-trixie, 2026-10-07
|
||||||
FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test
|
FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY . .
|
COPY backend/ .
|
||||||
RUN --mount=type=tmpfs,target=/go/pkg/mod \
|
RUN --mount=type=tmpfs,target=/go/pkg/mod \
|
||||||
--mount=type=tmpfs,target=/root/.cache/go-build \
|
--mount=type=tmpfs,target=/root/.cache/go-build \
|
||||||
go test -timeout 90s -race -cover ./... || \
|
go test -timeout 90s -race -cover ./... || \
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 90s -race -v ./...; exit 1; }
|
||||||
# Nothing is wanted from the frontend-test stage; the copy is what makes
|
# Nothing is wanted from the frontend stage; the copy is what makes
|
||||||
# this phase run its tests.
|
# this phase run its tests.
|
||||||
COPY --from=frontend-test /app/yarn.lock /dev/null
|
COPY --from=frontend /app/yarn.lock /dev/null
|
||||||
|
|
||||||
# Frontend build stage: the production build into dist/, which the
|
|
||||||
# runtime stage serves, on the yarn packages and source of the
|
|
||||||
# frontend-test stage. Nothing is wanted from the two phases; the copies
|
|
||||||
# are what make BuildKit build them first, so this stage cannot run
|
|
||||||
# unless lint and test passed.
|
|
||||||
FROM frontend-test AS frontend-build
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
COPY --from=test /src/go.sum /dev/null
|
|
||||||
# vite.config.js reads the commit for the page's footer with git.
|
|
||||||
RUN apk add --no-cache git
|
|
||||||
RUN yarn build
|
|
||||||
|
|
||||||
# Backend build stage. Nothing is wanted from the two phases; the copies
|
# Backend build stage. Nothing is wanted from the two phases; the copies
|
||||||
# are what make BuildKit build them first, so this stage cannot run
|
# are what make BuildKit build them first, so this stage cannot run
|
||||||
@@ -98,12 +90,13 @@ RUN apk add --no-cache git
|
|||||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||||
RUN git config --system --add safe.directory /src
|
RUN git config --system --add safe.directory /src
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY backend/go.mod backend/go.sum backend/
|
||||||
RUN go mod download
|
RUN cd backend && go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# The same go build as in script/build, which builds the binary on the
|
# backend/script/build is the one definition of the build command:
|
||||||
# host.
|
# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=..."
|
||||||
|
# It reads VERSION from the environment.
|
||||||
#
|
#
|
||||||
# The version is the VERSION build argument when one is given, otherwise
|
# The version is the VERSION build argument when one is given, otherwise
|
||||||
# `git describe --tags --always` on the .git in the build context: the
|
# `git describe --tags --always` on the .git in the build context: the
|
||||||
@@ -119,9 +112,7 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
|||||||
exit 1 ;; \
|
exit 1 ;; \
|
||||||
esac; \
|
esac; \
|
||||||
fi; \
|
fi; \
|
||||||
CGO_ENABLED=0 go build -trimpath \
|
VERSION="$version" backend/script/build
|
||||||
-ldflags "-s -w -X main.Version=$version" \
|
|
||||||
-o netwatch-server ./cmd/netwatch-server/
|
|
||||||
|
|
||||||
# Runtime stage, and the last one: a plain `docker build .` builds it
|
# Runtime stage, and the last one: a plain `docker build .` builds it
|
||||||
# and the stages it copies from, the two phases included.
|
# and the stages it copies from, the two phases included.
|
||||||
@@ -137,10 +128,10 @@ RUN addgroup -g 1000 -S netwatch && \
|
|||||||
# At start-up the nginx image renders every template here into
|
# At start-up the nginx image renders every template here into
|
||||||
# conf.d; bin/entrypoint.sh says how.
|
# conf.d; bin/entrypoint.sh says how.
|
||||||
RUN rm /etc/nginx/conf.d/default.conf
|
RUN rm /etc/nginx/conf.d/default.conf
|
||||||
COPY configs/nginx.conf /etc/nginx/templates/netwatch.conf.template
|
COPY nginx.conf /etc/nginx/templates/netwatch.conf.template
|
||||||
COPY configs/security-headers.conf /etc/nginx/security-headers.conf
|
COPY security-headers.conf /etc/nginx/security-headers.conf
|
||||||
COPY --from=frontend-build /app/dist /usr/share/nginx/html
|
COPY --from=frontend /app/dist /usr/share/nginx/html
|
||||||
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
|
COPY --from=builder /src/backend/netwatch-server /usr/local/bin/netwatch-server
|
||||||
COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||||
|
|
||||||
# bin/entrypoint.sh creates DATA_DIR at start and gives it and /data to
|
# bin/entrypoint.sh creates DATA_DIR at start and gives it and /data to
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
.PHONY: bootstrap setup dev build run clean test lint fmt fmt-check check \
|
.PHONY: bootstrap setup dev build test lint fmt fmt-check check \
|
||||||
add-dependency tidy frontend-viewport-test docker hooks
|
add-dependency tidy frontend-viewport-test docker hooks
|
||||||
|
|
||||||
# Standard targets are thin shims; the implementations live in script/
|
# Standard targets are thin shims; the implementations live in script/
|
||||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||||
# of README.md). test, lint, fmt, fmt-check and check cover the whole
|
# of README.md). test, lint, fmt, fmt-check and check cover the whole
|
||||||
# repo: the frontend in web/ and the Go backend in cmd/ and internal/.
|
# repo: the frontend here and the Go backend in backend/.
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
@script/bootstrap
|
@script/bootstrap
|
||||||
@@ -15,17 +15,10 @@ setup:
|
|||||||
dev:
|
dev:
|
||||||
@script/dev
|
@script/dev
|
||||||
|
|
||||||
# The frontend into dist/ and the netwatch-server binary.
|
# The frontend only; backend/Makefile's build target builds the Go server.
|
||||||
build:
|
build:
|
||||||
@script/build
|
@script/build
|
||||||
|
|
||||||
# Builds, then runs netwatch-server on port 8080.
|
|
||||||
run:
|
|
||||||
@script/run
|
|
||||||
|
|
||||||
clean:
|
|
||||||
@script/clean
|
|
||||||
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
|
|||||||
@@ -13,13 +13,10 @@ make setup
|
|||||||
# Run the page on the Vite dev server
|
# Run the page on the Vite dev server
|
||||||
make dev
|
make dev
|
||||||
|
|
||||||
# Build netwatch-server and run it on port 8080
|
|
||||||
make run
|
|
||||||
|
|
||||||
# Run the tests, both linters and the format check
|
# Run the tests, both linters and the format check
|
||||||
make check
|
make check
|
||||||
|
|
||||||
# Build the page into dist/ and netwatch-server into the repo root
|
# Build the page into dist/
|
||||||
make build
|
make build
|
||||||
|
|
||||||
# Build the image and run it
|
# Build the image and run it
|
||||||
@@ -28,8 +25,9 @@ docker run -p 8080:8080 netwatch
|
|||||||
```
|
```
|
||||||
|
|
||||||
`make check` and `make docker` need Docker. `make dev` passes `/api` to
|
`make check` and `make docker` need Docker. `make dev` passes `/api` to
|
||||||
`http://127.0.0.1:8080`, where `make run` starts `netwatch-server` with its
|
`http://127.0.0.1:8080`, where `make run` in `backend/` starts `netwatch-server`
|
||||||
defaults, so the reports the page posts are stored in `data/reports`.
|
with its defaults, so the reports the page posts are stored in
|
||||||
|
`backend/data/reports`.
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
|
|
||||||
@@ -37,52 +35,49 @@ This repository adheres to the
|
|||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
standard: normalized scripts in `script/` are the entrypoints for the
|
||||||
development workflow, and the Makefile targets are thin shims that call them.
|
development workflow, and the Makefile targets are thin shims that call them.
|
||||||
The scripts cover both the frontend and the Go backend, so `make check` fails if
|
The Go backend in `backend/` has its own `script/` directory and shim Makefile
|
||||||
either one is broken. We provide:
|
(see [backend/README.md](backend/README.md)). The root scripts cover both
|
||||||
|
halves, so the root `make check` fails if either one is broken. We provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (the pinned node via nvm unless
|
- `script/bootstrap` — install all dependencies (the pinned node via nvm unless
|
||||||
one new enough for the frontend's dependencies is installed, yarn via
|
one new enough for the frontend's dependencies is installed, yarn via
|
||||||
corepack, `yarn install --frozen-lockfile`, Go `GO_VERSION` unless the `go` on
|
corepack, `yarn install --frozen-lockfile`, Go `GO_VERSION` unless the `go` on
|
||||||
`PATH` is exactly that version, and the Go modules), linking what it installs
|
`PATH` is exactly that version, the Go modules, and gcc with the C library
|
||||||
itself into `~/.local/bin`, which has to be on `PATH`. It installs no Go
|
headers unless gcc is installed, for the race detector in `make test` in
|
||||||
linter and not Docker: `make test` and `make lint` run in Docker
|
`backend/`), linking what it installs itself into `~/.local/bin`, which has to
|
||||||
|
be on `PATH`. It installs no Go linter and not Docker: `make test` and
|
||||||
|
`make lint` run in Docker
|
||||||
- `script/setup` — make a fresh clone ready for development: bootstrap plus the
|
- `script/setup` — make a fresh clone ready for development: bootstrap plus the
|
||||||
git pre-commit hook
|
git pre-commit hook
|
||||||
- `script/dev` — run the Vite dev server, which proxies `/api` to a locally
|
- `script/dev` — run the Vite dev server, which proxies `/api` to a locally
|
||||||
running `netwatch-server`
|
running `netwatch-server`
|
||||||
- `script/build` — build the frontend for production into `dist/`, then the
|
- `script/build` — build the frontend for production into `dist/`;
|
||||||
static `netwatch-server` binary into the repo root with its version stamped
|
`backend/script/build` builds the Go server
|
||||||
in: `VERSION` from the environment, or when that is unset or empty,
|
|
||||||
`git describe` inside a git checkout, then `dev`
|
|
||||||
- `script/run` — build with `script/build`, then run `netwatch-server` with its
|
|
||||||
defaults (see [Settings](#settings))
|
|
||||||
- `script/clean` — remove what `script/build` writes
|
|
||||||
- `script/projectname` — print the project name (used for the Docker image tag)
|
- `script/projectname` — print the project name (used for the Docker image tag)
|
||||||
- `script/test` — build the `test` phase of `Dockerfile` without the cache: the
|
- `script/test` — build the `test` phase of `Dockerfile` without the cache: the
|
||||||
frontend's unit tests in its `frontend-test` stage, and the backend's Go tests
|
frontend's unit tests and production build in its `frontend` stage, and the
|
||||||
with the race detector and coverage. The frontend's production build is left
|
backend's Go tests with the race detector and coverage
|
||||||
to the image build, in the `frontend-build` stage
|
|
||||||
- `script/lint` — build the `lint` phase of `Dockerfile` without the cache:
|
- `script/lint` — build the `lint` phase of `Dockerfile` without the cache:
|
||||||
eslint in its `frontend-lint` stage, and golangci-lint over the Go packages
|
eslint in its `frontend-lint` stage, and golangci-lint over `backend/`
|
||||||
- `script/fmt` — format all files (writes): prettier over the JavaScript, CSS,
|
- `script/fmt` — format all files (writes): prettier over the JavaScript, CSS,
|
||||||
HTML and Markdown, then gofmt over `cmd/` and `internal/`. It runs on the
|
HTML and Markdown, then gofmt over `backend/`. It runs on the host, as
|
||||||
host, as `script/fmt-check` does, with `~/.local/bin` put on `PATH`
|
`script/fmt-check` does, with `~/.local/bin` put on `PATH`
|
||||||
- `script/fmt-check` — check formatting (read-only): prettier, then gofmt
|
- `script/fmt-check` — check formatting (read-only): prettier, then gofmt
|
||||||
- `script/check` — run test, lint, and fmt-check
|
- `script/check` — run test, lint, and fmt-check
|
||||||
- `script/add-dependency` — add a frontend package, or move one to another
|
- `script/add-dependency` — add a frontend package, or move one to another
|
||||||
version: `make add-dependency PACKAGE=<name>@<version>` runs `yarn add --dev`,
|
version: `make add-dependency PACKAGE=<name>@<version>` runs `yarn add --dev`,
|
||||||
which changes `package.json` and `yarn.lock` together, then
|
which changes `package.json` and `yarn.lock` together, then
|
||||||
`yarn install --frozen-lockfile`
|
`yarn install --frozen-lockfile`
|
||||||
- `script/tidy` — run `go mod tidy`: to add a Go module, import it and run
|
- `script/tidy` — run `go mod tidy` in `backend/`: to add a Go module, import it
|
||||||
`make tidy`; to move one to another version, edit its `require` line in
|
and run `make tidy`; to move one to another version, edit its `require` line
|
||||||
`go.mod`, then run `make tidy`
|
in `backend/go.mod`, then run `make tidy`
|
||||||
- `script/frontend-test` — run the unit tests in `test/unit/` on the host with
|
- `script/frontend-test` — run the unit tests in `test/unit/` on the host with
|
||||||
Node's built-in test runner, through the `test` script in `package.json`, and
|
Node's built-in test runner, through the `test` script in `package.json`, and
|
||||||
if any fails, run them again listing every test, and fail; then the production
|
if any fails, run them again listing every test, and fail; then the production
|
||||||
build. Each test run has a 90-second timeout. `make test` runs the same tests
|
build. Each test run has a 90-second timeout. `make test` runs the same in
|
||||||
in Docker, without the production build
|
Docker
|
||||||
- `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with
|
- `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with
|
||||||
prettier (writes)
|
prettier (writes), the markdown in `backend/` included
|
||||||
- `script/frontend-fmt-check` — check prettier formatting (read-only)
|
- `script/frontend-fmt-check` — check prettier formatting (read-only)
|
||||||
- `script/frontend-viewport-test` — responsive-layout verification of the built
|
- `script/frontend-viewport-test` — responsive-layout verification of the built
|
||||||
frontend in a containerised headless Chrome (see
|
frontend in a containerised headless Chrome (see
|
||||||
@@ -98,11 +93,11 @@ either one is broken. We provide:
|
|||||||
## Responsive layout
|
## Responsive layout
|
||||||
|
|
||||||
The narrow-viewport layout lives in the `max-width: 768px` media block in
|
The narrow-viewport layout lives in the `max-width: 768px` media block in
|
||||||
`web/src/styles.css`. It is verified automatically by
|
`src/styles.css`. It is verified automatically by `make frontend-viewport-test`,
|
||||||
`make frontend-viewport-test`, which drives a digest-pinned headless Chrome
|
which drives a digest-pinned headless Chrome against the built `dist/` and
|
||||||
against the built `dist/` and asserts on computed layout at widths derived from
|
asserts on computed layout at widths derived from that CSS — on every breakpoint
|
||||||
that CSS — on every breakpoint it declares and one pixel either side of it, plus
|
it declares and one pixel either side of it, plus a 320px floor, a desktop
|
||||||
a 320px floor, a desktop baseline and two landscape sizes. See
|
baseline and two landscape sizes. See
|
||||||
[test/viewport/README.md](test/viewport/README.md) for what it covers and what
|
[test/viewport/README.md](test/viewport/README.md) for what it covers and what
|
||||||
it genuinely cannot.
|
it genuinely cannot.
|
||||||
|
|
||||||
@@ -117,11 +112,8 @@ the same and nothing is stored.
|
|||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
The page's source is in `web/`, the Go backend's in `cmd/` and `internal/`, and
|
|
||||||
nginx's configuration in `configs/`.
|
|
||||||
|
|
||||||
The page is built with Vite and Tailwind CSS v4. Its code is all in
|
The page is built with Vite and Tailwind CSS v4. Its code is all in
|
||||||
`web/src/main.js`, with a class-based architecture:
|
`src/main.js`, with a class-based architecture:
|
||||||
|
|
||||||
- **`CONFIG`**: Configuration object (update interval, timeouts, axis ticks,
|
- **`CONFIG`**: Configuration object (update interval, timeouts, axis ticks,
|
||||||
etc.). The interval menu sets `updateInterval`, the one value the page writes
|
etc.). The interval menu sets `updateInterval`, the one value the page writes
|
||||||
@@ -158,10 +150,10 @@ is a pure function of host state.
|
|||||||
|
|
||||||
### Backend
|
### Backend
|
||||||
|
|
||||||
`netwatch-server` is a small Go HTTP server that stores the reports the page
|
`netwatch-server`, in `backend/`, is a small Go HTTP server that stores the
|
||||||
posts. It keeps them in memory and writes them to `DATA_DIR` as zstd-compressed
|
reports the page posts. It keeps them in memory and writes them to `DATA_DIR` as
|
||||||
files of JSON lines: every minute, whenever 10 MiB are waiting, and when it
|
zstd-compressed files of JSON lines: every minute, whenever 10 MiB are waiting,
|
||||||
stops. Its routes:
|
and when it stops. Its routes:
|
||||||
|
|
||||||
- `POST /api/v1/reports` — takes a report, without credentials; each client
|
- `POST /api/v1/reports` — takes a report, without credentials; each client
|
||||||
address may send a limited number a minute, and the report files are capped in
|
address may send a limited number a minute, and the report files are capped in
|
||||||
@@ -172,149 +164,11 @@ stops. Its routes:
|
|||||||
`METRICS_USERNAME` and `METRICS_PASSWORD` are set; each client address may
|
`METRICS_USERNAME` and `METRICS_PASSWORD` are set; each client address may
|
||||||
make a limited number of requests to it a minute
|
make a limited number of requests to it a minute
|
||||||
|
|
||||||
It is an `fx`-wired Go application: `cmd/netwatch-server/` starts it, and its
|
|
||||||
packages are in `internal/`:
|
|
||||||
|
|
||||||
- **`config`**: Loads configuration from environment variables and config files
|
|
||||||
via Viper.
|
|
||||||
- **`handlers`**: HTTP request handlers for the API (health check, report
|
|
||||||
ingestion).
|
|
||||||
- **`reportbuf`**: The in-memory buffer of reports and the files it writes them
|
|
||||||
to.
|
|
||||||
- **`server`**: Chi-based HTTP server with middleware wiring and route
|
|
||||||
registration.
|
|
||||||
- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting
|
|
||||||
infrastructure.
|
|
||||||
|
|
||||||
In the image, the `test` phase of `Dockerfile` tests it, the `builder` stage
|
In the image, the `test` phase of `Dockerfile` tests it, the `builder` stage
|
||||||
builds it, and `bin/entrypoint.sh` runs it as user `netwatch` on
|
builds it with `backend/script/build`, and `bin/entrypoint.sh` runs it as user
|
||||||
`127.0.0.1:8081`, behind nginx (see [Deployment](#deployment)). Outside the
|
`netwatch` on `127.0.0.1:8081`, behind nginx. Outside the image, `make run` in
|
||||||
image, `make run` builds it and runs it on port 8080.
|
`backend/` builds it and runs it on port 8080. Its settings, report storage and
|
||||||
|
limits are in [backend/README.md](backend/README.md).
|
||||||
#### Settings
|
|
||||||
|
|
||||||
| Variable | Default | Description |
|
|
||||||
| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
|
|
||||||
| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface |
|
|
||||||
| `PORT` | `8080` | HTTP listen port |
|
|
||||||
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
|
||||||
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) |
|
|
||||||
| `DEBUG` | `false` | Enable debug logging |
|
|
||||||
| `TRUSTED_PROXIES` | RFC 1918 ranges | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
|
||||||
| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) |
|
|
||||||
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
|
||||||
| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) |
|
|
||||||
| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) |
|
|
||||||
| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) |
|
|
||||||
|
|
||||||
`TRUSTED_PROXIES` defaults to `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. A list
|
|
||||||
replaces the default; set and empty, the server trusts no proxy. Loopback is not
|
|
||||||
in the default, so a reverse proxy on the same host has to be named, as in
|
|
||||||
`TRUSTED_PROXIES=127.0.0.1/32`. A request whose direct peer is outside this set
|
|
||||||
has its forwarded headers ignored, and the direct peer is logged and
|
|
||||||
rate-limited instead. The container image does not use this default; see
|
|
||||||
[Deployment](#deployment).
|
|
||||||
|
|
||||||
A variable set to a value the server cannot use, such as `PORT=abc`,
|
|
||||||
`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from
|
|
||||||
starting, with an error naming the variable. An empty variable counts as unset,
|
|
||||||
except `TRUSTED_PROXIES`.
|
|
||||||
|
|
||||||
#### Report storage
|
|
||||||
|
|
||||||
Reports are written as `reports-<timestamp>-<number>.jsonl.zst` files in
|
|
||||||
`DATA_DIR`. The timestamp is in UTC to the millisecond, so the names sort by
|
|
||||||
time. The number starts at 1 when the server starts and goes up by one for each
|
|
||||||
file the server starts to write, so two files written in the same millisecond
|
|
||||||
still get different names. A failed write uses up its number and leaves a gap in
|
|
||||||
the numbers: its file, if it was created, is removed. The file stays, counted
|
|
||||||
toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too.
|
|
||||||
Files are created with `O_EXCL` to prevent overwrites.
|
|
||||||
|
|
||||||
The reports of a failed write stay in memory, and the next flush, within a
|
|
||||||
minute, tries again to write them. From the failed write until a write succeeds,
|
|
||||||
`POST /api/v1/reports` refuses every report with 500 and keeps nothing of it, so
|
|
||||||
the page keeps its samples and sends them again at its next report. Reports
|
|
||||||
already answered 200 are lost only if the server stops before a write succeeds;
|
|
||||||
the final flush at stop tries them too, and if it fails, the process exits
|
|
||||||
non-zero.
|
|
||||||
|
|
||||||
#### Report limits
|
|
||||||
|
|
||||||
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
|
||||||
credentials, so it is bounded instead. Each refusal below answers with the same
|
|
||||||
`{"status":"error"}` body as any other error.
|
|
||||||
|
|
||||||
- **JSON only.** A report must be sent with `Content-Type: application/json`,
|
|
||||||
parameters such as `charset` allowed, as the page sends it; any other is
|
|
||||||
refused with 415, and nothing of it is stored. A page on another origin can
|
|
||||||
send a `text/plain` POST, or a form, without a CORS preflight; refusing them
|
|
||||||
keeps any site from making its visitors' browsers store reports (see
|
|
||||||
[CORS](#cors)).
|
|
||||||
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
|
||||||
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
|
||||||
`Retry-After: 60`. The minute slides: reports from the minute before still
|
|
||||||
count, fading out over the current one, so an address is sure never to be
|
|
||||||
refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60
|
|
||||||
seconds. The page sends one report a minute from each open tab, so the default
|
|
||||||
of 60 refuses nothing from up to 30 tabs behind one address, such as a
|
|
||||||
household or an office sharing it, however their reports bunch up. Report
|
|
||||||
responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and
|
|
||||||
`X-RateLimit-Reset` headers.
|
|
||||||
- **Size cap.** The report files in `DATA_DIR` may total at most
|
|
||||||
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
|
|
||||||
waiting in memory count at their uncompressed size until they are written,
|
|
||||||
those of a failed write included; the part of its file written is removed.
|
|
||||||
When a report would take the total past the cap, the oldest report files are
|
|
||||||
deleted to make room, and each deletion is logged with the file's name and
|
|
||||||
size; a file still being written is never deleted. A report is refused with
|
|
||||||
507, and nothing of it is stored, only when the reports waiting to be written
|
|
||||||
fill the cap on their own, and then no file is deleted. At start, report files
|
|
||||||
past the cap, as after lowering it, are deleted the same way. So the cap is
|
|
||||||
how much of the newest reports is kept: the default of 1 GiB is small enough
|
|
||||||
for any host; set it to the space you can give `DATA_DIR`.
|
|
||||||
|
|
||||||
#### CORS
|
|
||||||
|
|
||||||
The page calls the API from the origin it is served from, so by default the
|
|
||||||
server sends no CORS headers, and browsers let no other origin's pages call it.
|
|
||||||
To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS`
|
|
||||||
(for example `https://netwatch.example.com`); pages from a listed origin may
|
|
||||||
`GET` and `POST` with a `Content-Type` header. Each entry must be a plain
|
|
||||||
origin, `scheme://host` with an optional `:port`, as browsers send it: no path,
|
|
||||||
not even a trailing `/`, and no `*`. Any other entry stops the server from
|
|
||||||
starting, with an error naming `CORS_ALLOWED_ORIGINS`.
|
|
||||||
|
|
||||||
#### Metrics
|
|
||||||
|
|
||||||
With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves
|
|
||||||
Prometheus metrics at `GET /metrics` to requests with those as their basic auth
|
|
||||||
credentials, and answers any other with 401. For each request that reaches the
|
|
||||||
health check or `POST /api/v1/reports`, those the rate limit refuses included,
|
|
||||||
the metrics record its duration and response size, labelled with its path,
|
|
||||||
method and status; they also count those requests in progress, and include Go's
|
|
||||||
runtime and process metrics. No other request is recorded: not those to
|
|
||||||
`/metrics` itself, and not those answered before they reach either route, such
|
|
||||||
as a CORS preflight, or a request refused with 404 for a path no route has, 405
|
|
||||||
for a method its route does not take, or 413 for declaring a body length over
|
|
||||||
the 1 MiB limit. A report whose body goes over the limit without declaring its
|
|
||||||
length reaches the route, is answered 413 there, and is recorded with that
|
|
||||||
status. Clients can make up any number of paths and methods, and each would add
|
|
||||||
labels to the metrics for as long as the server runs.
|
|
||||||
|
|
||||||
`/metrics` is rate limited, so that its password cannot be guessed quickly: each
|
|
||||||
client address, resolved through `TRUSTED_PROXIES`, may make 60 requests to it a
|
|
||||||
minute, whatever their credentials. Past that it gets 429 with
|
|
||||||
`Retry-After: 60`, and its credentials are not checked. The minute slides as it
|
|
||||||
does for reports (see [Report limits](#report-limits)), so a scraper polling
|
|
||||||
every 2 seconds or less often is never refused. This allowance is apart from the
|
|
||||||
one for reports.
|
|
||||||
|
|
||||||
#### Sentry
|
|
||||||
|
|
||||||
With `SENTRY_DSN` set, each panic in a handler is reported to that Sentry
|
|
||||||
project under the release `netwatch-server-` followed by the server's version.
|
|
||||||
On shutdown the server waits up to 2 seconds for Sentry to finish sending.
|
|
||||||
|
|
||||||
### Monitoring targets
|
### Monitoring targets
|
||||||
|
|
||||||
@@ -416,26 +270,20 @@ behind a reverse proxy.
|
|||||||
|
|
||||||
The Docker image, built from `Dockerfile` by `make docker`, is the whole service
|
The Docker image, built from `Dockerfile` by `make docker`, is the whole service
|
||||||
in one container: nginx serves the built frontend and passes `/api/`,
|
in one container: nginx serves the built frontend and passes `/api/`,
|
||||||
`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`.
|
`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`,
|
||||||
`bin/entrypoint.sh` starts it listening only inside the container, on
|
which listens only inside the container, on `127.0.0.1:8081`. The image:
|
||||||
`127.0.0.1:8081`, with `TRUSTED_PROXIES=127.0.0.1/32`, so it takes the client
|
|
||||||
address nginx passes on and no other. The image:
|
|
||||||
|
|
||||||
- Listens on port 8080 by default (override with `PORT` env var)
|
- Listens on port 8080 by default (override with `PORT` env var)
|
||||||
- Takes the client address from `X-Forwarded-For` only on requests from the
|
- Takes the client address from `X-Forwarded-For` only on requests from the
|
||||||
reverse proxies named in `TRUSTED_PROXIES`, by default from any address in
|
reverse proxies named in `TRUSTED_PROXIES`, and by default from none
|
||||||
`10.0.0.0/8`, `172.16.0.0/12` or `192.168.0.0/16`. `bin/entrypoint.sh` checks
|
|
||||||
each entry with `netwatch-server check-cidr`, which parses it as the backend
|
|
||||||
parses its own `TRUSTED_PROXIES`
|
|
||||||
- Sends access logs to stdout
|
- Sends access logs to stdout
|
||||||
- Caches static assets with immutable headers
|
- Caches static assets with immutable headers
|
||||||
- Sends the security headers `REPO_POLICIES.md` requires on every response, as
|
- Sends the security headers `REPO_POLICIES.md` requires on every response, as
|
||||||
`configs/security-headers.conf` sets them, in place of the backend's own
|
`security-headers.conf` sets them, in place of the backend's own
|
||||||
- Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data`
|
- Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data`
|
||||||
volume. Before the backend starts, the image creates `DATA_DIR` and gives it
|
volume. Before the backend starts, the image creates `DATA_DIR` and gives it
|
||||||
and `/data` to user `netwatch` (uid 1000), which the backend runs as, with
|
and `/data` to user `netwatch` (uid 1000), which the backend runs as, so a
|
||||||
`netwatch-server prepare-data-dir`, which acts on nothing outside `/data`, so
|
host directory bind-mounted at `/data` ends up owned by uid 1000
|
||||||
a host directory bind-mounted at `/data` ends up owned by uid 1000
|
|
||||||
- Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx
|
- Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx
|
||||||
or the backend exits on its own, so the platform restarts it
|
or the backend exits on its own, so the platform restarts it
|
||||||
|
|
||||||
@@ -445,9 +293,9 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
|||||||
|
|
||||||
- **Port:** container port `8080`.
|
- **Port:** container port `8080`.
|
||||||
- **Volume:** container path `/data`; the reports are kept in `/data/reports`.
|
- **Volume:** container path `/data`; the reports are kept in `/data/reports`.
|
||||||
- **Environment variables:** none is required. An empty one counts as unset,
|
- **Environment variables:** none is required. An empty one counts as unset, and
|
||||||
except `TRUSTED_PROXIES`, and one set to a value netwatch cannot use stops the
|
one set to a value netwatch cannot use stops the container at start, with the
|
||||||
container at start, with the reason in its log.
|
reason in its log.
|
||||||
- `PORT`, default `8080`: the container port, from 1 to 65535. `8081` cannot
|
- `PORT`, default `8080`: the container port, from 1 to 65535. `8081` cannot
|
||||||
be used: the backend listens on it inside the container
|
be used: the backend listens on it inside the container
|
||||||
- `REPORTS_PER_MINUTE`, default `60`: reports each client address may send a
|
- `REPORTS_PER_MINUTE`, default `60`: reports each client address may send a
|
||||||
@@ -461,17 +309,16 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
|||||||
in: `/data` or a path below it, with no `.` or `..` part and no extra `/`.
|
in: `/data` or a path below it, with no `.` or `..` part and no extra `/`.
|
||||||
The container also stops if the path goes through a symbolic link that
|
The container also stops if the path goes through a symbolic link that
|
||||||
leads out of `/data` or is written as a full path
|
leads out of `/data` or is written as a full path
|
||||||
- `TRUSTED_PROXIES`, default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`: the
|
- `TRUSTED_PROXIES`, default empty: set it to the address the reverse proxy
|
||||||
addresses the reverse proxy in front of the container connects from, as IP
|
in front of the container connects from, as an IP address or CIDR; several
|
||||||
addresses or CIDRs separated by commas. nginx takes the client address
|
are separated by commas. nginx takes the client address from
|
||||||
from `X-Forwarded-For` only on a request from one of them, and the rate
|
`X-Forwarded-For` only on a request from one of them, and the rate limit
|
||||||
limit counts that address. A list replaces the default. Set and empty,
|
counts that address. Unset, `X-Forwarded-For` is ignored and every client
|
||||||
`X-Forwarded-For` is ignored and every client behind the proxy shares the
|
behind the proxy shares the proxy's one allowance of `REPORTS_PER_MINUTE`.
|
||||||
proxy's one allowance of `REPORTS_PER_MINUTE`. Any client that connects
|
Name only addresses nothing but the proxy connects from: any client that
|
||||||
from one of these addresses can write its own `X-Forwarded-For`, and
|
connects from one can write its own `X-Forwarded-For`, and through a port
|
||||||
through a port Docker publishes, every client may connect from the Docker
|
Docker publishes, every client may connect from the Docker network's
|
||||||
network's gateway, such as `172.17.0.1`, which the default includes; where
|
gateway, such as `172.17.0.1`.
|
||||||
that matters, name only the addresses nothing but the proxy connects from.
|
|
||||||
- `METRICS_USERNAME` and `METRICS_PASSWORD`, default empty: with both set,
|
- `METRICS_USERNAME` and `METRICS_PASSWORD`, default empty: with both set,
|
||||||
the backend records Prometheus metrics of its requests and serves them at
|
the backend records Prometheus metrics of its requests and serves them at
|
||||||
`/metrics` on the container port, to requests with this user name and
|
`/metrics` on the container port, to requests with this user name and
|
||||||
|
|||||||
@@ -16,110 +16,22 @@ Docker image, and the Gitea workflow `.gitea/workflows/check.yml` runs
|
|||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
Decide whether the repo moves to the layout `REPO_POLICIES.md` gives, with
|
||||||
`make check`, as it needs Docker and takes minutes.
|
`backend/` no longer repeating files from the root
|
||||||
|
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)).
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-14: stopping the server waits for a report file write already under
|
|
||||||
way, the periodic flush's or the one `Append` starts when 10 MiB are waiting,
|
|
||||||
before the final flush
|
|
||||||
([#122](https://git.eeqj.de/sneak/netwatch/issues/122)). It used to return at
|
|
||||||
once, and the process exiting cut the write off, losing its reports. A write
|
|
||||||
under way that fails puts its reports back in memory, so the final flush
|
|
||||||
writes them; the stop fails, and the process exits non-zero, only if that
|
|
||||||
flush fails too. Two tests hold the periodic write and the one at 10 MiB open
|
|
||||||
and find the stop waiting until each is released; a third makes the held write
|
|
||||||
fail and finds its report written by the final flush. They run in a
|
|
||||||
`testing/synctest` bubble, so the periodic flush comes without waiting a
|
|
||||||
minute
|
|
||||||
- 2026-10-14: `make test` no longer runs the frontend's production build, which
|
|
||||||
the tests do not need ([#118](https://git.eeqj.de/sneak/netwatch/issues/118)).
|
|
||||||
`yarn build` moved out of the stage that runs the frontend's unit tests, now
|
|
||||||
named `frontend-test`, into a new `frontend-build` stage, which starts from it
|
|
||||||
and, like the `builder` stage, waits on the lint and test phases, so the image
|
|
||||||
still cannot be built unless they pass. The runtime stage copies `dist/` from
|
|
||||||
it, and the `git` that `vite.config.js` reads the commit with is installed
|
|
||||||
there. No build cache was added
|
|
||||||
- 2026-10-14: opening the debug log no longer raises the measured latencies
|
|
||||||
([#131](https://git.eeqj.de/sneak/netwatch/issues/131)). With the panel open,
|
|
||||||
each line written drew the panel again with every line before it, up to 1000,
|
|
||||||
and a check that ends while the page draws is timed only when the drawing is
|
|
||||||
done. Each line is now added at the end of the open panel, its first line is
|
|
||||||
dropped once it holds more than 1000, and it scrolls to its newest line once a
|
|
||||||
frame rather than once a line; the whole panel is drawn only when it is
|
|
||||||
opened. A unit test writes more lines than the panel keeps and checks that the
|
|
||||||
lines already shown are left as they are
|
|
||||||
- 2026-10-14: unset, the backend's own `TRUSTED_PROXIES` trusts `10.0.0.0/8`,
|
|
||||||
`172.16.0.0/12` and `192.168.0.0/16`, and no longer loopback; a list replaces
|
|
||||||
that; set and empty, it trusts no proxy, where it used to get the default, as
|
|
||||||
viper reads an empty environment variable as unset
|
|
||||||
([#129](https://git.eeqj.de/sneak/netwatch/issues/129)). A value it cannot
|
|
||||||
parse still stops the start, and the image still starts the backend with
|
|
||||||
`TRUSTED_PROXIES=127.0.0.1/32`. Tests load the config with it unset, empty and
|
|
||||||
set to a list
|
|
||||||
- 2026-10-14: while report files cannot be written, `POST /api/v1/reports`
|
|
||||||
answers 500, and storing recovers once writing works again, with no restart
|
|
||||||
([#123](https://git.eeqj.de/sneak/netwatch/issues/123)). A failed write puts
|
|
||||||
its reports back in the report buffer, still counted against
|
|
||||||
`DATA_DIR_MAX_BYTES`, where they used to be lost; the next flush, within a
|
|
||||||
minute, tries them again. Until a write succeeds, every report is refused and
|
|
||||||
nothing of it is kept, so the buffer does not grow while writes fail. A test
|
|
||||||
posts reports through the report handler before, during and after a failure;
|
|
||||||
others check that a report appended while a write runs is kept, behind that
|
|
||||||
write's own, when the write fails, and that the cap still holds exactly once a
|
|
||||||
failed write's reports are written
|
|
||||||
- 2026-10-14: a request whose handler panics is logged with status 500 and
|
|
||||||
counted with `code` 500 in the metrics
|
|
||||||
([#125](https://git.eeqj.de/sneak/netwatch/issues/125)), not 200. `Recoverer`
|
|
||||||
is still the first middleware; `Write500OnPanic`, after the request log and
|
|
||||||
after the metrics, writes the 500 as the panic passes and lets it go on to
|
|
||||||
Sentry and `Recoverer`, which now writes a 500 only when no status was written
|
|
||||||
- 2026-10-14: with `TRUSTED_PROXIES` unset, the container's nginx trusts the
|
|
||||||
three private ranges of RFC 1918, `10.0.0.0/8`, `172.16.0.0/12` and
|
|
||||||
`192.168.0.0/16` ([#119](https://git.eeqj.de/sneak/netwatch/issues/119)), as
|
|
||||||
sneak ruled on 2026-09-22 for every app; it trusted no proxy before. A list
|
|
||||||
replaces them, and set and empty it trusts none. The backend inside the
|
|
||||||
container still trusts only `127.0.0.1/32`
|
|
||||||
- 2026-10-14: `POST /api/v1/reports` takes only a report sent with
|
|
||||||
`Content-Type: application/json`, parameters such as `charset` allowed, and
|
|
||||||
refuses any other with 415 and the usual error body, storing nothing
|
|
||||||
([#126](https://git.eeqj.de/sneak/netwatch/issues/126)). A page on another
|
|
||||||
origin can send a `text/plain` POST without a CORS preflight, so any site
|
|
||||||
could make its visitors' browsers store reports; a JSON POST waits for the
|
|
||||||
preflight, which only an origin in `CORS_ALLOWED_ORIGINS` passes. The page
|
|
||||||
already sent `application/json`. A test posts a report as `text/plain`, as
|
|
||||||
`application/x-www-form-urlencoded`, as `multipart/form-data`, with no
|
|
||||||
`Content-Type`, and as `application/json` with and without a charset, and
|
|
||||||
checks which are stored
|
|
||||||
- 2026-10-14: changing the interval no longer drops the samples not yet reported
|
- 2026-10-14: changing the interval no longer drops the samples not yet reported
|
||||||
to the backend ([#124](https://git.eeqj.de/sneak/netwatch/issues/124)). The
|
to the backend ([#124](https://git.eeqj.de/sneak/netwatch/issues/124)). The
|
||||||
interval menu resets each host with the new `HostState.reset`, which moves the
|
interval menu calls the new `HostState.clearHistory`, which moves the
|
||||||
history's entries to `clearedHistory`, keeping the last 100 of all it has
|
history's entries to `clearedHistory`, keeping the last 100 as the history
|
||||||
cleared, and `buildReport` reads both, so the next report has the samples
|
does, and `buildReport` reads both, so the next report has the samples
|
||||||
recorded before the change that no report has delivered. Until its next check,
|
recorded before the change that no report has delivered. The page's history
|
||||||
such a host is sent with the status of its last check, not `pending`. The
|
still starts over, and the per-host marks still leave out of a report the
|
||||||
page's history still starts over, and the per-host marks still leave out of a
|
samples a delivered one had. A unit test sends a report, records samples,
|
||||||
report the samples a delivered one had. Unit tests reset a host as the
|
clears the history as the interval menu does and finds them in the next
|
||||||
interval menu does and find its unreported samples in the next report, and
|
report; `src/main.js` now exports `Reporter` for it
|
||||||
only the last 100 after more were cleared; `web/src/main.js` now exports
|
|
||||||
`Reporter` for them
|
|
||||||
- 2026-10-14: the repo has the layout `REPO_POLICIES.md` gives
|
|
||||||
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). The Go module is at the
|
|
||||||
root (`go.mod`, `go.sum`, `.golangci.yml`, `cmd/netwatch-server/`,
|
|
||||||
`internal/`), so its path `sneak.berlin/go/netwatch` names the repository.
|
|
||||||
`go.mod` ignores `node_modules/`, where npm packages can ship `.go` files that
|
|
||||||
`make tidy` would otherwise read as this module's packages. The page's source
|
|
||||||
is in `web/` (`index.html`, `src/`), which `vite.config.js` makes Vite's root;
|
|
||||||
the build still goes to `dist/`. `nginx.conf` and `security-headers.conf` are
|
|
||||||
in `configs/`. `backend/` is gone: its `LICENSE` and `.dockerignore` are
|
|
||||||
deleted, its `.gitignore` entries are in the root one, its `README.md` is
|
|
||||||
folded into the root `README.md`, and its scripts into the root `script/`:
|
|
||||||
`script/build` builds the server binary after the frontend, `script/run` and
|
|
||||||
`script/clean` came from `backend/script/`, and `make test`, `make lint`,
|
|
||||||
`make fmt` and `make fmt-check` do what its `test`, `lint`, `fmt` and
|
|
||||||
`fmt-check` did. `script/bootstrap` no longer installs gcc, which only the
|
|
||||||
host Go tests needed. The image and the page are unchanged
|
|
||||||
- 2026-10-07: the files shared from `sneak/prompts` are its copies at commit
|
- 2026-10-07: the files shared from `sneak/prompts` are its copies at commit
|
||||||
`dd4027b` ([#113](https://git.eeqj.de/sneak/netwatch/issues/113)), with this
|
`dd4027b` ([#113](https://git.eeqj.de/sneak/netwatch/issues/113)), with this
|
||||||
repository's own entries after the shared content in `.gitignore`,
|
repository's own entries after the shared content in `.gitignore`,
|
||||||
@@ -514,6 +426,8 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
|||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
|
- Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
||||||
|
`make check`, as it needs Docker and takes minutes
|
||||||
- A backend test that posts a report to `POST /api/v1/reports` and checks the
|
- A backend test that posts a report to `POST /api/v1/reports` and checks the
|
||||||
compressed file it is written to
|
compressed file it is written to
|
||||||
- A backend route that decompresses the stored reports and answers queries on
|
- A backend route that decompresses the stored reports and answers queries on
|
||||||
@@ -523,9 +437,3 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
|||||||
- A configurable host list (an environment variable or a config file)
|
- A configurable host list (an environment variable or a config file)
|
||||||
- Export of the latency history (CSV or JSON)
|
- Export of the latency history (CSV or JSON)
|
||||||
- A notification when the health status changes to DEGRADED
|
- A notification when the health status changes to DEGRADED
|
||||||
- `make test` within the 20-second target
|
|
||||||
([#118](https://git.eeqj.de/sneak/netwatch/issues/118)): every run compiles
|
|
||||||
the Go standard library and every dependency with the race detector from
|
|
||||||
nothing. Whether test builds may keep Go's compiled packages between runs
|
|
||||||
waits on sneak's ruling in
|
|
||||||
[prompts#113](https://git.eeqj.de/sneak/prompts/issues/113)
|
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
node_modules
|
||||||
|
.DS_Store
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
/netwatch-server
|
||||||
|
*.log
|
||||||
|
*.out
|
||||||
|
*.test
|
||||||
|
.env
|
||||||
|
data/
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 @sneak (https://sneak.berlin)
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# Thin shims; the implementations live in backend/script/ (see the
|
||||||
|
# Entrypoints section of README.md). There is no check, hooks or docker
|
||||||
|
# target here: the root Makefile's check covers this directory, its
|
||||||
|
# hooks target installs the repo's only pre-commit hook, and its docker
|
||||||
|
# target builds the one image, which contains this backend.
|
||||||
|
|
||||||
|
.PHONY: all build test lint fmt fmt-check run clean
|
||||||
|
|
||||||
|
all: build
|
||||||
|
|
||||||
|
build:
|
||||||
|
@script/build
|
||||||
|
|
||||||
|
test:
|
||||||
|
@script/test
|
||||||
|
|
||||||
|
lint:
|
||||||
|
@script/lint
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
@script/fmt
|
||||||
|
|
||||||
|
fmt-check:
|
||||||
|
@script/fmt-check
|
||||||
|
|
||||||
|
run:
|
||||||
|
@script/run
|
||||||
|
|
||||||
|
clean:
|
||||||
|
@script/clean
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
netwatch-server is an MIT-licensed Go HTTP backend by
|
||||||
|
[@sneak](https://sneak.berlin) that receives telemetry reports from the NetWatch
|
||||||
|
SPA and persists them as zstd-compressed JSONL files on disk.
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
From this directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build and run locally
|
||||||
|
make run
|
||||||
|
```
|
||||||
|
|
||||||
|
From the repo root, whose `Dockerfile` builds the one image that ships this
|
||||||
|
backend behind nginx (see [Container image](#container-image)):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run tests, lint, and format check over the frontend and this backend
|
||||||
|
make check
|
||||||
|
|
||||||
|
# Build the image: nginx, the frontend and this backend
|
||||||
|
make docker
|
||||||
|
docker run -p 8080:8080 netwatch
|
||||||
|
```
|
||||||
|
|
||||||
|
## Entrypoints
|
||||||
|
|
||||||
|
This directory follows the same
|
||||||
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
|
pattern as the repo root: the targets in `backend/Makefile` are thin shims over
|
||||||
|
`backend/script/`. The root `Dockerfile` runs `build`, and the root scripts call
|
||||||
|
`fmt` and `fmt-check`:
|
||||||
|
|
||||||
|
- `script/build` — compile the static `netwatch-server` binary with its version
|
||||||
|
stamped in. The version is `VERSION` from the environment; when that is unset
|
||||||
|
or empty, it falls back to `git describe` inside a git checkout, then to `dev`
|
||||||
|
- `script/test` — run the Go tests on the host with the race detector and
|
||||||
|
coverage; the root `make test` runs them in the `test` phase of the root
|
||||||
|
`Dockerfile`. Go's `-timeout 90s` bounds the tests, not their compile, and
|
||||||
|
`-count=1` keeps Go from reporting a stored pass. If they fail, they run again
|
||||||
|
with `-v` for the details, and the script fails. The race detector needs a C
|
||||||
|
compiler
|
||||||
|
- `script/lint` — run the root `script/lint`, which builds the `lint` phase of
|
||||||
|
the root `Dockerfile`: golangci-lint over this directory, and eslint over the
|
||||||
|
frontend. golangci-lint never runs on the host
|
||||||
|
- `script/fmt` — format the Go sources (writes)
|
||||||
|
- `script/fmt-check` — check Go formatting (read-only)
|
||||||
|
- `script/run` — build and run the server locally
|
||||||
|
- `script/clean` — remove build artifacts
|
||||||
|
|
||||||
|
There is no `check`, `hooks` or `docker` target here: the root `make check`
|
||||||
|
covers this directory, the root `make hooks` installs the repo's only pre-commit
|
||||||
|
hook, and the root `make docker` builds the image that contains this backend.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
The NetWatch frontend collects latency measurements from the browser but has no
|
||||||
|
way to persist or aggregate them. This backend provides a minimal
|
||||||
|
`POST /api/v1/reports` endpoint that buffers incoming reports in memory and
|
||||||
|
flushes them to compressed files on disk for later analysis.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
The server is structured as an `fx`-wired Go application under
|
||||||
|
`cmd/netwatch-server/`. Internal packages in `internal/` follow standard Go
|
||||||
|
project layout:
|
||||||
|
|
||||||
|
- **`config`**: Loads configuration from environment variables and config files
|
||||||
|
via Viper.
|
||||||
|
- **`handlers`**: HTTP request handlers for the API (health check, report
|
||||||
|
ingestion).
|
||||||
|
- **`reportbuf`**: In-memory buffer that accumulates JSONL report lines and
|
||||||
|
flushes to zstd-compressed files when the buffer reaches 10 MiB or every 60
|
||||||
|
seconds.
|
||||||
|
- **`server`**: Chi-based HTTP server with middleware wiring and route
|
||||||
|
registration.
|
||||||
|
- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting
|
||||||
|
infrastructure.
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface |
|
||||||
|
| `PORT` | `8080` | HTTP listen port |
|
||||||
|
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
||||||
|
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) |
|
||||||
|
| `DEBUG` | `false` | Enable debug logging |
|
||||||
|
| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
||||||
|
| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) |
|
||||||
|
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
||||||
|
| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) |
|
||||||
|
| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) |
|
||||||
|
| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) |
|
||||||
|
|
||||||
|
`TRUSTED_PROXIES` defaults to
|
||||||
|
`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback
|
||||||
|
entries cover a reverse proxy on the same host. A request whose direct peer is
|
||||||
|
outside this set has its forwarded headers ignored, and the direct peer is
|
||||||
|
logged and rate-limited instead. The container image does not use this default;
|
||||||
|
see [Container image](#container-image).
|
||||||
|
|
||||||
|
A variable set to a value the server cannot use, such as `PORT=abc`,
|
||||||
|
`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from
|
||||||
|
starting, with an error naming the variable. An empty variable counts as unset.
|
||||||
|
|
||||||
|
### Container image
|
||||||
|
|
||||||
|
The root `Dockerfile` builds one image in which nginx listens on the public port
|
||||||
|
8080, serves the frontend, and proxies `/api/`, `/.well-known/healthcheck` and
|
||||||
|
`/metrics` to this server. The image's entrypoint, `bin/entrypoint.sh`, starts
|
||||||
|
the server as user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and
|
||||||
|
`PORT=8081`, so only nginx reaches it, and with `TRUSTED_PROXIES=127.0.0.1/32`,
|
||||||
|
so it takes the client address nginx passes on and no other. `DATA_DIR` is
|
||||||
|
`/data/reports`, on the `/data` volume; before starting the server, the
|
||||||
|
entrypoint creates it and gives it and `/data` to `netwatch` with
|
||||||
|
`netwatch-server prepare-data-dir`, which acts on nothing outside `/data`. nginx
|
||||||
|
replaces the security headers this server sets with those in the root
|
||||||
|
`security-headers.conf`, so those are what clients of the image see.
|
||||||
|
|
||||||
|
The container's own `TRUSTED_PROXIES` goes to nginx instead: IP addresses or
|
||||||
|
CIDRs, separated by commas, of the reverse proxies in front of the container.
|
||||||
|
nginx takes the client address from `X-Forwarded-For` only on a request from one
|
||||||
|
of them. Unset or empty, nginx trusts no proxy, and the client address is the
|
||||||
|
one each request comes from, so every client behind a proxy shares one rate
|
||||||
|
limit. An entry that is not an IP address or CIDR, such as a hostname or
|
||||||
|
`1.2.3`, stops the container at start with an error naming `TRUSTED_PROXIES`:
|
||||||
|
the entrypoint checks each entry with `netwatch-server check-cidr`, which parses
|
||||||
|
it as this server parses its own `TRUSTED_PROXIES`.
|
||||||
|
|
||||||
|
### Report storage
|
||||||
|
|
||||||
|
Reports are written as `reports-<timestamp>-<number>.jsonl.zst` files in
|
||||||
|
`DATA_DIR`. The timestamp is in UTC to the millisecond, so the names sort by
|
||||||
|
time. The number starts at 1 when the server starts and goes up by one for each
|
||||||
|
file the server starts to write, so two files written in the same millisecond
|
||||||
|
still get different names. A failed write uses up its number and leaves a gap in
|
||||||
|
the numbers: its file, if it was created, is removed. The file stays, counted
|
||||||
|
toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too.
|
||||||
|
Each file contains one JSON object per line, compressed with zstd. Files are
|
||||||
|
created with `O_EXCL` to prevent overwrites.
|
||||||
|
|
||||||
|
### Report limits
|
||||||
|
|
||||||
|
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
||||||
|
credentials, so it is bounded instead. Both refusals below answer with the same
|
||||||
|
`{"status":"error"}` body as any other error.
|
||||||
|
|
||||||
|
- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may
|
||||||
|
send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with
|
||||||
|
`Retry-After: 60`. The minute slides: reports from the minute before still
|
||||||
|
count, fading out over the current one, so an address is sure never to be
|
||||||
|
refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60
|
||||||
|
seconds. The page sends one report a minute from each open tab, so the default
|
||||||
|
of 60 refuses nothing from up to 30 tabs behind one address, such as a
|
||||||
|
household or an office sharing it, however their reports bunch up. Report
|
||||||
|
responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and
|
||||||
|
`X-RateLimit-Reset` headers.
|
||||||
|
- **Size cap.** The report files in `DATA_DIR` may total at most
|
||||||
|
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
|
||||||
|
waiting in memory count at their uncompressed size until they are written;
|
||||||
|
those lost to a failed write stop counting, and the part of its file written
|
||||||
|
is removed. When a report would take the total past the cap, the oldest report
|
||||||
|
files are deleted to make room, and each deletion is logged with the file's
|
||||||
|
name and size; a file still being written is never deleted. A report is
|
||||||
|
refused with 507, and nothing of it is stored, only when the reports waiting
|
||||||
|
to be written fill the cap on their own, and then no file is deleted. At
|
||||||
|
start, report files past the cap, as after lowering it, are deleted the same
|
||||||
|
way. So the cap is how much of the newest reports is kept: the default of 1
|
||||||
|
GiB is small enough for any host; set it to the space you can give `DATA_DIR`.
|
||||||
|
|
||||||
|
### CORS
|
||||||
|
|
||||||
|
The page calls the API from the origin it is served from, so by default the
|
||||||
|
server sends no CORS headers, and browsers let no other origin's pages call it.
|
||||||
|
To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS`
|
||||||
|
(for example `https://netwatch.example.com`); pages from a listed origin may
|
||||||
|
`GET` and `POST` with a `Content-Type` header. Each entry must be a plain
|
||||||
|
origin, `scheme://host` with an optional `:port`, as browsers send it: no path,
|
||||||
|
not even a trailing `/`, and no `*`. Any other entry stops the server from
|
||||||
|
starting, with an error naming `CORS_ALLOWED_ORIGINS`.
|
||||||
|
|
||||||
|
### Metrics
|
||||||
|
|
||||||
|
With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves
|
||||||
|
Prometheus metrics at `GET /metrics` to requests with those as their basic auth
|
||||||
|
credentials, and answers any other with 401. For each request that reaches the
|
||||||
|
health check or `POST /api/v1/reports`, those the rate limit refuses included,
|
||||||
|
the metrics record its duration and response size, labelled with its path,
|
||||||
|
method and status; they also count those requests in progress, and include Go's
|
||||||
|
runtime and process metrics. No other request is recorded: not those to
|
||||||
|
`/metrics` itself, and not those answered before they reach either route, such
|
||||||
|
as a CORS preflight, or a request refused with 404 for a path no route has, 405
|
||||||
|
for a method its route does not take, or 413 for declaring a body length over
|
||||||
|
the 1 MiB limit. A report whose body goes over the limit without declaring its
|
||||||
|
length reaches the route, is answered 413 there, and is recorded with that
|
||||||
|
status. Clients can make up any number of paths and methods, and each would add
|
||||||
|
labels to the metrics for as long as the server runs. With neither set, nothing
|
||||||
|
is recorded and `/metrics` answers 404. One without the other stops the server
|
||||||
|
from starting, with an error naming both; so does a `METRICS_USERNAME`
|
||||||
|
containing `:`, which basic auth cannot carry, with an error naming it.
|
||||||
|
|
||||||
|
`/metrics` is rate limited, so that its password cannot be guessed quickly: each
|
||||||
|
client address, resolved through `TRUSTED_PROXIES`, may make 60 requests to it a
|
||||||
|
minute, whatever their credentials. Past that it gets 429 with
|
||||||
|
`Retry-After: 60`, and its credentials are not checked. The minute slides as it
|
||||||
|
does for reports (see [Report limits](#report-limits)), so a scraper polling
|
||||||
|
every 2 seconds or less often is never refused. This allowance is apart from the
|
||||||
|
one for reports.
|
||||||
|
|
||||||
|
### Sentry
|
||||||
|
|
||||||
|
With `SENTRY_DSN` set, the server sends its errors to that Sentry project: each
|
||||||
|
panic in a handler is reported there, under the release `netwatch-server-`
|
||||||
|
followed by the server's version, and the request still gets 500 from the
|
||||||
|
server's panic recovery. On shutdown the server waits up to 2 seconds for Sentry
|
||||||
|
to finish sending. A DSN Sentry refuses stops the server from starting, with an
|
||||||
|
error naming `SENTRY_DSN`. With it empty, Sentry is not set up, and nothing is
|
||||||
|
sent to it.
|
||||||
|
|
||||||
|
## TODO
|
||||||
|
|
||||||
|
The to-do list, this backend's open work included, is [TODO.md](../TODO.md) at
|
||||||
|
the repo root.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT. See [LICENSE](LICENSE).
|
||||||
|
|
||||||
|
## Author
|
||||||
|
|
||||||
|
[@sneak](https://sneak.berlin)
|
||||||
@@ -2,9 +2,6 @@ module sneak.berlin/go/netwatch
|
|||||||
|
|
||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
// npm packages can ship .go files, which ./... and all would include.
|
|
||||||
ignore ./node_modules
|
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||||
github.com/getsentry/sentry-go v0.49.0
|
github.com/getsentry/sentry-go v0.49.0
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -22,14 +21,16 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// defaultTrustedProxies lists the networks whose forwarded
|
// defaultTrustedProxies lists the networks whose forwarded
|
||||||
// headers are honoured when TRUSTED_PROXIES is unset: the
|
// headers are honoured by default: IPv4 and IPv6 loopback,
|
||||||
// RFC 1918 ranges. The container image does not use it:
|
// for a reverse proxy on the same host, and the RFC1918
|
||||||
|
// ranges. The container image does not use it:
|
||||||
// bin/entrypoint.sh gives the server 127.0.0.1/32, since
|
// bin/entrypoint.sh gives the server 127.0.0.1/32, since
|
||||||
// nginx is its only client there.
|
// nginx is its only client there.
|
||||||
const defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
const defaultTrustedProxies = "127.0.0.1/32,::1/128," +
|
||||||
|
"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||||
|
|
||||||
// Default limits on stored reports; README.md gives the reasons for
|
// Default limits on stored reports; backend/README.md gives the
|
||||||
// these values.
|
// reasons for these values.
|
||||||
const (
|
const (
|
||||||
defaultReportsPerMinute = 60
|
defaultReportsPerMinute = 60
|
||||||
defaultDataDirMaxBytes = 1 << 30 // 1 GiB
|
defaultDataDirMaxBytes = 1 << 30 // 1 GiB
|
||||||
@@ -134,14 +135,6 @@ func New(
|
|||||||
viper.GetString("DEBUG"), errNotBool)
|
viper.GetString("DEBUG"), errNotBool)
|
||||||
}
|
}
|
||||||
|
|
||||||
// An empty TRUSTED_PROXIES trusts no proxy, but viper reads an
|
|
||||||
// empty environment variable as unset and would give the default,
|
|
||||||
// so the environment is read first.
|
|
||||||
trustedProxies, set := os.LookupEnv("TRUSTED_PROXIES")
|
|
||||||
if !set {
|
|
||||||
trustedProxies = viper.GetString("TRUSTED_PROXIES")
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &Config{
|
s := &Config{
|
||||||
BindAddress: viper.GetString("BIND_ADDRESS"),
|
BindAddress: viper.GetString("BIND_ADDRESS"),
|
||||||
CORSAllowedOrigins: splitList(viper.GetString("CORS_ALLOWED_ORIGINS")),
|
CORSAllowedOrigins: splitList(viper.GetString("CORS_ALLOWED_ORIGINS")),
|
||||||
@@ -153,7 +146,7 @@ func New(
|
|||||||
Port: port,
|
Port: port,
|
||||||
ReportsPerMinute: viper.GetInt("REPORTS_PER_MINUTE"),
|
ReportsPerMinute: viper.GetInt("REPORTS_PER_MINUTE"),
|
||||||
SentryDSN: viper.GetString("SENTRY_DSN"),
|
SentryDSN: viper.GetString("SENTRY_DSN"),
|
||||||
TrustedProxies: splitList(trustedProxies),
|
TrustedProxies: splitList(viper.GetString("TRUSTED_PROXIES")),
|
||||||
log: log,
|
log: log,
|
||||||
params: ¶ms,
|
params: ¶ms,
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -31,62 +29,6 @@ func requireConfigError(t *testing.T, setting string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// requireTrustedProxies builds the config as main does and fails the
|
|
||||||
// test unless its TrustedProxies are want.
|
|
||||||
func requireTrustedProxies(t *testing.T, want []string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var cfg *config.Config
|
|
||||||
|
|
||||||
app := fx.New(
|
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(globals.New, logger.New, config.New),
|
|
||||||
fx.Populate(&cfg),
|
|
||||||
)
|
|
||||||
|
|
||||||
err := app.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("config error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(cfg.TrustedProxies, want) {
|
|
||||||
t.Fatalf("TrustedProxies = %q, want %q", cfg.TrustedProxies, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTrustedProxiesUnsetTrustsRFC1918: unset, the server trusts the
|
|
||||||
// RFC 1918 ranges and nothing else, loopback included.
|
|
||||||
func TestTrustedProxiesUnsetTrustsRFC1918(t *testing.T) {
|
|
||||||
// t.Setenv first, so the variable is put back as it was after the
|
|
||||||
// test.
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "")
|
|
||||||
|
|
||||||
err := os.Unsetenv("TRUSTED_PROXIES")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
requireTrustedProxies(t,
|
|
||||||
[]string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTrustedProxiesEmptyTrustsNothing: set and empty, the server
|
|
||||||
// trusts no proxy. viper alone would read the empty variable as unset
|
|
||||||
// and give the default.
|
|
||||||
func TestTrustedProxiesEmptyTrustsNothing(t *testing.T) {
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "")
|
|
||||||
|
|
||||||
requireTrustedProxies(t, []string{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTrustedProxiesListReplacesDefault: a list is used as given, with
|
|
||||||
// none of the default ranges added.
|
|
||||||
func TestTrustedProxiesListReplacesDefault(t *testing.T) {
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "172.17.0.1/32, 2001:db8::/32")
|
|
||||||
|
|
||||||
requireTrustedProxies(t, []string{"172.17.0.1/32", "2001:db8::/32"})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSettingsLoadAsGiven: valid values pass the checks and are used
|
// TestSettingsLoadAsGiven: valid values pass the checks and are used
|
||||||
// as given. bin/entrypoint.sh starts the server with these
|
// as given. bin/entrypoint.sh starts the server with these
|
||||||
// BIND_ADDRESS and PORT values.
|
// BIND_ADDRESS and PORT values.
|
||||||
@@ -5,9 +5,7 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"mime"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"sneak.berlin/go/netwatch/internal/globals"
|
"sneak.berlin/go/netwatch/internal/globals"
|
||||||
@@ -20,10 +18,6 @@ import (
|
|||||||
|
|
||||||
const jsonContentType = "application/json; charset=utf-8"
|
const jsonContentType = "application/json; charset=utf-8"
|
||||||
|
|
||||||
// errNotJSON is what decodeJSON returns for a request whose
|
|
||||||
// Content-Type is not application/json.
|
|
||||||
var errNotJSON = errors.New("content type is not application/json")
|
|
||||||
|
|
||||||
// reportAppender is the subset of the report buffer the handlers
|
// reportAppender is the subset of the report buffer the handlers
|
||||||
// depend on. Defining it here keeps the storage failure path
|
// depend on. Defining it here keeps the storage failure path
|
||||||
// exercisable with a stub in tests.
|
// exercisable with a stub in tests.
|
||||||
@@ -86,25 +80,14 @@ func (s *Handlers) respondJSON(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// decodeJSON decodes the request body into v. It returns errNotJSON,
|
// decodeJSON decodes the request body into v. The body is
|
||||||
// reading nothing, unless the request's Content-Type is
|
// expected to already be bounded by the body-size middleware, so
|
||||||
// application/json, with or without parameters such as charset. The
|
// a caller can distinguish an over-limit body from malformed
|
||||||
// body is expected to already be bounded by the body-size middleware,
|
// JSON by testing the returned error for *http.MaxBytesError.
|
||||||
// so a caller can distinguish an over-limit body from malformed JSON
|
|
||||||
// by testing the returned error for *http.MaxBytesError.
|
|
||||||
func (s *Handlers) decodeJSON(
|
func (s *Handlers) decodeJSON(
|
||||||
_ http.ResponseWriter,
|
_ http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
v any,
|
v any,
|
||||||
) error {
|
) error {
|
||||||
// A page on another origin can POST text/plain or a form without
|
|
||||||
// a CORS preflight, so its visitors' browsers would send it
|
|
||||||
// whatever the CORS settings. A JSON POST waits for the
|
|
||||||
// preflight, so only JSON is taken.
|
|
||||||
mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
|
||||||
if err != nil || mediaType != "application/json" {
|
|
||||||
return errNotJSON
|
|
||||||
}
|
|
||||||
|
|
||||||
return json.NewDecoder(r.Body).Decode(v)
|
return json.NewDecoder(r.Body).Decode(v)
|
||||||
}
|
}
|
||||||
@@ -30,7 +30,7 @@ type report struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// HandleReport returns a handler that accepts telemetry
|
// HandleReport returns a handler that accepts telemetry
|
||||||
// reports from NetWatch clients, sent as application/json.
|
// reports from NetWatch clients.
|
||||||
func (s *Handlers) HandleReport() http.HandlerFunc {
|
func (s *Handlers) HandleReport() http.HandlerFunc {
|
||||||
type response struct {
|
type response struct {
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
@@ -66,15 +66,9 @@ func (s *Handlers) HandleReport() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// decodeErrorStatus logs a report decode failure and returns the
|
// decodeErrorStatus logs a report decode failure and returns the
|
||||||
// status to send: 415 when the report was not sent as JSON, 413 when
|
// status to send: 413 when the body exceeded the size limit,
|
||||||
// the body exceeded the size limit, otherwise 400 for malformed JSON.
|
// otherwise 400 for malformed JSON.
|
||||||
func (s *Handlers) decodeErrorStatus(err error) int {
|
func (s *Handlers) decodeErrorStatus(err error) int {
|
||||||
if errors.Is(err, errNotJSON) {
|
|
||||||
s.log.Warn("report refused: Content-Type is not application/json")
|
|
||||||
|
|
||||||
return http.StatusUnsupportedMediaType
|
|
||||||
}
|
|
||||||
|
|
||||||
var tooLarge *http.MaxBytesError
|
var tooLarge *http.MaxBytesError
|
||||||
if errors.As(err, &tooLarge) {
|
if errors.As(err, &tooLarge) {
|
||||||
s.log.Warn("report body too large", "limit_bytes", tooLarge.Limit)
|
s.log.Warn("report body too large", "limit_bytes", tooLarge.Limit)
|
||||||
@@ -93,7 +87,7 @@ func (s *Handlers) decodeErrorStatus(err error) int {
|
|||||||
|
|
||||||
// appendErrorStatus logs a failure to store a report and returns
|
// appendErrorStatus logs a failure to store a report and returns
|
||||||
// the status to send: 507 when the reports waiting to be written fill
|
// the status to send: 507 when the reports waiting to be written fill
|
||||||
// the size cap, otherwise 500, as when report files cannot be written.
|
// the size cap, otherwise 500.
|
||||||
func (s *Handlers) appendErrorStatus(err error) int {
|
func (s *Handlers) appendErrorStatus(err error) int {
|
||||||
if errors.Is(err, reportbuf.ErrFull) {
|
if errors.Is(err, reportbuf.ErrFull) {
|
||||||
s.log.Warn("report refused: " +
|
s.log.Warn("report refused: " +
|
||||||
@@ -27,17 +27,6 @@ type stubAppender struct {
|
|||||||
|
|
||||||
func (s stubAppender) Append(any) error { return s.err }
|
func (s stubAppender) Append(any) error { return s.err }
|
||||||
|
|
||||||
// recordingAppender keeps every report the handler stores.
|
|
||||||
type recordingAppender struct {
|
|
||||||
reports []any
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *recordingAppender) Append(v any) error {
|
|
||||||
a.reports = append(a.reports, v)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestHandlers(buf stubAppender, out io.Writer) *handlers.Handlers {
|
func newTestHandlers(buf stubAppender, out io.Writer) *handlers.Handlers {
|
||||||
return handlers.NewForTest(buf, slog.New(slog.NewJSONHandler(out, nil)))
|
return handlers.NewForTest(buf, slog.New(slog.NewJSONHandler(out, nil)))
|
||||||
}
|
}
|
||||||
@@ -94,7 +83,6 @@ func TestHandleReportAcceptsValidReports(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(tt.body),
|
strings.NewReader(tt.body),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -115,89 +103,6 @@ func TestHandleReportAcceptsValidReports(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleReportStoresOnlyJSON checks that a report is stored only
|
|
||||||
// when sent as application/json, as the page sends it. One sent as
|
|
||||||
// text/plain or as a form, which a page on another origin can send
|
|
||||||
// without a CORS preflight, is refused with 415.
|
|
||||||
func TestHandleReportStoresOnlyJSON(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
contentType string
|
|
||||||
wantStatus int
|
|
||||||
wantStored int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "text/plain",
|
|
||||||
contentType: "text/plain;charset=UTF-8",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a urlencoded form",
|
|
||||||
contentType: "application/x-www-form-urlencoded",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a multipart form",
|
|
||||||
contentType: "multipart/form-data; boundary=x",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no Content-Type",
|
|
||||||
contentType: "",
|
|
||||||
wantStatus: http.StatusUnsupportedMediaType,
|
|
||||||
wantStored: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "application/json",
|
|
||||||
contentType: "application/json",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantStored: 1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "application/json with a charset",
|
|
||||||
contentType: "application/json;charset=UTF-8",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantStored: 1,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var stored recordingAppender
|
|
||||||
|
|
||||||
h := handlers.NewForTest(&stored, slog.New(slog.DiscardHandler))
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(),
|
|
||||||
http.MethodPost, "/api/v1/reports",
|
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
|
||||||
)
|
|
||||||
|
|
||||||
if tt.contentType != "" {
|
|
||||||
req.Header.Set("Content-Type", tt.contentType)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != tt.wantStatus {
|
|
||||||
t.Errorf("status = %d, want %d", rec.Code, tt.wantStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(stored.reports) != tt.wantStored {
|
|
||||||
t.Errorf("reports stored = %d, want %d",
|
|
||||||
len(stored.reports), tt.wantStored)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -208,7 +113,6 @@ func TestHandleReportStorageFailureIsNon2xx(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -234,7 +138,6 @@ func TestHandleReportFullIs507(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -258,7 +161,6 @@ func TestHandleReportMalformedJSONIs400(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{not json`),
|
strings.NewReader(`{not json`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -283,8 +185,6 @@ func TestHandleReportOversizeIs413(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"`+strings.Repeat("x", 200)+`"}`),
|
strings.NewReader(`{"clientId":"`+strings.Repeat("x", 200)+`"}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
// No declared length, so only the middleware's read cap can
|
// No declared length, so only the middleware's read cap can
|
||||||
// stop this body.
|
// stop this body.
|
||||||
req.ContentLength = -1
|
req.ContentLength = -1
|
||||||
@@ -313,7 +213,6 @@ func TestHandleReportDoesNotLogRawGeo(t *testing.T) {
|
|||||||
`{"clientId":"c1","geo":{"raw":"`+sentinel+`"},"hosts":[]}`,
|
`{"clientId":"c1","geo":{"raw":"`+sentinel+`"},"hosts":[]}`,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -346,7 +245,6 @@ func TestHandleReportLogsClientIDCutToBound(t *testing.T) {
|
|||||||
`{"clientId":"`+long+`","timestamp":"`+long+`","hosts":[]}`,
|
`{"clientId":"`+long+`","timestamp":"`+long+`","hosts":[]}`,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -386,7 +284,6 @@ func TestHandleReportDecodeErrorLogIsBounded(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"hosts":[{"history":[{"t":`+huge+`}]}]}`),
|
strings.NewReader(`{"hosts":[{"history":[{"t":`+huge+`}]}]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
h.HandleReport().ServeHTTP(rec, req)
|
||||||
|
|
||||||
@@ -297,16 +297,12 @@ func (s *Middleware) MaxBodyBytes(
|
|||||||
|
|
||||||
// Recoverer returns middleware that recovers from a panic in a
|
// Recoverer returns middleware that recovers from a panic in a
|
||||||
// downstream handler, logs the panic and stack trace through
|
// downstream handler, logs the panic and stack trace through
|
||||||
// slog, and responds 500 with no body, unless a status was already
|
// slog, and responds 500 with no body. http.ErrAbortHandler is
|
||||||
// written, as Write500OnPanic writes one for a handler's panic.
|
// re-panicked so the server can abort the response as intended.
|
||||||
// http.ErrAbortHandler is re-panicked so the server can abort the
|
|
||||||
// response as intended.
|
|
||||||
func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
|
func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
|
||||||
return func(next http.Handler) http.Handler {
|
return func(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(
|
return http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
rec := recover()
|
rec := recover()
|
||||||
if rec == nil {
|
if rec == nil {
|
||||||
@@ -324,43 +320,10 @@ func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
|
|||||||
"stack", string(debug.Stack()),
|
"stack", string(debug.Stack()),
|
||||||
)
|
)
|
||||||
|
|
||||||
if ww.Status() == 0 {
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
ww.WriteHeader(http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
next.ServeHTTP(ww, r)
|
next.ServeHTTP(w, r)
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write500OnPanic returns middleware that writes 500 when a downstream
|
|
||||||
// handler panics before writing a status, and lets the panic go on to
|
|
||||||
// Sentry and Recoverer. Recoverer, which comes first, would write its
|
|
||||||
// 500 only after the request log and the metrics had recorded the
|
|
||||||
// status they saw; placed after each of them, this makes them record
|
|
||||||
// the 500.
|
|
||||||
func (s *Middleware) Write500OnPanic() func(http.Handler) http.Handler {
|
|
||||||
return func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
rec := recover()
|
|
||||||
if rec == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if ww.Status() == 0 {
|
|
||||||
ww.WriteHeader(http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
|
|
||||||
panic(rec)
|
|
||||||
}()
|
|
||||||
|
|
||||||
next.ServeHTTP(ww, r)
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
-169
@@ -4,7 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"log"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
@@ -17,9 +16,7 @@ import (
|
|||||||
"sneak.berlin/go/netwatch/internal/logger"
|
"sneak.berlin/go/netwatch/internal/logger"
|
||||||
"sneak.berlin/go/netwatch/internal/middleware"
|
"sneak.berlin/go/netwatch/internal/middleware"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -326,172 +323,6 @@ func TestRecovererRepanicsOnAbortHandler(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandlerPanicIsLoggedAndCountedAs500 sends a request to a handler
|
|
||||||
// that panics through the middleware SetupRoutes puts around a route
|
|
||||||
// with metrics, in its order, leaving out those that write no status
|
|
||||||
// for a panic and Sentry, which TestSentry covers. The request log line
|
|
||||||
// and the metrics must give the 500 the client gets, and the panic must
|
|
||||||
// still reach Recoverer, which logs it.
|
|
||||||
func TestHandlerPanicIsLoggedAndCountedAs500(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logbuf bytes.Buffer
|
|
||||||
|
|
||||||
mw := middleware.NewWithLogger(
|
|
||||||
slog.New(slog.NewJSONHandler(&logbuf, nil)),
|
|
||||||
)
|
|
||||||
registry := prometheus.NewRegistry()
|
|
||||||
|
|
||||||
handler := chi.Chain(
|
|
||||||
mw.Recoverer(),
|
|
||||||
chimiddleware.RequestID,
|
|
||||||
mw.Logging(),
|
|
||||||
mw.Write500OnPanic(),
|
|
||||||
mw.Metrics(registry),
|
|
||||||
mw.Write500OnPanic(),
|
|
||||||
).HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
|
||||||
panic("boom")
|
|
||||||
})
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", http.NoBody)
|
|
||||||
handler.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusInternalServerError {
|
|
||||||
t.Fatalf("status = %d, want %d",
|
|
||||||
rec.Code, http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
|
|
||||||
request := findLogLine(t, logbuf.Bytes(), "request")
|
|
||||||
if request.Status != http.StatusInternalServerError {
|
|
||||||
t.Errorf("request log status = %d, want %d",
|
|
||||||
request.Status, http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
|
|
||||||
recovered := findLogLine(t, logbuf.Bytes(), "panic recovered")
|
|
||||||
if recovered.Panic != "boom" {
|
|
||||||
t.Errorf("panic recovered line has panic %q, want %q",
|
|
||||||
recovered.Panic, "boom")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"http_request_duration_seconds", "http_response_size_bytes",
|
|
||||||
} {
|
|
||||||
codes := metricCodes(t, registry, name)
|
|
||||||
if len(codes) != 1 || codes[0] != "500" {
|
|
||||||
t.Errorf("%s code labels = %v, want [500]", name, codes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// loggedLine holds the fields the tests read from a JSON log line.
|
|
||||||
type loggedLine struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Status int `json:"status"`
|
|
||||||
Panic string `json:"panic"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// findLogLine returns the first of the JSON log lines in logged whose
|
|
||||||
// message is msg.
|
|
||||||
func findLogLine(t *testing.T, logged []byte, msg string) loggedLine {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dec := json.NewDecoder(bytes.NewReader(logged))
|
|
||||||
|
|
||||||
for {
|
|
||||||
var line loggedLine
|
|
||||||
|
|
||||||
err := dec.Decode(&line)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("no %q line in the log: %v (%q)", msg, err, logged)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line.Msg == msg {
|
|
||||||
return line
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// metricCodes returns the code label of each series of the metric
|
|
||||||
// family name in registry.
|
|
||||||
func metricCodes(
|
|
||||||
t *testing.T,
|
|
||||||
registry *prometheus.Registry,
|
|
||||||
name string,
|
|
||||||
) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
families, err := registry.Gather()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("gather metrics: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var codes []string
|
|
||||||
|
|
||||||
for _, family := range families {
|
|
||||||
if family.GetName() != name {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, metric := range family.GetMetric() {
|
|
||||||
for _, label := range metric.GetLabel() {
|
|
||||||
if label.GetName() == "code" {
|
|
||||||
codes = append(codes, label.GetValue())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return codes
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecovererWritesNoSecondStatus checks, on a real server, that
|
|
||||||
// Recoverer does not write its 500 again after Write500OnPanic has
|
|
||||||
// written one, which the server would log as a superfluous
|
|
||||||
// WriteHeader call.
|
|
||||||
func TestRecovererWritesNoSecondStatus(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var serverLog bytes.Buffer
|
|
||||||
|
|
||||||
mw := middleware.NewWithLogger(slog.New(slog.DiscardHandler))
|
|
||||||
|
|
||||||
server := httptest.NewUnstartedServer(chi.Chain(
|
|
||||||
mw.Recoverer(),
|
|
||||||
mw.Write500OnPanic(),
|
|
||||||
).HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
|
||||||
panic("boom")
|
|
||||||
}))
|
|
||||||
server.Config.ErrorLog = log.New(&serverLog, "", 0)
|
|
||||||
server.Start()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(),
|
|
||||||
http.MethodGet, server.URL, http.NoBody)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("new request: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := server.Client().Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("request: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
|
|
||||||
// Close waits for the handler to return, so whatever the server
|
|
||||||
// logs for this request is in serverLog after it.
|
|
||||||
server.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusInternalServerError {
|
|
||||||
t.Errorf("status = %d, want %d",
|
|
||||||
resp.StatusCode, http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
|
|
||||||
if serverLog.Len() != 0 {
|
|
||||||
t.Errorf("server logged %q", serverLog.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoggingCutsRequestStringsToBound sends an over-long URL and
|
// TestLoggingCutsRequestStringsToBound sends an over-long URL and
|
||||||
// over-long header values, and checks the request log writes each
|
// over-long header values, and checks the request log writes each
|
||||||
// one cut to logger.MaxLoggedFieldBytes.
|
// one cut to logger.MaxLoggedFieldBytes.
|
||||||
@@ -91,15 +91,6 @@ type Buffer struct {
|
|||||||
// of the report files in dataDir, plus the reports waiting to
|
// of the report files in dataDir, plus the reports waiting to
|
||||||
// be written to one at their uncompressed size.
|
// be written to one at their uncompressed size.
|
||||||
usedBytes int64
|
usedBytes int64
|
||||||
// writeErr is the error of the last write of a report file to
|
|
||||||
// finish, nil if it succeeded or none has. While it is set, Append
|
|
||||||
// refuses reports, and the reports of that write wait in buf for
|
|
||||||
// the next flush.
|
|
||||||
writeErr error
|
|
||||||
// writers are the goroutines that write report files: flushLoop,
|
|
||||||
// and each one Append starts at flushSizeThreshold. OnStop waits
|
|
||||||
// for them before the final flush.
|
|
||||||
writers sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a Buffer and registers lifecycle hooks to
|
// New creates a Buffer and registers lifecycle hooks to
|
||||||
@@ -151,7 +142,7 @@ func New(
|
|||||||
|
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
|
|
||||||
b.writers.Go(b.flushLoop)
|
go b.flushLoop()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
@@ -162,14 +153,7 @@ func New(
|
|||||||
var err error
|
var err error
|
||||||
|
|
||||||
b.stopOnce.Do(func() {
|
b.stopOnce.Do(func() {
|
||||||
// Closing done ends flushLoop after any flush it is
|
|
||||||
// running. Wait for it and for the writes Append
|
|
||||||
// started, which the process exiting would cut off;
|
|
||||||
// one that fails puts its reports back in buf, for the
|
|
||||||
// final flush to write.
|
|
||||||
close(b.done)
|
close(b.done)
|
||||||
b.writers.Wait()
|
|
||||||
|
|
||||||
err = b.flushLocked()
|
err = b.flushLocked()
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -183,12 +167,11 @@ func New(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Append marshals v as a single JSON line and appends it to
|
// Append marshals v as a single JSON line and appends it to
|
||||||
// the buffer. It stores nothing and returns an error if the last
|
// the buffer. If the line would take usedBytes past maxBytes, the
|
||||||
// write of a report file failed, until a write succeeds again. If the
|
// oldest report files are deleted to make room; it stores nothing
|
||||||
// line would take usedBytes past maxBytes, the oldest report files are
|
// and returns ErrFull if that cannot make room. If the buffer
|
||||||
// deleted to make room; it stores nothing and returns ErrFull if that
|
// reaches the size threshold, it is drained and written to disk
|
||||||
// cannot make room. If the buffer reaches the size threshold, it is
|
// asynchronously.
|
||||||
// drained and written to disk asynchronously.
|
|
||||||
func (b *Buffer) Append(v any) error {
|
func (b *Buffer) Append(v any) error {
|
||||||
line, err := json.Marshal(v)
|
line, err := json.Marshal(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -199,13 +182,6 @@ func (b *Buffer) Append(v any) error {
|
|||||||
|
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
|
|
||||||
if b.writeErr != nil {
|
|
||||||
err = fmt.Errorf("report files cannot be written: %w", b.writeErr)
|
|
||||||
b.mu.Unlock()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.deleteOldestFiles(lineBytes)
|
b.deleteOldestFiles(lineBytes)
|
||||||
|
|
||||||
if b.usedBytes+lineBytes > b.maxBytes {
|
if b.usedBytes+lineBytes > b.maxBytes {
|
||||||
@@ -222,12 +198,12 @@ func (b *Buffer) Append(v any) error {
|
|||||||
data := b.drainBuf()
|
data := b.drainBuf()
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
|
|
||||||
b.writers.Go(func() {
|
go func() {
|
||||||
writeErr := b.writeFile(data)
|
writeErr := b.writeFile(data)
|
||||||
if writeErr != nil {
|
if writeErr != nil {
|
||||||
b.log.Error("flush reports failed", "error", writeErr)
|
b.log.Error("flush reports failed", "error", writeErr)
|
||||||
}
|
}
|
||||||
})
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -315,8 +291,7 @@ func (b *Buffer) drainBuf() []byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// writeFile writes data, reports drained from the buffer, to a new
|
// writeFile writes data, reports drained from the buffer, to a new
|
||||||
// timestamped zstd-compressed JSONL file in the data directory. If the
|
// timestamped zstd-compressed JSONL file in the data directory.
|
||||||
// write fails, it puts the reports back in the buffer.
|
|
||||||
func (b *Buffer) writeFile(data []byte) error {
|
func (b *Buffer) writeFile(data []byte) error {
|
||||||
// The timestamp comes first, so the names sort by time; the number
|
// The timestamp comes first, so the names sort by time; the number
|
||||||
// after it tells apart files named in the same millisecond.
|
// after it tells apart files named in the same millisecond.
|
||||||
@@ -325,27 +300,19 @@ func (b *Buffer) writeFile(data []byte) error {
|
|||||||
|
|
||||||
size, err := b.createFile(filepath.Join(b.dataDir, name), data)
|
size, err := b.createFile(filepath.Join(b.dataDir, name), data)
|
||||||
|
|
||||||
|
// The reports no longer wait to be written, so they stop counting
|
||||||
|
// at their uncompressed size. If the write failed they are lost;
|
||||||
|
// otherwise they count as the file, which from here on may be
|
||||||
|
// deleted to make room.
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
defer b.mu.Unlock()
|
defer b.mu.Unlock()
|
||||||
|
|
||||||
b.writeErr = err
|
b.usedBytes -= int64(len(data))
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// The reports go back in front of any appended since, for the
|
|
||||||
// next flush to write, and still count at their uncompressed
|
|
||||||
// size.
|
|
||||||
appended := bytes.Clone(b.buf.Bytes())
|
|
||||||
b.buf.Reset()
|
|
||||||
b.buf.Write(data)
|
|
||||||
b.buf.Write(appended)
|
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The reports no longer wait to be written, so they stop counting
|
|
||||||
// at their uncompressed size and count as the file, which from here
|
|
||||||
// on may be deleted to make room.
|
|
||||||
b.usedBytes -= int64(len(data))
|
|
||||||
b.usedBytes += size
|
b.usedBytes += size
|
||||||
b.filesBytes += size
|
b.filesBytes += size
|
||||||
|
|
||||||
+26
-424
@@ -5,8 +5,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -15,13 +13,10 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/netwatch/internal/config"
|
"sneak.berlin/go/netwatch/internal/config"
|
||||||
"sneak.berlin/go/netwatch/internal/globals"
|
"sneak.berlin/go/netwatch/internal/globals"
|
||||||
"sneak.berlin/go/netwatch/internal/handlers"
|
|
||||||
"sneak.berlin/go/netwatch/internal/healthcheck"
|
|
||||||
"sneak.berlin/go/netwatch/internal/logger"
|
"sneak.berlin/go/netwatch/internal/logger"
|
||||||
"sneak.berlin/go/netwatch/internal/reportbuf"
|
"sneak.berlin/go/netwatch/internal/reportbuf"
|
||||||
|
|
||||||
@@ -108,174 +103,9 @@ func TestFailedFinalFlushFailsStop(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestStopWaitsForPeriodicWrite holds open the write the periodic flush
|
// startBuffer starts a Buffer through fx, as main does, with the
|
||||||
// starts, then stops the app. The stop must wait until the write is
|
// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
|
||||||
// released, and the report must then be in its file. The test runs in
|
func startBuffer(t *testing.T) *reportbuf.Buffer {
|
||||||
// a synctest bubble, whose clock moves on at once when every goroutine
|
|
||||||
// in it is waiting, so the periodic flush does not take a minute.
|
|
||||||
func TestStopWaitsForPeriodicWrite(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
buf, app := startApp(t)
|
|
||||||
|
|
||||||
created := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
buf.OnFileCreated(func(*os.File) {
|
|
||||||
created <- struct{}{}
|
|
||||||
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
|
|
||||||
err := buf.Append(map[string]string{"id": "periodic"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("append report: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
<-created
|
|
||||||
|
|
||||||
err = stopWhileHeld(t, app, release)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stop: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read report files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := `{"id":"periodic"}` + "\n"
|
|
||||||
if len(files) != 1 || files[0] != want {
|
|
||||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopWaitsForWriteAtSizeThreshold holds open the write Append
|
|
||||||
// starts when the buffer reaches FlushSizeThreshold, then stops the
|
|
||||||
// app. The stop must wait until the write is released, and the reports
|
|
||||||
// must then be in its file.
|
|
||||||
func TestStopWaitsForWriteAtSizeThreshold(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
buf, app := startApp(t)
|
|
||||||
|
|
||||||
created := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
buf.OnFileCreated(func(*os.File) {
|
|
||||||
created <- struct{}{}
|
|
||||||
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
|
|
||||||
appended := appendToThreshold(t, buf)
|
|
||||||
|
|
||||||
<-created
|
|
||||||
|
|
||||||
err := stopWhileHeld(t, app, release)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stop: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read report files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(files) != 1 || files[0] != appended {
|
|
||||||
t.Fatalf("%d report files, want one holding the %d bytes "+
|
|
||||||
"appended", len(files), len(appended))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFinalFlushWritesReportsOfFailedWrite holds open the write the
|
|
||||||
// periodic flush starts, makes it fail, then stops the app. The stop
|
|
||||||
// waits for the write, which puts its report back in the buffer, and
|
|
||||||
// the final flush writes it, so the stop succeeds with nothing lost.
|
|
||||||
func TestFinalFlushWritesReportsOfFailedWrite(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
buf, app := startApp(t)
|
|
||||||
|
|
||||||
created := make(chan *os.File)
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
buf.OnFileCreated(func(f *os.File) {
|
|
||||||
created <- f
|
|
||||||
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
|
|
||||||
err := buf.Append(map[string]string{"id": "failed"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("append report: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
held := <-created
|
|
||||||
|
|
||||||
// Closing the file under the write makes the write fail.
|
|
||||||
_ = held.Close()
|
|
||||||
|
|
||||||
// The final flush's write goes through.
|
|
||||||
buf.OnFileCreated(func(*os.File) {})
|
|
||||||
|
|
||||||
err = stopWhileHeld(t, app, release)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stop: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read report files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := `{"id":"failed"}` + "\n"
|
|
||||||
if len(files) != 1 || files[0] != want {
|
|
||||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopWhileHeld stops app while a report file write is held open until
|
|
||||||
// release is closed, and fails the test if the stop returns before
|
|
||||||
// that. It then closes release and returns the stop's error. It must
|
|
||||||
// be called in a synctest bubble.
|
|
||||||
func stopWhileHeld(
|
|
||||||
t *testing.T,
|
|
||||||
app *fxtest.App,
|
|
||||||
release chan<- struct{},
|
|
||||||
) error {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
stopped := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() { stopped <- app.Stop(t.Context()) }()
|
|
||||||
|
|
||||||
// Wait returns once every other goroutine in the bubble is waiting:
|
|
||||||
// the stop has got as far as it can while the write is held.
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
if len(stopped) != 0 {
|
|
||||||
t.Error("stop returned while a report file write was held open")
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
|
|
||||||
return <-stopped
|
|
||||||
}
|
|
||||||
|
|
||||||
// startApp starts a Buffer through fx, as main does, with the DATA_DIR
|
|
||||||
// and DATA_DIR_MAX_BYTES the calling test has set, and returns it with
|
|
||||||
// the app, for tests that stop the app themselves.
|
|
||||||
func startApp(t *testing.T) (*reportbuf.Buffer, *fxtest.App) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var buf *reportbuf.Buffer
|
var buf *reportbuf.Buffer
|
||||||
@@ -293,16 +123,6 @@ func startApp(t *testing.T) (*reportbuf.Buffer, *fxtest.App) {
|
|||||||
app.RequireStart()
|
app.RequireStart()
|
||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
return buf, app
|
|
||||||
}
|
|
||||||
|
|
||||||
// startBuffer starts a Buffer through fx, as main does, with the
|
|
||||||
// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
|
|
||||||
func startBuffer(t *testing.T) *reportbuf.Buffer {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
buf, _ := startApp(t)
|
|
||||||
|
|
||||||
return buf
|
return buf
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -687,17 +507,13 @@ func TestFileBeingWrittenIsNeverDeleted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFailedWriteKeepsItsReports makes a write fail once its file is
|
// TestFailedWriteStopsCounting makes a write fail once its file is
|
||||||
// created, under a cap with room for one report. The part of the file
|
// created. Its reports are lost, so they stop counting, and the part of
|
||||||
// written is removed, so it takes no room, and the report waits in
|
// the file written is removed, so it takes no room.
|
||||||
// memory for the next flush, which writes it. Once written, it counts
|
func TestFailedWriteStopsCounting(t *testing.T) {
|
||||||
// as its file and no longer as a report waiting, so the cap still holds
|
report := map[string]string{"id": "failed"}
|
||||||
// exactly: one more report is taken and the one after it is refused.
|
|
||||||
func TestFailedWriteKeepsItsReports(t *testing.T) {
|
|
||||||
report := map[string]string{"id": "kept"}
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
t.Setenv("DATA_DIR_MAX_BYTES", strconv.Itoa(lineBytes(t, report)))
|
t.Setenv("DATA_DIR_MAX_BYTES", strconv.Itoa(lineBytes(t, report)))
|
||||||
|
|
||||||
buf := startBuffer(t)
|
buf := startBuffer(t)
|
||||||
@@ -728,213 +544,10 @@ func TestFailedWriteKeepsItsReports(t *testing.T) {
|
|||||||
// Writes from here on, the final one at stop included, succeed.
|
// Writes from here on, the final one at stop included, succeed.
|
||||||
buf.OnFileCreated(func(*os.File) {})
|
buf.OnFileCreated(func(*os.File) {})
|
||||||
|
|
||||||
err = buf.Flush()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("flush after the failed write: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read report files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := `{"id":"kept"}` + "\n"
|
|
||||||
if len(files) != 1 || files[0] != want {
|
|
||||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The file is deleted to make room if need be.
|
|
||||||
err = buf.Append(report)
|
err = buf.Append(report)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("report after the written one: %v", err)
|
t.Fatalf("report after the failed write: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = buf.Append(report)
|
|
||||||
if !errors.Is(err, reportbuf.ErrFull) {
|
|
||||||
t.Fatalf("report past the cap: error = %v, want ErrFull", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFailedWriteKeepsReportsAppendedDuringIt holds a write open,
|
|
||||||
// appends a report while it is held, then makes the held write fail.
|
|
||||||
// Both reports are kept, and the next flush writes them in the order
|
|
||||||
// they were appended, the held write's first.
|
|
||||||
func TestFailedWriteKeepsReportsAppendedDuringIt(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
buf := startBuffer(t)
|
|
||||||
|
|
||||||
created := make(chan *os.File)
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
buf.OnFileCreated(func(f *os.File) {
|
|
||||||
created <- f
|
|
||||||
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
|
|
||||||
err := buf.Append(map[string]string{"id": "held"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("report of the held write: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
flushed := make(chan error)
|
|
||||||
|
|
||||||
go func() { flushed <- buf.Flush() }()
|
|
||||||
|
|
||||||
held := <-created
|
|
||||||
|
|
||||||
// Errorf, not Fatalf, until the held write is released, so that a
|
|
||||||
// failure here does not leave it held.
|
|
||||||
err = buf.Append(map[string]string{"id": "appended"})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("report appended while the write was held: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Closing the file under the write makes the write fail.
|
|
||||||
_ = held.Close()
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
|
|
||||||
err = <-flushed
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("held write succeeded, though its file was closed under it")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Writes from here on, the final one at stop included, succeed.
|
|
||||||
buf.OnFileCreated(func(*os.File) {})
|
|
||||||
|
|
||||||
err = buf.Flush()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("flush after the failed write: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read report files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := `{"id":"held"}` + "\n" + `{"id":"appended"}` + "\n"
|
|
||||||
if len(files) != 1 || files[0] != want {
|
|
||||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReportsRefusedWhileWritesFail posts reports through the report
|
|
||||||
// handler, as the page does. Once a write has failed, a report is
|
|
||||||
// refused with 500 until a write succeeds again; the reports taken
|
|
||||||
// before and after are stored, and the one refused is not.
|
|
||||||
func TestReportsRefusedWhileWritesFail(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
buf, h := startBufferWithHandlers(t)
|
|
||||||
|
|
||||||
if code := postReport(t, h, "before"); code != http.StatusOK {
|
|
||||||
t.Fatalf("report before the failed write: status = %d, want %d",
|
|
||||||
code, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
// With the data directory gone, the write has nowhere to go.
|
|
||||||
err := os.Remove(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("remove data dir: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = buf.Flush()
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("flush succeeded with the data directory gone")
|
|
||||||
}
|
|
||||||
|
|
||||||
if code := postReport(t, h, "refused"); code != http.StatusInternalServerError {
|
|
||||||
t.Fatalf("report after the failed write: status = %d, want %d",
|
|
||||||
code, http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.Mkdir(dir, 0o750)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("make data dir: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = buf.Flush()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("flush with the data directory back: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if code := postReport(t, h, "after"); code != http.StatusOK {
|
|
||||||
t.Fatalf("report after a write succeeded: status = %d, want %d",
|
|
||||||
code, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = buf.Flush()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("flush of the report after: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read report files: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stored := strings.Join(files, "")
|
|
||||||
|
|
||||||
for _, clientID := range []string{"before", "after"} {
|
|
||||||
if !strings.Contains(stored, `"clientId":"`+clientID+`"`) {
|
|
||||||
t.Errorf("report %q not stored", clientID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(stored, `"clientId":"refused"`) {
|
|
||||||
t.Error("refused report stored")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startBufferWithHandlers starts a Buffer and the Handlers that store
|
|
||||||
// reports in it through fx, as main does, with the DATA_DIR the calling
|
|
||||||
// test has set.
|
|
||||||
func startBufferWithHandlers(
|
|
||||||
t *testing.T,
|
|
||||||
) (*reportbuf.Buffer, *handlers.Handlers) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
buf *reportbuf.Buffer
|
|
||||||
h *handlers.Handlers
|
|
||||||
)
|
|
||||||
|
|
||||||
app := fxtest.New(t,
|
|
||||||
fx.Provide(
|
|
||||||
config.New,
|
|
||||||
globals.New,
|
|
||||||
handlers.New,
|
|
||||||
healthcheck.New,
|
|
||||||
logger.New,
|
|
||||||
reportbuf.New,
|
|
||||||
),
|
|
||||||
fx.Populate(&buf, &h),
|
|
||||||
)
|
|
||||||
|
|
||||||
app.RequireStart()
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
return buf, h
|
|
||||||
}
|
|
||||||
|
|
||||||
// postReport posts a report from clientID to the report handler, as the
|
|
||||||
// page does, and returns the status of the answer.
|
|
||||||
func postReport(t *testing.T, h *handlers.Handlers, clientID string) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(),
|
|
||||||
http.MethodPost, "/api/v1/reports",
|
|
||||||
strings.NewReader(`{"clientId":"`+clientID+`","hosts":[]}`),
|
|
||||||
)
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
h.HandleReport().ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
return rec.Code
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestConcurrentAppendsStopAtCap appends from many goroutines at once
|
// TestConcurrentAppendsStopAtCap appends from many goroutines at once
|
||||||
@@ -1068,32 +681,6 @@ func TestFlushAtSizeThreshold(t *testing.T) {
|
|||||||
|
|
||||||
buf := startBuffer(t)
|
buf := startBuffer(t)
|
||||||
|
|
||||||
appended := appendToThreshold(t, buf)
|
|
||||||
|
|
||||||
// Append writes the file in the background, so wait for it.
|
|
||||||
deadline := time.Now().Add(10 * time.Second)
|
|
||||||
|
|
||||||
for {
|
|
||||||
files, err := readReportFiles(dir)
|
|
||||||
if err == nil && len(files) == 1 && files[0] == appended {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("%d report files (error: %v), want one holding the "+
|
|
||||||
"%d bytes appended", len(files), err, len(appended))
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// appendToThreshold appends reports to buf until it holds
|
|
||||||
// FlushSizeThreshold bytes, so that the last append starts writing them
|
|
||||||
// to a report file, and returns the lines appended.
|
|
||||||
func appendToThreshold(t *testing.T, buf *reportbuf.Buffer) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
// Large reports, so the threshold takes a few hundred appends.
|
// Large reports, so the threshold takes a few hundred appends.
|
||||||
pad := strings.Repeat("a", 64<<10)
|
pad := strings.Repeat("a", 64<<10)
|
||||||
|
|
||||||
@@ -1116,7 +703,22 @@ func appendToThreshold(t *testing.T, buf *reportbuf.Buffer) string {
|
|||||||
appended.WriteByte('\n')
|
appended.WriteByte('\n')
|
||||||
}
|
}
|
||||||
|
|
||||||
return appended.String()
|
// Append writes the file in the background, so wait for it.
|
||||||
|
deadline := time.Now().Add(10 * time.Second)
|
||||||
|
|
||||||
|
for {
|
||||||
|
files, err := readReportFiles(dir)
|
||||||
|
if err == nil && len(files) == 1 && files[0] == appended.String() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("%d report files (error: %v), want one holding the "+
|
||||||
|
"%d bytes appended", len(files), err, appended.Len())
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// reportFilesBytes returns the total size of the report files in dir.
|
// reportFilesBytes returns the total size of the report files in dir.
|
||||||
@@ -34,16 +34,13 @@ func (s *Server) SetupRoutes() {
|
|||||||
s.router.Use(s.mw.Recoverer())
|
s.router.Use(s.mw.Recoverer())
|
||||||
s.router.Use(middleware.RequestID)
|
s.router.Use(middleware.RequestID)
|
||||||
s.router.Use(s.mw.Logging())
|
s.router.Use(s.mw.Logging())
|
||||||
// s.mw.Recoverer would write a panic's 500 only after the panic has
|
|
||||||
// passed the request log, so it is written here, where the log sees it.
|
|
||||||
s.router.Use(s.mw.Write500OnPanic())
|
|
||||||
s.router.Use(s.mw.SecurityHeaders())
|
s.router.Use(s.mw.SecurityHeaders())
|
||||||
s.router.Use(s.mw.CORS(s.params.Config.CORSAllowedOrigins))
|
s.router.Use(s.mw.CORS(s.params.Config.CORSAllowedOrigins))
|
||||||
s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
|
s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
|
||||||
s.router.Use(middleware.Timeout(requestTimeout))
|
s.router.Use(middleware.Timeout(requestTimeout))
|
||||||
|
|
||||||
// Sentry reports a panic, then panics again, so that it still
|
// Sentry reports a panic, then panics again, so that s.mw.Recoverer
|
||||||
// reaches s.mw.Recoverer.
|
// still answers 500.
|
||||||
if s.params.Config.SentryDSN != "" {
|
if s.params.Config.SentryDSN != "" {
|
||||||
s.router.Use(sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle)
|
s.router.Use(sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle)
|
||||||
}
|
}
|
||||||
@@ -67,8 +64,6 @@ func (s *Server) SetupRoutes() {
|
|||||||
// other.
|
// other.
|
||||||
if s.params.Config.MetricsUsername != "" {
|
if s.params.Config.MetricsUsername != "" {
|
||||||
r.Use(s.mw.Metrics(registry))
|
r.Use(s.mw.Metrics(registry))
|
||||||
// Likewise inside the metrics.
|
|
||||||
r.Use(s.mw.Write500OnPanic())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
r.Get("/.well-known/healthcheck", s.h.HandleHealthCheck())
|
r.Get("/.well-known/healthcheck", s.h.HandleHealthCheck())
|
||||||
@@ -66,7 +66,6 @@ func TestReportsAreRateLimited(t *testing.T) {
|
|||||||
http.MethodPost, "/api/v1/reports",
|
http.MethodPost, "/api/v1/reports",
|
||||||
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
strings.NewReader(`{"clientId":"c1","hosts":[]}`),
|
||||||
)
|
)
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
srv.ServeHTTP(rec, req)
|
srv.ServeHTTP(rec, req)
|
||||||
|
|
||||||
return rec.Code
|
return rec.Code
|
||||||
Executable
+21
@@ -0,0 +1,21 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/build: compile the static netwatch-server binary into the
|
||||||
|
# backend project root, with its version stamped in.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
# VERSION comes from the environment (the root Dockerfile passes its
|
||||||
|
# ARG VERSION in). Unset or empty, it is git describe, or "dev" where
|
||||||
|
# there is no git or no repository history.
|
||||||
|
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
|
||||||
|
|
||||||
|
CGO_ENABLED=0 go build -trimpath \
|
||||||
|
-ldflags "-s -w -X main.Version=$version" \
|
||||||
|
-o netwatch-server ./cmd/netwatch-server/
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+12
@@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/clean: remove build artifacts.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
rm -f netwatch-server
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+12
@@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/fmt: format the Go sources (writes).
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
go fmt ./...
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+18
@@ -0,0 +1,18 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/fmt-check: check Go formatting (read-only). Same scope as
|
||||||
|
# script/fmt, but fails instead of writing.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
unformatted="$(gofmt -l .)"
|
||||||
|
if [ -n "$unformatted" ]; then
|
||||||
|
echo "Files not formatted:" >&2
|
||||||
|
echo "$unformatted" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/lint: lint the whole repo as the root make lint does, by
|
||||||
|
# building the lint phase of the root Dockerfile. golangci-lint never
|
||||||
|
# runs on the host (REPO_POLICIES.md).
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
exec "$ROOT/script/lint"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/run: build with script/build, then run netwatch-server locally
|
# script/run: build and run netwatch-server locally.
|
||||||
# with its defaults: port 8080, reports stored in data/reports.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
Executable
+23
@@ -0,0 +1,23 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/test: run the backend test suite on the host with the race
|
||||||
|
# detector and coverage. The root make test runs the same in the test
|
||||||
|
# phase of the root Dockerfile. Go's own -timeout bounds the tests and
|
||||||
|
# not their compile, so a cold build cache cannot fail it. -count=1
|
||||||
|
# keeps Go's test result cache out of both runs, so neither can report
|
||||||
|
# a stored pass. The race detector needs cgo, and so a C compiler. If
|
||||||
|
# the tests fail, they run again with -v for the details, and the
|
||||||
|
# script fails even if that run passes.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
go test -count=1 -timeout 90s -race -cover ./... || {
|
||||||
|
echo "--- Rerunning with -v for details ---"
|
||||||
|
go test -count=1 -timeout 90s -race -v ./...
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
+10
-12
@@ -34,11 +34,9 @@ fi
|
|||||||
|
|
||||||
# TRUSTED_PROXIES names the reverse proxies in front of the container,
|
# TRUSTED_PROXIES names the reverse proxies in front of the container,
|
||||||
# as IP addresses or CIDRs separated by commas. nginx takes the client
|
# as IP addresses or CIDRs separated by commas. nginx takes the client
|
||||||
# address from X-Forwarded-For only on a request from one of them.
|
# address from X-Forwarded-For only on a request from one of them, so
|
||||||
# Unset, they are the three private ranges of RFC 1918; set and empty,
|
# unset or empty, it trusts no one. nginx.conf includes the file written
|
||||||
# nginx trusts no one, which is why the default below follows - and not
|
# here, one set_real_ip_from line per entry.
|
||||||
# :-. configs/nginx.conf includes the file written here, one
|
|
||||||
# set_real_ip_from line per entry.
|
|
||||||
#
|
#
|
||||||
# nginx looks up an entry it cannot read as an address as a hostname,
|
# nginx looks up an entry it cannot read as an address as a hostname,
|
||||||
# and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry
|
# and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry
|
||||||
@@ -47,7 +45,7 @@ fi
|
|||||||
# TRUSTED_PROXIES. Its error, naming the CIDR, is dropped for the one
|
# TRUSTED_PROXIES. Its error, naming the CIDR, is dropped for the one
|
||||||
# below, naming the entry as written. set -f keeps a * in an entry from
|
# below, naming the entry as written. set -f keeps a * in an entry from
|
||||||
# becoming a list of file names.
|
# becoming a list of file names.
|
||||||
TRUSTED_PROXIES="${TRUSTED_PROXIES-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}"
|
TRUSTED_PROXIES="${TRUSTED_PROXIES:-}"
|
||||||
set -f
|
set -f
|
||||||
for proxy in $(printf '%s' "$TRUSTED_PROXIES" | tr ',' ' '); do
|
for proxy in $(printf '%s' "$TRUSTED_PROXIES" | tr ',' ' '); do
|
||||||
case "$proxy" in
|
case "$proxy" in
|
||||||
@@ -78,10 +76,10 @@ stop_requested=""
|
|||||||
trap 'stop_requested=yes' TERM INT
|
trap 'stop_requested=yes' TERM INT
|
||||||
|
|
||||||
# netwatch-server runs as the netwatch user and listens on loopback
|
# netwatch-server runs as the netwatch user and listens on loopback
|
||||||
# only, on a port other than the public one; configs/nginx.conf proxies
|
# only, on a port other than the public one; nginx.conf proxies to this
|
||||||
# to this address. Its only client is nginx, so it takes the client
|
# address. Its only client is nginx, so it takes the client address
|
||||||
# address nginx passes on from 127.0.0.1 alone, whatever TRUSTED_PROXIES
|
# nginx passes on from 127.0.0.1 alone, whatever TRUSTED_PROXIES the
|
||||||
# the container has. The netwatch user has no login shell, hence -s
|
# container has. The netwatch user has no login shell, hence -s
|
||||||
# /bin/sh. busybox su replaces itself with the command instead of
|
# /bin/sh. busybox su replaces itself with the command instead of
|
||||||
# staying on as its parent, so $! is the server's own PID.
|
# staying on as its parent, so $! is the server's own PID.
|
||||||
BIND_ADDRESS=127.0.0.1 PORT=8081 TRUSTED_PROXIES=127.0.0.1/32 \
|
BIND_ADDRESS=127.0.0.1 PORT=8081 TRUSTED_PROXIES=127.0.0.1/32 \
|
||||||
@@ -90,8 +88,8 @@ backend=$!
|
|||||||
|
|
||||||
# nginx starts through the nginx image's own entrypoint, which applies
|
# nginx starts through the nginx image's own entrypoint, which applies
|
||||||
# the image's start-up configuration and then replaces itself with
|
# the image's start-up configuration and then replaces itself with
|
||||||
# nginx. Part of that start-up configuration renders configs/nginx.conf
|
# nginx. Part of that start-up configuration renders nginx.conf into
|
||||||
# into conf.d with nginx listening on PORT. NGINX_ENVSUBST_FILTER limits
|
# conf.d with nginx listening on PORT. NGINX_ENVSUBST_FILTER limits
|
||||||
# that rendering to PORT: a variable nginx itself uses, such as $uri,
|
# that rendering to PORT: a variable nginx itself uses, such as $uri,
|
||||||
# would otherwise be replaced by an environment variable of the same
|
# would otherwise be replaced by an environment variable of the same
|
||||||
# name.
|
# name.
|
||||||
|
|||||||
+1
-1
@@ -8,7 +8,7 @@ export default defineConfig([
|
|||||||
js.configs.recommended,
|
js.configs.recommended,
|
||||||
{
|
{
|
||||||
// The page, and facts.js, which the viewport test runs inside it.
|
// The page, and facts.js, which the viewport test runs inside it.
|
||||||
files: ["web/src/**/*.js", "test/viewport/facts.js"],
|
files: ["src/**/*.js", "test/viewport/facts.js"],
|
||||||
languageOptions: {
|
languageOptions: {
|
||||||
globals: {
|
globals: {
|
||||||
...globals.browser,
|
...globals.browser,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
type="image/svg+xml"
|
type="image/svg+xml"
|
||||||
href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>📡</text></svg>"
|
href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>📡</text></svg>"
|
||||||
/>
|
/>
|
||||||
<!-- Linked here, not imported by web/src/main.js, so the unit tests can
|
<!-- Linked here, not imported by src/main.js, so the unit tests can
|
||||||
import that module in Node, which cannot import CSS. -->
|
import that module in Node, which cannot import CSS. -->
|
||||||
<link rel="stylesheet" href="/src/styles.css" />
|
<link rel="stylesheet" href="/src/styles.css" />
|
||||||
</head>
|
</head>
|
||||||
@@ -18,10 +18,9 @@ server {
|
|||||||
|
|
||||||
# The client address comes from X-Forwarded-For only on a request
|
# The client address comes from X-Forwarded-For only on a request
|
||||||
# from the reverse proxies in TRUSTED_PROXIES: bin/entrypoint.sh
|
# from the reverse proxies in TRUSTED_PROXIES: bin/entrypoint.sh
|
||||||
# writes one set_real_ip_from line for each into this file. Unset,
|
# writes one set_real_ip_from line for each into this file, and
|
||||||
# TRUSTED_PROXIES is the three private ranges of RFC 1918; set and
|
# leaves it empty when TRUSTED_PROXIES is unset, so that by default
|
||||||
# empty, the file is empty and the client address is the one each
|
# the client address is the one each request comes from.
|
||||||
# request comes from.
|
|
||||||
include /etc/nginx/trusted-proxies.conf;
|
include /etc/nginx/trusted-proxies.conf;
|
||||||
real_ip_header X-Forwarded-For;
|
real_ip_header X-Forwarded-For;
|
||||||
real_ip_recursive on;
|
real_ip_recursive on;
|
||||||
@@ -54,7 +53,7 @@ server {
|
|||||||
|
|
||||||
# netwatch-server sets the same security headers on its own
|
# netwatch-server sets the same security headers on its own
|
||||||
# responses. Its copies are dropped so that each header goes out
|
# responses. Its copies are dropped so that each header goes out
|
||||||
# once, as configs/security-headers.conf sets it.
|
# once, as security-headers.conf sets it.
|
||||||
proxy_hide_header Strict-Transport-Security;
|
proxy_hide_header Strict-Transport-Security;
|
||||||
proxy_hide_header Content-Security-Policy;
|
proxy_hide_header Content-Security-Policy;
|
||||||
proxy_hide_header X-Frame-Options;
|
proxy_hide_header X-Frame-Options;
|
||||||
+8
-2
@@ -17,7 +17,7 @@
|
|||||||
# golangci-lint is not installed: make lint runs it in Docker, as make
|
# golangci-lint is not installed: make lint runs it in Docker, as make
|
||||||
# test runs the tests, and this script does not install Docker either.
|
# test runs the tests, and this script does not install Docker either.
|
||||||
#
|
#
|
||||||
# Unlike the org model: Go for the backend, a newer node for eslint.
|
# Unlike the org model: Go and gcc for backend/, a newer node for eslint.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -272,13 +272,19 @@ main() {
|
|||||||
|
|
||||||
if missing make; then pkg_install gnumake make make make; fi
|
if missing make; then pkg_install gnumake make make make; fi
|
||||||
if missing git; then pkg_install git git git git; fi
|
if missing git; then pkg_install git git git git; fi
|
||||||
|
# The race detector in backend/'s make test needs cgo, which Go turns
|
||||||
|
# on only when it finds its C compiler, gcc on Linux. apt and apk
|
||||||
|
# ship the C library headers apart from gcc.
|
||||||
|
if missing gcc; then
|
||||||
|
pkg_install gcc "gcc libc6-dev" gcc "gcc musl-dev"
|
||||||
|
fi
|
||||||
|
|
||||||
ensure_node
|
ensure_node
|
||||||
ensure_yarn
|
ensure_yarn
|
||||||
yarn install --frozen-lockfile
|
yarn install --frozen-lockfile
|
||||||
|
|
||||||
ensure_go
|
ensure_go
|
||||||
go mod download
|
(cd "$ROOT/backend" && go mod download)
|
||||||
|
|
||||||
if missing docker; then
|
if missing docker; then
|
||||||
echo "bootstrap: docker not found; make test and make lint, and so" >&2
|
echo "bootstrap: docker not found; make test and make lint, and so" >&2
|
||||||
|
|||||||
+2
-11
@@ -1,7 +1,6 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/build: build the frontend for production into dist/, then the
|
# script/build: build the frontend for production into dist/. The Go
|
||||||
# static netwatch-server binary into the repo root, with its version
|
# backend is built by backend/script/build.
|
||||||
# stamped in.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -9,14 +8,6 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
yarn build
|
yarn build
|
||||||
|
|
||||||
# VERSION comes from the environment. Unset or empty, it is git
|
|
||||||
# describe, or "dev" where there is no git or no repository history.
|
|
||||||
version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}"
|
|
||||||
|
|
||||||
CGO_ENABLED=0 go build -trimpath \
|
|
||||||
-ldflags "-s -w -X main.Version=$version" \
|
|
||||||
-o netwatch-server ./cmd/netwatch-server/
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -1,13 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/clean: remove what script/build writes: dist/ and the
|
|
||||||
# netwatch-server binary.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
rm -rf dist netwatch-server
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
+3
-5
@@ -1,9 +1,7 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/fmt: format the whole repo (writes): prettier over the
|
# script/fmt: format the whole repo (writes): prettier over the
|
||||||
# JavaScript, CSS, HTML and Markdown, then gofmt over the Go sources in
|
# JavaScript, CSS, HTML and Markdown, then gofmt over the Go backend.
|
||||||
# cmd/ and internal/; not over the whole root, as node_modules/ holds
|
# The org model formats only markdown; this repo also has JS and Go.
|
||||||
# .go files of its own. The org model formats only markdown; this repo
|
|
||||||
# also has JS and Go.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -14,7 +12,7 @@ main() {
|
|||||||
# ~/.local/bin, which the shell that called it may not have on PATH.
|
# ~/.local/bin, which the shell that called it may not have on PATH.
|
||||||
PATH="$HOME/.local/bin:$PATH"
|
PATH="$HOME/.local/bin:$PATH"
|
||||||
"$ROOT/script/frontend-fmt"
|
"$ROOT/script/frontend-fmt"
|
||||||
gofmt -l -w cmd internal
|
"$ROOT/backend/script/fmt"
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+1
-6
@@ -12,12 +12,7 @@ main() {
|
|||||||
# ~/.local/bin, which the shell that called it may not have on PATH.
|
# ~/.local/bin, which the shell that called it may not have on PATH.
|
||||||
PATH="$HOME/.local/bin:$PATH"
|
PATH="$HOME/.local/bin:$PATH"
|
||||||
"$ROOT/script/frontend-fmt-check"
|
"$ROOT/script/frontend-fmt-check"
|
||||||
unformatted="$(gofmt -l cmd internal)"
|
"$ROOT/backend/script/fmt-check"
|
||||||
if [ -n "$unformatted" ]; then
|
|
||||||
echo "Go files not formatted:" >&2
|
|
||||||
echo "$unformatted" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+4
-4
@@ -1,9 +1,9 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/frontend-fmt: format the JavaScript, CSS, HTML and Markdown,
|
# script/frontend-fmt: format the JavaScript, CSS, HTML and Markdown,
|
||||||
# repo-wide (writes). Those are the languages REPO_POLICIES.md gives
|
# repo-wide (writes), the markdown in backend/ included. Those are the
|
||||||
# prettier; the YAML is left alone, as .golangci.yml must stay the org
|
# languages REPO_POLICIES.md gives prettier; the YAML is left alone, as
|
||||||
# standard byte for byte. Prettier does not read Go; script/fmt runs
|
# backend/.golangci.yml must stay the org standard byte for byte.
|
||||||
# gofmt over the Go sources.
|
# Prettier does not read Go; backend/script/fmt formats the Go sources.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|||||||
@@ -2,10 +2,9 @@
|
|||||||
# script/frontend-test: run the frontend test suite on the host: the
|
# script/frontend-test: run the frontend test suite on the host: the
|
||||||
# unit tests in test/unit/, through the test script in package.json,
|
# unit tests in test/unit/, through the test script in package.json,
|
||||||
# then the production build, which fails on broken code. make test runs
|
# then the production build, which fails on broken code. make test runs
|
||||||
# the same tests in the frontend-test stage of Dockerfile, and the image
|
# the same in the frontend stage of Dockerfile. The tests print a dot
|
||||||
# build runs the production build in its frontend-build stage. The tests
|
# each; if any fails, they run again with every test listed, and the
|
||||||
# print a dot each; if any fails, they run again with every test listed,
|
# script fails even if that run passes. NODE_OPTIONS chooses the
|
||||||
# and the script fails even if that run passes. NODE_OPTIONS chooses the
|
|
||||||
# reporter because yarn adds its arguments after the test files, where
|
# reporter because yarn adds its arguments after the test files, where
|
||||||
# node would take a reporter option for one more file.
|
# node would take a reporter option for one more file.
|
||||||
set -eu
|
set -eu
|
||||||
|
|||||||
@@ -2,11 +2,11 @@
|
|||||||
# script/frontend-viewport-test: verify the responsive layout of the built
|
# script/frontend-viewport-test: verify the responsive layout of the built
|
||||||
# frontend in a real browser engine.
|
# frontend in a real browser engine.
|
||||||
#
|
#
|
||||||
# Builds dist/, serves it with the same nginx image and the same
|
# Builds dist/, serves it with the same nginx image and the same nginx.conf
|
||||||
# configs/nginx.conf the shipping container uses, points a containerised
|
# the shipping container uses, points a containerised headless Chrome at it
|
||||||
# headless Chrome at it over CDP, and asserts on computed layout at every
|
# over CDP, and asserts on computed layout at every viewport width derived
|
||||||
# viewport width derived from the app's own CSS. See
|
# from the app's own CSS. See test/viewport/README.md for what this covers
|
||||||
# test/viewport/README.md for what this covers and what it cannot.
|
# and what it cannot.
|
||||||
#
|
#
|
||||||
# Deliberately not part of script/check: it needs Docker and takes far
|
# Deliberately not part of script/check: it needs Docker and takes far
|
||||||
# longer than the 20s budget make test has to stay inside.
|
# longer than the 20s budget make test has to stay inside.
|
||||||
@@ -61,18 +61,18 @@ main() {
|
|||||||
# host.
|
# host.
|
||||||
docker network create --internal "$NETWORK" > /dev/null
|
docker network create --internal "$NETWORK" > /dev/null
|
||||||
|
|
||||||
# configs/nginx.conf is a template: the image renders it over its own
|
# nginx.conf is a template: the image renders it over its own
|
||||||
# default.conf, with the same port and limit bin/entrypoint.sh uses.
|
# default.conf, with the same port and limit bin/entrypoint.sh uses.
|
||||||
# The empty file it includes trusts no proxy, as bin/entrypoint.sh
|
# The empty file it includes trusts no proxy, as bin/entrypoint.sh
|
||||||
# writes it when TRUSTED_PROXIES is set and empty. It also includes
|
# writes it when TRUSTED_PROXIES is unset. nginx.conf also includes
|
||||||
# the security headers, so the page runs under the shipped policy.
|
# the security headers, so the page runs under the shipped policy.
|
||||||
docker run -d --rm --name "$SERVER" \
|
docker run -d --rm --name "$SERVER" \
|
||||||
--network "$NETWORK" --network-alias netwatch \
|
--network "$NETWORK" --network-alias netwatch \
|
||||||
-e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \
|
-e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \
|
||||||
-v "$ROOT/dist:/usr/share/nginx/html:ro" \
|
-v "$ROOT/dist:/usr/share/nginx/html:ro" \
|
||||||
-v "$ROOT/configs/nginx.conf:/etc/nginx/templates/default.conf.template:ro" \
|
-v "$ROOT/nginx.conf:/etc/nginx/templates/default.conf.template:ro" \
|
||||||
-v /dev/null:/etc/nginx/trusted-proxies.conf:ro \
|
-v /dev/null:/etc/nginx/trusted-proxies.conf:ro \
|
||||||
-v "$ROOT/configs/security-headers.conf:/etc/nginx/security-headers.conf:ro" \
|
-v "$ROOT/security-headers.conf:/etc/nginx/security-headers.conf:ro" \
|
||||||
"$SERVER_IMAGE" > /dev/null
|
"$SERVER_IMAGE" > /dev/null
|
||||||
|
|
||||||
# The image's own entrypoint already exposes CDP on 9222 and passes
|
# The image's own entrypoint already exposes CDP on 9222 and passes
|
||||||
|
|||||||
+3
-3
@@ -1,12 +1,12 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/tidy: run go mod tidy, which adds the modules the Go sources
|
# script/tidy: run go mod tidy in backend/, which adds the modules the
|
||||||
# import, drops those they no longer do, and updates go.sum.
|
# Go sources import, drops those they no longer do, and updates go.sum.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT/backend"
|
||||||
go mod tidy
|
go mod tidy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# The security headers REPO_POLICIES.md requires on every response.
|
# The security headers REPO_POLICIES.md requires on every response.
|
||||||
# configs/nginx.conf includes this file, which Dockerfile copies to
|
# nginx.conf includes this file, which Dockerfile copies to
|
||||||
# /etc/nginx/security-headers.conf. always sends each header on error
|
# /etc/nginx/security-headers.conf. always sends each header on error
|
||||||
# responses too.
|
# responses too.
|
||||||
|
|
||||||
@@ -8,11 +8,11 @@ add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" alway
|
|||||||
# Scripts and styles load only from the page's own origin. Inline ones
|
# Scripts and styles load only from the page's own origin. Inline ones
|
||||||
# are blocked, style attributes in markup included, so style elements
|
# are blocked, style attributes in markup included, so style elements
|
||||||
# through classes or element.style. data: images are for the favicon
|
# through classes or element.style. data: images are for the favicon
|
||||||
# in web/index.html. connect-src is * because the browser checks each
|
# in index.html. connect-src is * because the browser checks each probe in
|
||||||
# probe in web/src/main.js against it, and also every redirect the probe
|
# src/main.js against it, and also every redirect the probe follows,
|
||||||
# follows, and several of those hosts redirect to others; a list of
|
# and several of those hosts redirect to others; a list of hosts here
|
||||||
# hosts here would block those probes. It also covers the reports the
|
# would block those probes. It also covers the reports the page sends
|
||||||
# page sends to its own origin.
|
# to its own origin.
|
||||||
add_header Content-Security-Policy "default-src 'self'; connect-src *; img-src 'self' data:; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" always;
|
add_header Content-Security-Policy "default-src 'self'; connect-src *; img-src 'self' data:; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" always;
|
||||||
|
|
||||||
add_header X-Frame-Options DENY always;
|
add_header X-Frame-Options DENY always;
|
||||||
@@ -144,15 +144,13 @@ const WAN_HOSTS = [
|
|||||||
|
|
||||||
const debugLog = [];
|
const debugLog = [];
|
||||||
|
|
||||||
export const log = (() => {
|
const log = (() => {
|
||||||
function append(level, message) {
|
function append(level, message) {
|
||||||
const entry = { timestamp: new Date(), level, message };
|
debugLog.push({ timestamp: new Date(), level, message });
|
||||||
debugLog.push(entry);
|
|
||||||
if (debugLog.length > CONFIG.maxLogEntries)
|
if (debugLog.length > CONFIG.maxLogEntries)
|
||||||
debugLog.splice(0, debugLog.length - CONFIG.maxLogEntries);
|
debugLog.splice(0, debugLog.length - CONFIG.maxLogEntries);
|
||||||
const panel = document.getElementById("debug-panel");
|
const panel = document.getElementById("debug-panel");
|
||||||
if (panel && !panel.classList.contains("hidden"))
|
if (panel && !panel.classList.contains("hidden")) renderDebugLog();
|
||||||
appendDebugLogLine(entry);
|
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
error: (msg) => append("error", msg),
|
error: (msg) => append("error", msg),
|
||||||
@@ -259,14 +257,6 @@ function latencyStats(latencies) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// The status a check's error gives its host: "error" for a timeout,
|
|
||||||
// "offline" for any other error, and "online" for none.
|
|
||||||
function statusFromError(error) {
|
|
||||||
if (error === "timeout") return "error";
|
|
||||||
if (error) return "offline";
|
|
||||||
return "online";
|
|
||||||
}
|
|
||||||
|
|
||||||
export class HostState {
|
export class HostState {
|
||||||
constructor(host, pinned = false) {
|
constructor(host, pinned = false) {
|
||||||
this.name = host.name;
|
this.name = host.name;
|
||||||
@@ -275,9 +265,8 @@ export class HostState {
|
|||||||
// error }, or a round skipped while paused, { timestamp,
|
// error }, or a round skipped while paused, { timestamp,
|
||||||
// latency: null, paused: true }.
|
// latency: null, paused: true }.
|
||||||
this.history = [];
|
this.history = [];
|
||||||
// The last 100 (CONFIG.maxHistoryPoints) entries the interval menu
|
// The entries the history held before the interval menu last
|
||||||
// has cleared from the history, which the reporter may not have
|
// cleared it, which the reporter may not have sent yet.
|
||||||
// sent yet.
|
|
||||||
this.clearedHistory = [];
|
this.clearedHistory = [];
|
||||||
this.lastLatency = null;
|
this.lastLatency = null;
|
||||||
this.status = "pending"; // 'online' | 'offline' | 'error' | 'pending'
|
this.status = "pending"; // 'online' | 'offline' | 'error' | 'pending'
|
||||||
@@ -294,7 +283,9 @@ export class HostState {
|
|||||||
});
|
});
|
||||||
this._trim();
|
this._trim();
|
||||||
this.lastLatency = result.latency;
|
this.lastLatency = result.latency;
|
||||||
this.status = statusFromError(result.error);
|
if (result.error === "timeout") this.status = "error";
|
||||||
|
else if (result.error) this.status = "offline";
|
||||||
|
else this.status = "online";
|
||||||
this.consecutiveFailures = result.error
|
this.consecutiveFailures = result.error
|
||||||
? this.consecutiveFailures + 1
|
? this.consecutiveFailures + 1
|
||||||
: 0;
|
: 0;
|
||||||
@@ -305,16 +296,14 @@ export class HostState {
|
|||||||
this._trim();
|
this._trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
// What the interval menu does to each host: the history starts over and
|
// Starts the history over, as the interval menu does. Its entries move
|
||||||
// the host waits for its next check. The history's entries move to
|
// to clearedHistory, which keeps the last maxHistoryPoints of them, as
|
||||||
// clearedHistory, so the reporter still sends those it has not.
|
// the history does, so the reporter still sends those it has not.
|
||||||
reset() {
|
clearHistory() {
|
||||||
this.clearedHistory = [...this.clearedHistory, ...this.history].slice(
|
this.clearedHistory = [...this.clearedHistory, ...this.history].slice(
|
||||||
-CONFIG.maxHistoryPoints,
|
-CONFIG.maxHistoryPoints,
|
||||||
);
|
);
|
||||||
this.history = [];
|
this.history = [];
|
||||||
this.lastLatency = null;
|
|
||||||
this.status = "pending";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The min, max, median and average latency of the checks in the history
|
// The min, max, median and average latency of the checks in the history
|
||||||
@@ -456,10 +445,8 @@ function getClientId() {
|
|||||||
// of { name, url, status, clearedHistory, history }, `since` maps a host url
|
// of { name, url, status, clearedHistory, history }, `since` maps a host url
|
||||||
// to the Unix-ms timestamp of the last sample already reported for it, and
|
// to the Unix-ms timestamp of the last sample already reported for it, and
|
||||||
// `now` is a Date. Only non-paused samples newer than the mark are included,
|
// `now` is a Date. Only non-paused samples newer than the mark are included,
|
||||||
// from clearedHistory as well as history. A host whose status is pending,
|
// from clearedHistory as well as history. Returns null when no host has an
|
||||||
// as the interval menu leaves it until its next check, is sent with the
|
// unreported sample.
|
||||||
// status of its last check, which is the last sample sent. Returns null
|
|
||||||
// when no host has an unreported sample.
|
|
||||||
export function buildReport(hosts, clientId, now, since) {
|
export function buildReport(hosts, clientId, now, since) {
|
||||||
const reportHosts = [];
|
const reportHosts = [];
|
||||||
const marks = new Map();
|
const marks = new Map();
|
||||||
@@ -481,10 +468,7 @@ export function buildReport(hosts, clientId, now, since) {
|
|||||||
reportHosts.push({
|
reportHosts.push({
|
||||||
name: host.name,
|
name: host.name,
|
||||||
url: host.url,
|
url: host.url,
|
||||||
status:
|
status: host.status,
|
||||||
host.status === "pending"
|
|
||||||
? statusFromError(samples.at(-1).error)
|
|
||||||
: host.status,
|
|
||||||
history: samples,
|
history: samples,
|
||||||
});
|
});
|
||||||
marks.set(host.url, high);
|
marks.set(host.url, high);
|
||||||
@@ -1109,8 +1093,9 @@ function updateHealthBox(state) {
|
|||||||
|
|
||||||
// --- Debug Log Renderer ------------------------------------------------------
|
// --- Debug Log Renderer ------------------------------------------------------
|
||||||
|
|
||||||
// One line of the debug log panel, its message set as text.
|
function renderDebugLog() {
|
||||||
function debugLogLine(entry) {
|
const el = document.getElementById("debug-log");
|
||||||
|
if (!el) return;
|
||||||
const levelColors = {
|
const levelColors = {
|
||||||
error: "text-red-400",
|
error: "text-red-400",
|
||||||
warning: "text-orange-400",
|
warning: "text-orange-400",
|
||||||
@@ -1118,43 +1103,19 @@ function debugLogLine(entry) {
|
|||||||
info: "text-gray-300",
|
info: "text-gray-300",
|
||||||
debug: "text-gray-500",
|
debug: "text-gray-500",
|
||||||
};
|
};
|
||||||
const ts = formatUTCTimestamp(entry.timestamp);
|
el.replaceChildren(
|
||||||
const lvl = entry.level.toUpperCase().padEnd(7);
|
...debugLog.map((entry) => {
|
||||||
const line = document.createElement("div");
|
const ts = formatUTCTimestamp(entry.timestamp);
|
||||||
line.className = levelColors[entry.level] || "text-gray-400";
|
const lvl = entry.level.toUpperCase().padEnd(7);
|
||||||
line.textContent = `${ts} ${lvl} ${entry.message}`;
|
const line = document.createElement("div");
|
||||||
return line;
|
line.className = levelColors[entry.level] || "text-gray-400";
|
||||||
}
|
line.textContent = `${ts} ${lvl} ${entry.message}`;
|
||||||
|
return line;
|
||||||
// Draws every line of the debug log into the panel, when it is opened.
|
}),
|
||||||
function renderDebugLog() {
|
);
|
||||||
const el = document.getElementById("debug-log");
|
|
||||||
if (!el) return;
|
|
||||||
el.replaceChildren(...debugLog.map((entry) => debugLogLine(entry)));
|
|
||||||
el.scrollTop = el.scrollHeight;
|
el.scrollTop = el.scrollHeight;
|
||||||
}
|
}
|
||||||
|
|
||||||
// The request that scrolls the open panel to its newest line in the next
|
|
||||||
// frame. One waits at most, also in a hidden tab, where frames do not run.
|
|
||||||
let debugLogScrollFrame = 0;
|
|
||||||
|
|
||||||
// Adds entry to the open panel as its last line, and drops the first line
|
|
||||||
// once the panel holds more than maxLogEntries. A check that ends while the
|
|
||||||
// page is busy is timed only when the page is done, so the lines already
|
|
||||||
// shown are left as they are, and the panel is scrolled once a frame
|
|
||||||
// rather than once a line: scrolling it lays out all of its lines again.
|
|
||||||
function appendDebugLogLine(entry) {
|
|
||||||
const el = document.getElementById("debug-log");
|
|
||||||
if (!el) return;
|
|
||||||
el.append(debugLogLine(entry));
|
|
||||||
if (el.children.length > CONFIG.maxLogEntries)
|
|
||||||
el.removeChild(el.firstElementChild);
|
|
||||||
cancelAnimationFrame(debugLogScrollFrame);
|
|
||||||
debugLogScrollFrame = requestAnimationFrame(() => {
|
|
||||||
el.scrollTop = el.scrollHeight;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Sorting -----------------------------------------------------------------
|
// --- Sorting -----------------------------------------------------------------
|
||||||
|
|
||||||
// Sort WAN hosts: pinned hosts first (alphabetically), then unpinned hosts
|
// Sort WAN hosts: pinned hosts first (alphabetically), then unpinned hosts
|
||||||
@@ -1555,7 +1516,9 @@ async function init() {
|
|||||||
|
|
||||||
// Clear all host history and reset state
|
// Clear all host history and reset state
|
||||||
for (const host of state.allHosts) {
|
for (const host of state.allHosts) {
|
||||||
host.reset();
|
host.clearHistory();
|
||||||
|
host.lastLatency = null;
|
||||||
|
host.status = "pending";
|
||||||
}
|
}
|
||||||
state.tickCount = 0;
|
state.tickCount = 0;
|
||||||
|
|
||||||
+39
-116
@@ -1,4 +1,4 @@
|
|||||||
// Unit tests for web/src/main.js, run with Node's built-in test runner by the
|
// Unit tests for src/main.js, run with Node's built-in test runner by the
|
||||||
// test script in package.json. Importing the module does not start the
|
// test script in package.json. Importing the module does not start the
|
||||||
// page.
|
// page.
|
||||||
|
|
||||||
@@ -13,22 +13,20 @@ import {
|
|||||||
humanDuration,
|
humanDuration,
|
||||||
latencyClass,
|
latencyClass,
|
||||||
latencyHex,
|
latencyHex,
|
||||||
log,
|
|
||||||
measureLatency,
|
measureLatency,
|
||||||
Reporter,
|
Reporter,
|
||||||
tick,
|
tick,
|
||||||
} from "../../web/src/main.js";
|
} from "../../src/main.js";
|
||||||
|
|
||||||
// There is no page here, so the tests stand in for it. The debug log looks
|
// There is no page here, so the tests stand in for it. The debug log looks
|
||||||
// for its panel by id and finds none, an element the page creates is a
|
// for its panel by id and finds none. Each element of a host's row that
|
||||||
// plain object, and no frame the page asks for ever comes. Each element of
|
// tick or greyOutUI draws into is a plain object, made the first time a
|
||||||
// a host's row that tick or greyOutUI draws into is a plain object, made
|
// test looks it up and kept in elements under its selector until the next
|
||||||
// the first time a test looks it up and kept in elements under its
|
// test starts. As on a page, writing its text replaces its markup; the
|
||||||
// selector until the next test starts. As on a page, writing its text
|
// status dot greyOutUI looks for in it is not there. Drawing a sparkline
|
||||||
// replaces its markup; the status dot greyOutUI looks for in it is not
|
// does nothing; it looks for the pixel ratio on window and finds none. In
|
||||||
// there. Drawing a sparkline does nothing; it looks for the pixel ratio on
|
// each test, console.error and console.info print nothing and keep what
|
||||||
// window and finds none. In each test, console.error and console.info
|
// they are given.
|
||||||
// print nothing and keep what they are given.
|
|
||||||
const doNothing = () => {};
|
const doNothing = () => {};
|
||||||
let elements;
|
let elements;
|
||||||
beforeEach((t) => {
|
beforeEach((t) => {
|
||||||
@@ -48,11 +46,8 @@ const canvasContext = {
|
|||||||
arc: doNothing,
|
arc: doNothing,
|
||||||
};
|
};
|
||||||
globalThis.window = {};
|
globalThis.window = {};
|
||||||
globalThis.requestAnimationFrame = doNothing;
|
|
||||||
globalThis.cancelAnimationFrame = doNothing;
|
|
||||||
globalThis.document = {
|
globalThis.document = {
|
||||||
getElementById: () => null,
|
getElementById: () => null,
|
||||||
createElement: () => ({}),
|
|
||||||
querySelector: (selector) =>
|
querySelector: (selector) =>
|
||||||
(elements[selector] ??= {
|
(elements[selector] ??= {
|
||||||
getContext: () => canvasContext,
|
getContext: () => canvasContext,
|
||||||
@@ -384,21 +379,11 @@ test("a check that fails in the first round, which is discarded, writes nothing
|
|||||||
assert.deepEqual(consoleLines("info"), []);
|
assert.deepEqual(consoleLines("info"), []);
|
||||||
});
|
});
|
||||||
|
|
||||||
// The reports Reporter sent in this test, each as the hosts it sent.
|
// Every report is delivered here, and only Target has checks to report.
|
||||||
function reportedHosts() {
|
// Between the first report and the second the interval changes: the
|
||||||
return fetch.mock.calls.map(
|
// interval menu calls clearHistory on each target. The third report would
|
||||||
(call) => JSON.parse(call.arguments[1].body).hosts,
|
// hold no check the first two did not, so it is not sent.
|
||||||
);
|
test("checks recorded before an interval change and not yet reported go in the next report, and no check is reported twice", async (t) => {
|
||||||
}
|
|
||||||
|
|
||||||
// In the next two tests every report is delivered, and only Target has
|
|
||||||
// checks to report. target.reset() does to Target what the interval menu
|
|
||||||
// does to each target.
|
|
||||||
|
|
||||||
// The interval changes after the first report. The second report is sent
|
|
||||||
// before Target's first check since then, and the third after it. A fourth
|
|
||||||
// would hold no check the first three did not, so it is not sent.
|
|
||||||
test("checks recorded before an interval change and not yet reported go in the next report, with the status of the last check, and no check is reported twice", async (t) => {
|
|
||||||
const state = new AppState([
|
const state = new AppState([
|
||||||
{ name: "Target", url: "https://target.test" },
|
{ name: "Target", url: "https://target.test" },
|
||||||
]);
|
]);
|
||||||
@@ -410,42 +395,36 @@ test("checks recorded before an interval change and not yet reported go in the n
|
|||||||
await reporter.flush();
|
await reporter.flush();
|
||||||
target.pushSample(2000, { latency: 20, error: null });
|
target.pushSample(2000, { latency: 20, error: null });
|
||||||
target.pushSample(3000, { latency: null, error: "timeout" });
|
target.pushSample(3000, { latency: null, error: "timeout" });
|
||||||
target.reset();
|
target.clearHistory();
|
||||||
await reporter.flush();
|
|
||||||
target.pushSample(4000, { latency: 40, error: null });
|
target.pushSample(4000, { latency: 40, error: null });
|
||||||
await reporter.flush();
|
await reporter.flush();
|
||||||
await reporter.flush();
|
await reporter.flush();
|
||||||
|
|
||||||
|
assert.equal(fetch.mock.calls.length, 2);
|
||||||
|
const [first, second] = fetch.mock.calls.map(
|
||||||
|
(call) => JSON.parse(call.arguments[1].body).hosts,
|
||||||
|
);
|
||||||
const name = "Target";
|
const name = "Target";
|
||||||
const url = "https://target.test";
|
const url = "https://target.test";
|
||||||
assert.deepEqual(reportedHosts(), [
|
assert.deepEqual(first, [
|
||||||
[
|
{
|
||||||
{
|
name,
|
||||||
name,
|
url,
|
||||||
url,
|
status: "online",
|
||||||
status: "online",
|
history: [{ t: 1000, latency: 10, error: null }],
|
||||||
history: [{ t: 1000, latency: 10, error: null }],
|
},
|
||||||
},
|
]);
|
||||||
],
|
assert.deepEqual(second, [
|
||||||
[
|
{
|
||||||
{
|
name,
|
||||||
name,
|
url,
|
||||||
url,
|
status: "online",
|
||||||
status: "error",
|
history: [
|
||||||
history: [
|
{ t: 2000, latency: 20, error: null },
|
||||||
{ t: 2000, latency: 20, error: null },
|
{ t: 3000, latency: null, error: "timeout" },
|
||||||
{ t: 3000, latency: null, error: "timeout" },
|
{ t: 4000, latency: 40, error: null },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
],
|
|
||||||
[
|
|
||||||
{
|
|
||||||
name,
|
|
||||||
url,
|
|
||||||
status: "online",
|
|
||||||
history: [{ t: 4000, latency: 40, error: null }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
]);
|
]);
|
||||||
// The page's history starts over.
|
// The page's history starts over.
|
||||||
assert.deepEqual(target.history, [
|
assert.deepEqual(target.history, [
|
||||||
@@ -453,62 +432,6 @@ test("checks recorded before an interval change and not yet reported go in the n
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
// The interval changes twice, after 60 checks each time, before the first
|
|
||||||
// report. Of the 120 checks the interval menu cleared, the report has the
|
|
||||||
// last 100 (CONFIG.maxHistoryPoints).
|
|
||||||
test("of the checks interval changes cleared before they were reported, the next report has the last 100", async (t) => {
|
|
||||||
const state = new AppState([
|
|
||||||
{ name: "Target", url: "https://target.test" },
|
|
||||||
]);
|
|
||||||
const target = state.local[0];
|
|
||||||
const reporter = new Reporter(state, "client", CONFIG.reportInterval);
|
|
||||||
t.mock.method(globalThis, "fetch", async () => ({ ok: true }));
|
|
||||||
|
|
||||||
for (let n = 1; n <= 120; n++) {
|
|
||||||
target.pushSample(n * 1000, { latency: 10, error: null });
|
|
||||||
if (n % 60 === 0) target.reset();
|
|
||||||
}
|
|
||||||
await reporter.flush();
|
|
||||||
|
|
||||||
const sent = reportedHosts()[0][0].history;
|
|
||||||
assert.equal(sent.length, 100);
|
|
||||||
assert.equal(sent[0].t, 21000);
|
|
||||||
assert.equal(sent.at(-1).t, 120000);
|
|
||||||
});
|
|
||||||
|
|
||||||
// panel stands in for the open debug log panel, and for the element in it
|
|
||||||
// that holds its lines, whichever the page looks up by id; the lines are
|
|
||||||
// kept in order in lines. Drawing the whole panel, as opening it does,
|
|
||||||
// replaces every line with a new one.
|
|
||||||
test("with the debug log open, writing more lines than it keeps adds each to the panel and leaves the lines already shown as they are", (t) => {
|
|
||||||
const lines = [];
|
|
||||||
const panel = {
|
|
||||||
classList: { contains: () => false },
|
|
||||||
children: lines,
|
|
||||||
get firstElementChild() {
|
|
||||||
return lines[0];
|
|
||||||
},
|
|
||||||
append: (line) => lines.push(line),
|
|
||||||
removeChild: (line) => lines.splice(lines.indexOf(line), 1),
|
|
||||||
replaceChildren: (...newLines) =>
|
|
||||||
lines.splice(0, lines.length, ...newLines),
|
|
||||||
};
|
|
||||||
t.mock.method(globalThis.document, "getElementById", () => panel);
|
|
||||||
const kept = CONFIG.maxLogEntries;
|
|
||||||
|
|
||||||
for (let i = 1; i <= kept; i++) log.info(`line ${i}`);
|
|
||||||
const shown = [...lines];
|
|
||||||
for (let i = kept + 1; i <= kept + 5; i++) log.info(`line ${i}`);
|
|
||||||
|
|
||||||
assert.deepEqual(
|
|
||||||
lines.map((line) => line.textContent.match(/line \d+$/)?.[0]),
|
|
||||||
Array.from({ length: kept }, (_, i) => `line ${i + 6}`),
|
|
||||||
);
|
|
||||||
// The first five lines shown were dropped, and the rest are the same
|
|
||||||
// lines, not drawn again.
|
|
||||||
shown.slice(5).forEach((line, i) => assert.equal(lines[i], line));
|
|
||||||
});
|
|
||||||
|
|
||||||
// The page shows < > " & and ' in a row's markup as
|
// The page shows < > " & and ' in a row's markup as
|
||||||
// < > " & and '.
|
// < > " & and '.
|
||||||
test(`a target whose name and URL hold < > " & and ' shows those characters in its row`, () => {
|
test(`a target whose name and URL hold < > " & and ' shows those characters in its row`, () => {
|
||||||
|
|||||||
+10
-10
@@ -7,11 +7,11 @@ make frontend-viewport-test
|
|||||||
```
|
```
|
||||||
|
|
||||||
It builds `dist/`, serves it from the same digest-pinned `nginx` image and the
|
It builds `dist/`, serves it from the same digest-pinned `nginx` image and the
|
||||||
same `configs/nginx.conf` the shipping container uses, drives a digest-pinned
|
same `nginx.conf` the shipping container uses, drives a digest-pinned headless
|
||||||
headless Chrome against it over CDP, and asserts on computed layout at every
|
Chrome against it over CDP, and asserts on computed layout at every viewport
|
||||||
viewport width derived from the app's own CSS. Screenshots land in
|
width derived from the app's own CSS. Screenshots land in `tmp/viewport/`
|
||||||
`tmp/viewport/` alongside a `results.json`; they are artifacts for a human to
|
alongside a `results.json`; they are artifacts for a human to look at when
|
||||||
look at when something fails, not the evidence. The assertions are the evidence.
|
something fails, not the evidence. The assertions are the evidence.
|
||||||
|
|
||||||
The target is deliberately outside `make check`: it takes minutes, and
|
The target is deliberately outside `make check`: it takes minutes, and
|
||||||
`make test` has to stay under 60 seconds.
|
`make test` has to stay under 60 seconds.
|
||||||
@@ -19,10 +19,10 @@ The target is deliberately outside `make check`: it takes minutes, and
|
|||||||
## How the widths are chosen
|
## How the widths are chosen
|
||||||
|
|
||||||
Not from a list of phone models. `viewports.js` parses the `@media` conditions
|
Not from a list of phone models. `viewports.js` parses the `@media` conditions
|
||||||
out of `web/src/styles.css` and scans `web/src/main.js` and `web/index.html` for
|
out of `src/styles.css` and scans `src/main.js` and `index.html` for Tailwind
|
||||||
Tailwind responsive prefixes, then tests every breakpoint it finds at one pixel
|
responsive prefixes, then tests every breakpoint it finds at one pixel below it,
|
||||||
below it, exactly on it, and one pixel above it. A generic 375px "phone" test
|
exactly on it, and one pixel above it. A generic 375px "phone" test sails
|
||||||
sails straight past an off-by-one at a media query boundary; `max-width: 768px`
|
straight past an off-by-one at a media query boundary; `max-width: 768px`
|
||||||
matches _at_ 768, and the sweep pins down which side of that line each layout is
|
matches _at_ 768, and the sweep pins down which side of that line each layout is
|
||||||
on.
|
on.
|
||||||
|
|
||||||
@@ -108,7 +108,7 @@ function of viewport width and CSS, and is covered above.
|
|||||||
|
|
||||||
Complementary layers, not two stacks. The unit tests in `test/unit/`, which
|
Complementary layers, not two stacks. The unit tests in `test/unit/`, which
|
||||||
`make test` runs with Node's built-in test runner, exercise the functions
|
`make test` runs with Node's built-in test runner, exercise the functions
|
||||||
`web/src/main.js` exports in-process with no browser. This harness exercises
|
`src/main.js` exports in-process with no browser. This harness exercises
|
||||||
rendered layout in a real engine and is the only thing here that can see a media
|
rendered layout in a real engine and is the only thing here that can see a media
|
||||||
query. Neither replaces the other; assertions about computed styles and element
|
query. Neither replaces the other; assertions about computed styles and element
|
||||||
geometry belong here, assertions about functions belong in `test/unit/`.
|
geometry belong here, assertions about functions belong in `test/unit/`.
|
||||||
|
|||||||
@@ -51,8 +51,7 @@ const PROBE_DELAYS_MS = [2, 45, 123, 456, 780];
|
|||||||
// offline row rendering is exercised too.
|
// offline row rendering is exercised too.
|
||||||
const UNREACHABLE_MODULUS = 7;
|
const UNREACHABLE_MODULUS = 7;
|
||||||
// The gateway candidate that "answers", so gateway detection succeeds and
|
// The gateway candidate that "answers", so gateway detection succeeds and
|
||||||
// the Local Gateway row renders. Matches GATEWAY_CANDIDATES in
|
// the Local Gateway row renders. Matches GATEWAY_CANDIDATES in src/main.js.
|
||||||
// web/src/main.js.
|
|
||||||
const RESPONSIVE_GATEWAY = "http://192.168.1.1";
|
const RESPONSIVE_GATEWAY = "http://192.168.1.1";
|
||||||
|
|
||||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|||||||
@@ -161,27 +161,26 @@ const ANCHOR_VIEWPORTS = [
|
|||||||
export function deriveViewports(root) {
|
export function deriveViewports(root) {
|
||||||
const conditions = [
|
const conditions = [
|
||||||
...mediaConditionsFromCss(
|
...mediaConditionsFromCss(
|
||||||
readFileSync(join(root, "web/src/styles.css"), "utf8"),
|
readFileSync(join(root, "src/styles.css"), "utf8"),
|
||||||
"web/src/styles.css",
|
"src/styles.css",
|
||||||
),
|
),
|
||||||
...mediaConditionsFromMarkup([
|
...mediaConditionsFromMarkup([
|
||||||
{
|
{
|
||||||
path: "web/src/main.js",
|
path: "src/main.js",
|
||||||
text: readFileSync(join(root, "web/src/main.js"), "utf8"),
|
text: readFileSync(join(root, "src/main.js"), "utf8"),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: "web/index.html",
|
path: "index.html",
|
||||||
text: readFileSync(join(root, "web/index.html"), "utf8"),
|
text: readFileSync(join(root, "index.html"), "utf8"),
|
||||||
},
|
},
|
||||||
]),
|
]),
|
||||||
];
|
];
|
||||||
|
|
||||||
if (conditions.length === 0) {
|
if (conditions.length === 0) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"no responsive breakpoints found in web/src/styles.css, " +
|
"no responsive breakpoints found in src/styles.css, src/main.js or " +
|
||||||
"web/src/main.js or web/index.html — either the responsive " +
|
"index.html — either the responsive layout was deleted or this " +
|
||||||
"layout was deleted or this derivation has stopped matching " +
|
"derivation has stopped matching the sources",
|
||||||
"the sources",
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,10 +6,6 @@ const commitHash = execSync("git rev-parse --short HEAD").toString().trim();
|
|||||||
const commitFull = execSync("git rev-parse HEAD").toString().trim();
|
const commitFull = execSync("git rev-parse HEAD").toString().trim();
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
// The page's source is in web/; the build still goes to dist/ at the
|
|
||||||
// repo root, which Dockerfile and the viewport test serve. Vite
|
|
||||||
// empties only an outDir inside root unless told to.
|
|
||||||
root: "web",
|
|
||||||
plugins: [tailwindcss()],
|
plugins: [tailwindcss()],
|
||||||
server: {
|
server: {
|
||||||
// Proxy /api to a locally running netwatch-server so `yarn dev`
|
// Proxy /api to a locally running netwatch-server so `yarn dev`
|
||||||
@@ -23,8 +19,6 @@ export default defineConfig({
|
|||||||
__COMMIT_FULL__: JSON.stringify(commitFull),
|
__COMMIT_FULL__: JSON.stringify(commitFull),
|
||||||
},
|
},
|
||||||
build: {
|
build: {
|
||||||
outDir: "../dist",
|
|
||||||
emptyOutDir: true,
|
|
||||||
target: "esnext",
|
target: "esnext",
|
||||||
minify: "esbuild",
|
minify: "esbuild",
|
||||||
cssMinify: true,
|
cssMinify: true,
|
||||||
|
|||||||
Reference in New Issue
Block a user