Compare commits
1
Commits
next
..
d3d3b47cf4
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
d3d3b47cf4 |
+21
-29
@@ -5,11 +5,11 @@
|
||||
#
|
||||
# The lint and test phases are the gates: `make lint` (script/lint)
|
||||
# builds the lint stage alone and `make test` (script/test) the test
|
||||
# stage alone, and the two build stages, frontend-build and builder,
|
||||
# depend on both, so the image cannot be built unless they pass. Each
|
||||
# phase covers the frontend as well, through a copy from a node stage.
|
||||
# Inside them each tool is invoked directly, never through make or
|
||||
# script/, whose lint and test are themselves docker builds.
|
||||
# stage alone, and the builder stage depends on both, so the image
|
||||
# cannot be built unless they pass. Each covers the frontend as well,
|
||||
# through a copy from a node stage. Inside them each tool is invoked
|
||||
# directly, never through make or script/, whose lint and test are
|
||||
# themselves docker builds.
|
||||
|
||||
# Frontend lint stage: eslint with the rules in eslint.config.js. The
|
||||
# lint phase below runs it.
|
||||
@@ -35,26 +35,30 @@ RUN golangci-lint run --config .golangci.yml ./...
|
||||
# phase run it.
|
||||
COPY --from=frontend-lint /app/yarn.lock /dev/null
|
||||
|
||||
# Frontend test stage: the unit tests in test/unit/. The test phase
|
||||
# below runs it, and the frontend-build stage starts from it. The tests
|
||||
# print a dot each; if any fails, they run again with every test listed,
|
||||
# and the step fails even if that run passes. NODE_OPTIONS chooses the
|
||||
# reporter because yarn adds its arguments after the test files, where
|
||||
# node would take a reporter option for one more file.
|
||||
# Frontend stage: the unit tests in test/unit/, then the production
|
||||
# build into dist/, which the runtime stage serves. The test phase below
|
||||
# runs it. The tests print a dot each; if any fails, they run again with
|
||||
# every test listed, and the step fails even if that run passes.
|
||||
# NODE_OPTIONS chooses the reporter because yarn adds its arguments
|
||||
# after the test files, where node would take a reporter option for one
|
||||
# more file.
|
||||
# node:22-alpine as of 2026-02-22
|
||||
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend-test
|
||||
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend
|
||||
WORKDIR /app
|
||||
COPY package.json yarn.lock ./
|
||||
RUN yarn install --frozen-lockfile
|
||||
# vite.config.js reads the commit for the page's footer with git.
|
||||
RUN apk add --no-cache git
|
||||
COPY . .
|
||||
RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \
|
||||
{ echo "--- Rerunning with every test listed for details ---"; \
|
||||
NODE_OPTIONS=--test-reporter=spec timeout 90 yarn --silent run test; \
|
||||
exit 1; }
|
||||
RUN yarn build
|
||||
|
||||
# Test phase: the backend's tests with the race detector and coverage,
|
||||
# and the frontend's through the copy from the frontend-test stage at
|
||||
# the end. -race needs cgo and so a C compiler, which the Debian Go image
|
||||
# and the frontend's through the copy from the frontend stage at the
|
||||
# end. -race needs cgo and so a C compiler, which the Debian Go image
|
||||
# ships and the alpine one does not. -timeout 90s is a backstop above
|
||||
# the 60-second cap on the suite. The rerun with -v only shows details:
|
||||
# the step fails however it ends, because the first run already failed.
|
||||
@@ -71,21 +75,9 @@ RUN --mount=type=tmpfs,target=/go/pkg/mod \
|
||||
go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
# Nothing is wanted from the frontend-test stage; the copy is what makes
|
||||
# Nothing is wanted from the frontend stage; the copy is what makes
|
||||
# this phase run its tests.
|
||||
COPY --from=frontend-test /app/yarn.lock /dev/null
|
||||
|
||||
# Frontend build stage: the production build into dist/, which the
|
||||
# runtime stage serves, on the yarn packages and source of the
|
||||
# frontend-test stage. Nothing is wanted from the two phases; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed.
|
||||
FROM frontend-test AS frontend-build
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
# vite.config.js reads the commit for the page's footer with git.
|
||||
RUN apk add --no-cache git
|
||||
RUN yarn build
|
||||
COPY --from=frontend /app/yarn.lock /dev/null
|
||||
|
||||
# Backend build stage. Nothing is wanted from the two phases; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
@@ -139,7 +131,7 @@ RUN addgroup -g 1000 -S netwatch && \
|
||||
RUN rm /etc/nginx/conf.d/default.conf
|
||||
COPY configs/nginx.conf /etc/nginx/templates/netwatch.conf.template
|
||||
COPY configs/security-headers.conf /etc/nginx/security-headers.conf
|
||||
COPY --from=frontend-build /app/dist /usr/share/nginx/html
|
||||
COPY --from=frontend /app/dist /usr/share/nginx/html
|
||||
COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server
|
||||
COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
|
||||
|
||||
@@ -59,9 +59,8 @@ either one is broken. We provide:
|
||||
- `script/clean` — remove what `script/build` writes
|
||||
- `script/projectname` — print the project name (used for the Docker image tag)
|
||||
- `script/test` — build the `test` phase of `Dockerfile` without the cache: the
|
||||
frontend's unit tests in its `frontend-test` stage, and the backend's Go tests
|
||||
with the race detector and coverage. The frontend's production build is left
|
||||
to the image build, in the `frontend-build` stage
|
||||
frontend's unit tests and production build in its `frontend` stage, and the
|
||||
backend's Go tests with the race detector and coverage
|
||||
- `script/lint` — build the `lint` phase of `Dockerfile` without the cache:
|
||||
eslint in its `frontend-lint` stage, and golangci-lint over the Go packages
|
||||
- `script/fmt` — format all files (writes): prettier over the JavaScript, CSS,
|
||||
@@ -79,8 +78,8 @@ either one is broken. We provide:
|
||||
- `script/frontend-test` — run the unit tests in `test/unit/` on the host with
|
||||
Node's built-in test runner, through the `test` script in `package.json`, and
|
||||
if any fails, run them again listing every test, and fail; then the production
|
||||
build. Each test run has a 90-second timeout. `make test` runs the same tests
|
||||
in Docker, without the production build
|
||||
build. Each test run has a 90-second timeout. `make test` runs the same in
|
||||
Docker
|
||||
- `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with
|
||||
prettier (writes)
|
||||
- `script/frontend-fmt-check` — check prettier formatting (read-only)
|
||||
@@ -150,11 +149,9 @@ same-origin path `/api/v1/reports`: a random per-browser `clientId` kept in
|
||||
`localStorage`, `geo` sent as null, and each host's unreported, non-paused
|
||||
samples (timestamp, latency, error). A per-host high-water mark makes every
|
||||
report a delta, so only new samples are sent; the mark advances only on a
|
||||
delivered report, and while paused nothing is sent. Changing the interval clears
|
||||
each host's history on the page; the samples in it not yet reported still go in
|
||||
the next report. Delivery failure is quiet — one debug-log line per outage,
|
||||
retried at the next interval, never blocking probing. The report-building step
|
||||
is a pure function of host state.
|
||||
delivered report, and while paused nothing is sent. Delivery failure is quiet —
|
||||
one debug-log line per outage, retried at the next interval, never blocking
|
||||
probing. The report-building step is a pure function of host state.
|
||||
|
||||
### Backend
|
||||
|
||||
@@ -200,25 +197,23 @@ image, `make run` builds it and runs it on port 8080.
|
||||
| `DATA_DIR` | `./data/reports` | Directory for compressed reports |
|
||||
| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) |
|
||||
| `DEBUG` | `false` | Enable debug logging |
|
||||
| `TRUSTED_PROXIES` | RFC 1918 ranges | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
||||
| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution |
|
||||
| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) |
|
||||
| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) |
|
||||
| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) |
|
||||
| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) |
|
||||
| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) |
|
||||
|
||||
`TRUSTED_PROXIES` defaults to `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. A list
|
||||
replaces the default; set and empty, the server trusts no proxy. Loopback is not
|
||||
in the default, so a reverse proxy on the same host has to be named, as in
|
||||
`TRUSTED_PROXIES=127.0.0.1/32`. A request whose direct peer is outside this set
|
||||
has its forwarded headers ignored, and the direct peer is logged and
|
||||
rate-limited instead. The container image does not use this default; see
|
||||
[Deployment](#deployment).
|
||||
`TRUSTED_PROXIES` defaults to
|
||||
`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback
|
||||
entries cover a reverse proxy on the same host. A request whose direct peer is
|
||||
outside this set has its forwarded headers ignored, and the direct peer is
|
||||
logged and rate-limited instead. The container image does not use this default;
|
||||
see [Deployment](#deployment).
|
||||
|
||||
A variable set to a value the server cannot use, such as `PORT=abc`,
|
||||
`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from
|
||||
starting, with an error naming the variable. An empty variable counts as unset,
|
||||
except `TRUSTED_PROXIES`.
|
||||
starting, with an error naming the variable. An empty variable counts as unset.
|
||||
|
||||
#### Report storage
|
||||
|
||||
@@ -231,14 +226,6 @@ the numbers: its file, if it was created, is removed. The file stays, counted
|
||||
toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too.
|
||||
Files are created with `O_EXCL` to prevent overwrites.
|
||||
|
||||
The reports of a failed write stay in memory, and the next flush, within a
|
||||
minute, tries again to write them. From the failed write until a write succeeds,
|
||||
`POST /api/v1/reports` refuses every report with 500 and keeps nothing of it, so
|
||||
the page keeps its samples and sends them again at its next report. Reports
|
||||
already answered 200 are lost only if the server stops before a write succeeds;
|
||||
the final flush at stop tries them too, and if it fails, the process exits
|
||||
non-zero.
|
||||
|
||||
#### Report limits
|
||||
|
||||
`POST /api/v1/reports` takes reports from anyone who can reach it, without
|
||||
@@ -263,16 +250,16 @@ credentials, so it is bounded instead. Each refusal below answers with the same
|
||||
`X-RateLimit-Reset` headers.
|
||||
- **Size cap.** The report files in `DATA_DIR` may total at most
|
||||
`DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports
|
||||
waiting in memory count at their uncompressed size until they are written,
|
||||
those of a failed write included; the part of its file written is removed.
|
||||
When a report would take the total past the cap, the oldest report files are
|
||||
deleted to make room, and each deletion is logged with the file's name and
|
||||
size; a file still being written is never deleted. A report is refused with
|
||||
507, and nothing of it is stored, only when the reports waiting to be written
|
||||
fill the cap on their own, and then no file is deleted. At start, report files
|
||||
past the cap, as after lowering it, are deleted the same way. So the cap is
|
||||
how much of the newest reports is kept: the default of 1 GiB is small enough
|
||||
for any host; set it to the space you can give `DATA_DIR`.
|
||||
waiting in memory count at their uncompressed size until they are written;
|
||||
those lost to a failed write stop counting, and the part of its file written
|
||||
is removed. When a report would take the total past the cap, the oldest report
|
||||
files are deleted to make room, and each deletion is logged with the file's
|
||||
name and size; a file still being written is never deleted. A report is
|
||||
refused with 507, and nothing of it is stored, only when the reports waiting
|
||||
to be written fill the cap on their own, and then no file is deleted. At
|
||||
start, report files past the cap, as after lowering it, are deleted the same
|
||||
way. So the cap is how much of the newest reports is kept: the default of 1
|
||||
GiB is small enough for any host; set it to the space you can give `DATA_DIR`.
|
||||
|
||||
#### CORS
|
||||
|
||||
@@ -423,10 +410,9 @@ address nginx passes on and no other. The image:
|
||||
|
||||
- Listens on port 8080 by default (override with `PORT` env var)
|
||||
- Takes the client address from `X-Forwarded-For` only on requests from the
|
||||
reverse proxies named in `TRUSTED_PROXIES`, by default from any address in
|
||||
`10.0.0.0/8`, `172.16.0.0/12` or `192.168.0.0/16`. `bin/entrypoint.sh` checks
|
||||
each entry with `netwatch-server check-cidr`, which parses it as the backend
|
||||
parses its own `TRUSTED_PROXIES`
|
||||
reverse proxies named in `TRUSTED_PROXIES`, and by default from none.
|
||||
`bin/entrypoint.sh` checks each entry with `netwatch-server check-cidr`, which
|
||||
parses it as the backend parses its own `TRUSTED_PROXIES`
|
||||
- Sends access logs to stdout
|
||||
- Caches static assets with immutable headers
|
||||
- Sends the security headers `REPO_POLICIES.md` requires on every response, as
|
||||
@@ -445,9 +431,9 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
||||
|
||||
- **Port:** container port `8080`.
|
||||
- **Volume:** container path `/data`; the reports are kept in `/data/reports`.
|
||||
- **Environment variables:** none is required. An empty one counts as unset,
|
||||
except `TRUSTED_PROXIES`, and one set to a value netwatch cannot use stops the
|
||||
container at start, with the reason in its log.
|
||||
- **Environment variables:** none is required. An empty one counts as unset, and
|
||||
one set to a value netwatch cannot use stops the container at start, with the
|
||||
reason in its log.
|
||||
- `PORT`, default `8080`: the container port, from 1 to 65535. `8081` cannot
|
||||
be used: the backend listens on it inside the container
|
||||
- `REPORTS_PER_MINUTE`, default `60`: reports each client address may send a
|
||||
@@ -461,17 +447,16 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs:
|
||||
in: `/data` or a path below it, with no `.` or `..` part and no extra `/`.
|
||||
The container also stops if the path goes through a symbolic link that
|
||||
leads out of `/data` or is written as a full path
|
||||
- `TRUSTED_PROXIES`, default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`: the
|
||||
addresses the reverse proxy in front of the container connects from, as IP
|
||||
addresses or CIDRs separated by commas. nginx takes the client address
|
||||
from `X-Forwarded-For` only on a request from one of them, and the rate
|
||||
limit counts that address. A list replaces the default. Set and empty,
|
||||
`X-Forwarded-For` is ignored and every client behind the proxy shares the
|
||||
proxy's one allowance of `REPORTS_PER_MINUTE`. Any client that connects
|
||||
from one of these addresses can write its own `X-Forwarded-For`, and
|
||||
through a port Docker publishes, every client may connect from the Docker
|
||||
network's gateway, such as `172.17.0.1`, which the default includes; where
|
||||
that matters, name only the addresses nothing but the proxy connects from.
|
||||
- `TRUSTED_PROXIES`, default empty: set it to the address the reverse proxy
|
||||
in front of the container connects from, as an IP address or CIDR; several
|
||||
are separated by commas. nginx takes the client address from
|
||||
`X-Forwarded-For` only on a request from one of them, and the rate limit
|
||||
counts that address. Unset, `X-Forwarded-For` is ignored and every client
|
||||
behind the proxy shares the proxy's one allowance of `REPORTS_PER_MINUTE`.
|
||||
Name only addresses nothing but the proxy connects from: any client that
|
||||
connects from one can write its own `X-Forwarded-For`, and through a port
|
||||
Docker publishes, every client may connect from the Docker network's
|
||||
gateway, such as `172.17.0.1`.
|
||||
- `METRICS_USERNAME` and `METRICS_PASSWORD`, default empty: with both set,
|
||||
the backend records Prometheus metrics of its requests and serves them at
|
||||
`/metrics` on the container port, to requests with this user name and
|
||||
|
||||
@@ -21,66 +21,6 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-14: stopping the server waits for a report file write already under
|
||||
way, the periodic flush's or the one `Append` starts when 10 MiB are waiting,
|
||||
before the final flush
|
||||
([#122](https://git.eeqj.de/sneak/netwatch/issues/122)). It used to return at
|
||||
once, and the process exiting cut the write off, losing its reports. A write
|
||||
under way that fails puts its reports back in memory, so the final flush
|
||||
writes them; the stop fails, and the process exits non-zero, only if that
|
||||
flush fails too. Two tests hold the periodic write and the one at 10 MiB open
|
||||
and find the stop waiting until each is released; a third makes the held write
|
||||
fail and finds its report written by the final flush. They run in a
|
||||
`testing/synctest` bubble, so the periodic flush comes without waiting a
|
||||
minute
|
||||
- 2026-10-14: `make test` no longer runs the frontend's production build, which
|
||||
the tests do not need ([#118](https://git.eeqj.de/sneak/netwatch/issues/118)).
|
||||
`yarn build` moved out of the stage that runs the frontend's unit tests, now
|
||||
named `frontend-test`, into a new `frontend-build` stage, which starts from it
|
||||
and, like the `builder` stage, waits on the lint and test phases, so the image
|
||||
still cannot be built unless they pass. The runtime stage copies `dist/` from
|
||||
it, and the `git` that `vite.config.js` reads the commit with is installed
|
||||
there. No build cache was added
|
||||
- 2026-10-14: opening the debug log no longer raises the measured latencies
|
||||
([#131](https://git.eeqj.de/sneak/netwatch/issues/131)). With the panel open,
|
||||
each line written drew the panel again with every line before it, up to 1000,
|
||||
and a check that ends while the page draws is timed only when the drawing is
|
||||
done. Each line is now added at the end of the open panel, its first line is
|
||||
dropped once it holds more than 1000, and it scrolls to its newest line once a
|
||||
frame rather than once a line; the whole panel is drawn only when it is
|
||||
opened. A unit test writes more lines than the panel keeps and checks that the
|
||||
lines already shown are left as they are
|
||||
- 2026-10-14: unset, the backend's own `TRUSTED_PROXIES` trusts `10.0.0.0/8`,
|
||||
`172.16.0.0/12` and `192.168.0.0/16`, and no longer loopback; a list replaces
|
||||
that; set and empty, it trusts no proxy, where it used to get the default, as
|
||||
viper reads an empty environment variable as unset
|
||||
([#129](https://git.eeqj.de/sneak/netwatch/issues/129)). A value it cannot
|
||||
parse still stops the start, and the image still starts the backend with
|
||||
`TRUSTED_PROXIES=127.0.0.1/32`. Tests load the config with it unset, empty and
|
||||
set to a list
|
||||
- 2026-10-14: while report files cannot be written, `POST /api/v1/reports`
|
||||
answers 500, and storing recovers once writing works again, with no restart
|
||||
([#123](https://git.eeqj.de/sneak/netwatch/issues/123)). A failed write puts
|
||||
its reports back in the report buffer, still counted against
|
||||
`DATA_DIR_MAX_BYTES`, where they used to be lost; the next flush, within a
|
||||
minute, tries them again. Until a write succeeds, every report is refused and
|
||||
nothing of it is kept, so the buffer does not grow while writes fail. A test
|
||||
posts reports through the report handler before, during and after a failure;
|
||||
others check that a report appended while a write runs is kept, behind that
|
||||
write's own, when the write fails, and that the cap still holds exactly once a
|
||||
failed write's reports are written
|
||||
- 2026-10-14: a request whose handler panics is logged with status 500 and
|
||||
counted with `code` 500 in the metrics
|
||||
([#125](https://git.eeqj.de/sneak/netwatch/issues/125)), not 200. `Recoverer`
|
||||
is still the first middleware; `Write500OnPanic`, after the request log and
|
||||
after the metrics, writes the 500 as the panic passes and lets it go on to
|
||||
Sentry and `Recoverer`, which now writes a 500 only when no status was written
|
||||
- 2026-10-14: with `TRUSTED_PROXIES` unset, the container's nginx trusts the
|
||||
three private ranges of RFC 1918, `10.0.0.0/8`, `172.16.0.0/12` and
|
||||
`192.168.0.0/16` ([#119](https://git.eeqj.de/sneak/netwatch/issues/119)), as
|
||||
sneak ruled on 2026-09-22 for every app; it trusted no proxy before. A list
|
||||
replaces them, and set and empty it trusts none. The backend inside the
|
||||
container still trusts only `127.0.0.1/32`
|
||||
- 2026-10-14: `POST /api/v1/reports` takes only a report sent with
|
||||
`Content-Type: application/json`, parameters such as `charset` allowed, and
|
||||
refuses any other with 415 and the usual error body, storing nothing
|
||||
@@ -92,18 +32,6 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
||||
`application/x-www-form-urlencoded`, as `multipart/form-data`, with no
|
||||
`Content-Type`, and as `application/json` with and without a charset, and
|
||||
checks which are stored
|
||||
- 2026-10-14: changing the interval no longer drops the samples not yet reported
|
||||
to the backend ([#124](https://git.eeqj.de/sneak/netwatch/issues/124)). The
|
||||
interval menu resets each host with the new `HostState.reset`, which moves the
|
||||
history's entries to `clearedHistory`, keeping the last 100 of all it has
|
||||
cleared, and `buildReport` reads both, so the next report has the samples
|
||||
recorded before the change that no report has delivered. Until its next check,
|
||||
such a host is sent with the status of its last check, not `pending`. The
|
||||
page's history still starts over, and the per-host marks still leave out of a
|
||||
report the samples a delivered one had. Unit tests reset a host as the
|
||||
interval menu does and find its unreported samples in the next report, and
|
||||
only the last 100 after more were cleared; `web/src/main.js` now exports
|
||||
`Reporter` for them
|
||||
- 2026-10-14: the repo has the layout `REPO_POLICIES.md` gives
|
||||
([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). The Go module is at the
|
||||
root (`go.mod`, `go.sum`, `.golangci.yml`, `cmd/netwatch-server/`,
|
||||
@@ -523,9 +451,3 @@ Run `make frontend-viewport-test` in CI as its own step; it is not part of
|
||||
- A configurable host list (an environment variable or a config file)
|
||||
- Export of the latency history (CSV or JSON)
|
||||
- A notification when the health status changes to DEGRADED
|
||||
- `make test` within the 20-second target
|
||||
([#118](https://git.eeqj.de/sneak/netwatch/issues/118)): every run compiles
|
||||
the Go standard library and every dependency with the race detector from
|
||||
nothing. Whether test builds may keep Go's compiled packages between runs
|
||||
waits on sneak's ruling in
|
||||
[prompts#113](https://git.eeqj.de/sneak/prompts/issues/113)
|
||||
|
||||
+4
-6
@@ -34,11 +34,9 @@ fi
|
||||
|
||||
# TRUSTED_PROXIES names the reverse proxies in front of the container,
|
||||
# as IP addresses or CIDRs separated by commas. nginx takes the client
|
||||
# address from X-Forwarded-For only on a request from one of them.
|
||||
# Unset, they are the three private ranges of RFC 1918; set and empty,
|
||||
# nginx trusts no one, which is why the default below follows - and not
|
||||
# :-. configs/nginx.conf includes the file written here, one
|
||||
# set_real_ip_from line per entry.
|
||||
# address from X-Forwarded-For only on a request from one of them, so
|
||||
# unset or empty, it trusts no one. configs/nginx.conf includes the file
|
||||
# written here, one set_real_ip_from line per entry.
|
||||
#
|
||||
# nginx looks up an entry it cannot read as an address as a hostname,
|
||||
# and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry
|
||||
@@ -47,7 +45,7 @@ fi
|
||||
# TRUSTED_PROXIES. Its error, naming the CIDR, is dropped for the one
|
||||
# below, naming the entry as written. set -f keeps a * in an entry from
|
||||
# becoming a list of file names.
|
||||
TRUSTED_PROXIES="${TRUSTED_PROXIES-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16}"
|
||||
TRUSTED_PROXIES="${TRUSTED_PROXIES:-}"
|
||||
set -f
|
||||
for proxy in $(printf '%s' "$TRUSTED_PROXIES" | tr ',' ' '); do
|
||||
case "$proxy" in
|
||||
|
||||
+3
-4
@@ -18,10 +18,9 @@ server {
|
||||
|
||||
# The client address comes from X-Forwarded-For only on a request
|
||||
# from the reverse proxies in TRUSTED_PROXIES: bin/entrypoint.sh
|
||||
# writes one set_real_ip_from line for each into this file. Unset,
|
||||
# TRUSTED_PROXIES is the three private ranges of RFC 1918; set and
|
||||
# empty, the file is empty and the client address is the one each
|
||||
# request comes from.
|
||||
# writes one set_real_ip_from line for each into this file, and
|
||||
# leaves it empty when TRUSTED_PROXIES is unset, so that by default
|
||||
# the client address is the one each request comes from.
|
||||
include /etc/nginx/trusted-proxies.conf;
|
||||
real_ip_header X-Forwarded-For;
|
||||
real_ip_recursive on;
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"math"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -22,11 +21,13 @@ import (
|
||||
)
|
||||
|
||||
// defaultTrustedProxies lists the networks whose forwarded
|
||||
// headers are honoured when TRUSTED_PROXIES is unset: the
|
||||
// RFC 1918 ranges. The container image does not use it:
|
||||
// headers are honoured by default: IPv4 and IPv6 loopback,
|
||||
// for a reverse proxy on the same host, and the RFC1918
|
||||
// ranges. The container image does not use it:
|
||||
// bin/entrypoint.sh gives the server 127.0.0.1/32, since
|
||||
// nginx is its only client there.
|
||||
const defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||
const defaultTrustedProxies = "127.0.0.1/32,::1/128," +
|
||||
"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||
|
||||
// Default limits on stored reports; README.md gives the reasons for
|
||||
// these values.
|
||||
@@ -134,14 +135,6 @@ func New(
|
||||
viper.GetString("DEBUG"), errNotBool)
|
||||
}
|
||||
|
||||
// An empty TRUSTED_PROXIES trusts no proxy, but viper reads an
|
||||
// empty environment variable as unset and would give the default,
|
||||
// so the environment is read first.
|
||||
trustedProxies, set := os.LookupEnv("TRUSTED_PROXIES")
|
||||
if !set {
|
||||
trustedProxies = viper.GetString("TRUSTED_PROXIES")
|
||||
}
|
||||
|
||||
s := &Config{
|
||||
BindAddress: viper.GetString("BIND_ADDRESS"),
|
||||
CORSAllowedOrigins: splitList(viper.GetString("CORS_ALLOWED_ORIGINS")),
|
||||
@@ -153,7 +146,7 @@ func New(
|
||||
Port: port,
|
||||
ReportsPerMinute: viper.GetInt("REPORTS_PER_MINUTE"),
|
||||
SentryDSN: viper.GetString("SENTRY_DSN"),
|
||||
TrustedProxies: splitList(trustedProxies),
|
||||
TrustedProxies: splitList(viper.GetString("TRUSTED_PROXIES")),
|
||||
log: log,
|
||||
params: ¶ms,
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -31,62 +29,6 @@ func requireConfigError(t *testing.T, setting string) {
|
||||
}
|
||||
}
|
||||
|
||||
// requireTrustedProxies builds the config as main does and fails the
|
||||
// test unless its TrustedProxies are want.
|
||||
func requireTrustedProxies(t *testing.T, want []string) {
|
||||
t.Helper()
|
||||
|
||||
var cfg *config.Config
|
||||
|
||||
app := fx.New(
|
||||
fx.NopLogger,
|
||||
fx.Provide(globals.New, logger.New, config.New),
|
||||
fx.Populate(&cfg),
|
||||
)
|
||||
|
||||
err := app.Err()
|
||||
if err != nil {
|
||||
t.Fatalf("config error = %v", err)
|
||||
}
|
||||
|
||||
if !slices.Equal(cfg.TrustedProxies, want) {
|
||||
t.Fatalf("TrustedProxies = %q, want %q", cfg.TrustedProxies, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrustedProxiesUnsetTrustsRFC1918: unset, the server trusts the
|
||||
// RFC 1918 ranges and nothing else, loopback included.
|
||||
func TestTrustedProxiesUnsetTrustsRFC1918(t *testing.T) {
|
||||
// t.Setenv first, so the variable is put back as it was after the
|
||||
// test.
|
||||
t.Setenv("TRUSTED_PROXIES", "")
|
||||
|
||||
err := os.Unsetenv("TRUSTED_PROXIES")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
requireTrustedProxies(t,
|
||||
[]string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"})
|
||||
}
|
||||
|
||||
// TestTrustedProxiesEmptyTrustsNothing: set and empty, the server
|
||||
// trusts no proxy. viper alone would read the empty variable as unset
|
||||
// and give the default.
|
||||
func TestTrustedProxiesEmptyTrustsNothing(t *testing.T) {
|
||||
t.Setenv("TRUSTED_PROXIES", "")
|
||||
|
||||
requireTrustedProxies(t, []string{})
|
||||
}
|
||||
|
||||
// TestTrustedProxiesListReplacesDefault: a list is used as given, with
|
||||
// none of the default ranges added.
|
||||
func TestTrustedProxiesListReplacesDefault(t *testing.T) {
|
||||
t.Setenv("TRUSTED_PROXIES", "172.17.0.1/32, 2001:db8::/32")
|
||||
|
||||
requireTrustedProxies(t, []string{"172.17.0.1/32", "2001:db8::/32"})
|
||||
}
|
||||
|
||||
// TestSettingsLoadAsGiven: valid values pass the checks and are used
|
||||
// as given. bin/entrypoint.sh starts the server with these
|
||||
// BIND_ADDRESS and PORT values.
|
||||
|
||||
@@ -93,7 +93,7 @@ func (s *Handlers) decodeErrorStatus(err error) int {
|
||||
|
||||
// appendErrorStatus logs a failure to store a report and returns
|
||||
// the status to send: 507 when the reports waiting to be written fill
|
||||
// the size cap, otherwise 500, as when report files cannot be written.
|
||||
// the size cap, otherwise 500.
|
||||
func (s *Handlers) appendErrorStatus(err error) int {
|
||||
if errors.Is(err, reportbuf.ErrFull) {
|
||||
s.log.Warn("report refused: " +
|
||||
|
||||
@@ -297,16 +297,12 @@ func (s *Middleware) MaxBodyBytes(
|
||||
|
||||
// Recoverer returns middleware that recovers from a panic in a
|
||||
// downstream handler, logs the panic and stack trace through
|
||||
// slog, and responds 500 with no body, unless a status was already
|
||||
// written, as Write500OnPanic writes one for a handler's panic.
|
||||
// http.ErrAbortHandler is re-panicked so the server can abort the
|
||||
// response as intended.
|
||||
// slog, and responds 500 with no body. http.ErrAbortHandler is
|
||||
// re-panicked so the server can abort the response as intended.
|
||||
func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
||||
|
||||
defer func() {
|
||||
rec := recover()
|
||||
if rec == nil {
|
||||
@@ -324,43 +320,10 @@ func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
|
||||
"stack", string(debug.Stack()),
|
||||
)
|
||||
|
||||
if ww.Status() == 0 {
|
||||
ww.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}()
|
||||
|
||||
next.ServeHTTP(ww, r)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Write500OnPanic returns middleware that writes 500 when a downstream
|
||||
// handler panics before writing a status, and lets the panic go on to
|
||||
// Sentry and Recoverer. Recoverer, which comes first, would write its
|
||||
// 500 only after the request log and the metrics had recorded the
|
||||
// status they saw; placed after each of them, this makes them record
|
||||
// the 500.
|
||||
func (s *Middleware) Write500OnPanic() func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
||||
|
||||
defer func() {
|
||||
rec := recover()
|
||||
if rec == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if ww.Status() == 0 {
|
||||
ww.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
panic(rec)
|
||||
}()
|
||||
|
||||
next.ServeHTTP(ww, r)
|
||||
next.ServeHTTP(w, r)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -17,9 +16,7 @@ import (
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
"sneak.berlin/go/netwatch/internal/middleware"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -326,172 +323,6 @@ func TestRecovererRepanicsOnAbortHandler(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandlerPanicIsLoggedAndCountedAs500 sends a request to a handler
|
||||
// that panics through the middleware SetupRoutes puts around a route
|
||||
// with metrics, in its order, leaving out those that write no status
|
||||
// for a panic and Sentry, which TestSentry covers. The request log line
|
||||
// and the metrics must give the 500 the client gets, and the panic must
|
||||
// still reach Recoverer, which logs it.
|
||||
func TestHandlerPanicIsLoggedAndCountedAs500(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var logbuf bytes.Buffer
|
||||
|
||||
mw := middleware.NewWithLogger(
|
||||
slog.New(slog.NewJSONHandler(&logbuf, nil)),
|
||||
)
|
||||
registry := prometheus.NewRegistry()
|
||||
|
||||
handler := chi.Chain(
|
||||
mw.Recoverer(),
|
||||
chimiddleware.RequestID,
|
||||
mw.Logging(),
|
||||
mw.Write500OnPanic(),
|
||||
mw.Metrics(registry),
|
||||
mw.Write500OnPanic(),
|
||||
).HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
panic("boom")
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", http.NoBody)
|
||||
handler.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status = %d, want %d",
|
||||
rec.Code, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
request := findLogLine(t, logbuf.Bytes(), "request")
|
||||
if request.Status != http.StatusInternalServerError {
|
||||
t.Errorf("request log status = %d, want %d",
|
||||
request.Status, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
recovered := findLogLine(t, logbuf.Bytes(), "panic recovered")
|
||||
if recovered.Panic != "boom" {
|
||||
t.Errorf("panic recovered line has panic %q, want %q",
|
||||
recovered.Panic, "boom")
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"http_request_duration_seconds", "http_response_size_bytes",
|
||||
} {
|
||||
codes := metricCodes(t, registry, name)
|
||||
if len(codes) != 1 || codes[0] != "500" {
|
||||
t.Errorf("%s code labels = %v, want [500]", name, codes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loggedLine holds the fields the tests read from a JSON log line.
|
||||
type loggedLine struct {
|
||||
Msg string `json:"msg"`
|
||||
Status int `json:"status"`
|
||||
Panic string `json:"panic"`
|
||||
}
|
||||
|
||||
// findLogLine returns the first of the JSON log lines in logged whose
|
||||
// message is msg.
|
||||
func findLogLine(t *testing.T, logged []byte, msg string) loggedLine {
|
||||
t.Helper()
|
||||
|
||||
dec := json.NewDecoder(bytes.NewReader(logged))
|
||||
|
||||
for {
|
||||
var line loggedLine
|
||||
|
||||
err := dec.Decode(&line)
|
||||
if err != nil {
|
||||
t.Fatalf("no %q line in the log: %v (%q)", msg, err, logged)
|
||||
}
|
||||
|
||||
if line.Msg == msg {
|
||||
return line
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// metricCodes returns the code label of each series of the metric
|
||||
// family name in registry.
|
||||
func metricCodes(
|
||||
t *testing.T,
|
||||
registry *prometheus.Registry,
|
||||
name string,
|
||||
) []string {
|
||||
t.Helper()
|
||||
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatalf("gather metrics: %v", err)
|
||||
}
|
||||
|
||||
var codes []string
|
||||
|
||||
for _, family := range families {
|
||||
if family.GetName() != name {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, metric := range family.GetMetric() {
|
||||
for _, label := range metric.GetLabel() {
|
||||
if label.GetName() == "code" {
|
||||
codes = append(codes, label.GetValue())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return codes
|
||||
}
|
||||
|
||||
// TestRecovererWritesNoSecondStatus checks, on a real server, that
|
||||
// Recoverer does not write its 500 again after Write500OnPanic has
|
||||
// written one, which the server would log as a superfluous
|
||||
// WriteHeader call.
|
||||
func TestRecovererWritesNoSecondStatus(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var serverLog bytes.Buffer
|
||||
|
||||
mw := middleware.NewWithLogger(slog.New(slog.DiscardHandler))
|
||||
|
||||
server := httptest.NewUnstartedServer(chi.Chain(
|
||||
mw.Recoverer(),
|
||||
mw.Write500OnPanic(),
|
||||
).HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
panic("boom")
|
||||
}))
|
||||
server.Config.ErrorLog = log.New(&serverLog, "", 0)
|
||||
server.Start()
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(),
|
||||
http.MethodGet, server.URL, http.NoBody)
|
||||
if err != nil {
|
||||
t.Fatalf("new request: %v", err)
|
||||
}
|
||||
|
||||
resp, err := server.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("request: %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Body.Close()
|
||||
|
||||
// Close waits for the handler to return, so whatever the server
|
||||
// logs for this request is in serverLog after it.
|
||||
server.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusInternalServerError {
|
||||
t.Errorf("status = %d, want %d",
|
||||
resp.StatusCode, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if serverLog.Len() != 0 {
|
||||
t.Errorf("server logged %q", serverLog.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoggingCutsRequestStringsToBound sends an over-long URL and
|
||||
// over-long header values, and checks the request log writes each
|
||||
// one cut to logger.MaxLoggedFieldBytes.
|
||||
|
||||
@@ -91,15 +91,6 @@ type Buffer struct {
|
||||
// of the report files in dataDir, plus the reports waiting to
|
||||
// be written to one at their uncompressed size.
|
||||
usedBytes int64
|
||||
// writeErr is the error of the last write of a report file to
|
||||
// finish, nil if it succeeded or none has. While it is set, Append
|
||||
// refuses reports, and the reports of that write wait in buf for
|
||||
// the next flush.
|
||||
writeErr error
|
||||
// writers are the goroutines that write report files: flushLoop,
|
||||
// and each one Append starts at flushSizeThreshold. OnStop waits
|
||||
// for them before the final flush.
|
||||
writers sync.WaitGroup
|
||||
}
|
||||
|
||||
// New creates a Buffer and registers lifecycle hooks to
|
||||
@@ -151,7 +142,7 @@ func New(
|
||||
|
||||
b.mu.Unlock()
|
||||
|
||||
b.writers.Go(b.flushLoop)
|
||||
go b.flushLoop()
|
||||
|
||||
return nil
|
||||
},
|
||||
@@ -162,14 +153,7 @@ func New(
|
||||
var err error
|
||||
|
||||
b.stopOnce.Do(func() {
|
||||
// Closing done ends flushLoop after any flush it is
|
||||
// running. Wait for it and for the writes Append
|
||||
// started, which the process exiting would cut off;
|
||||
// one that fails puts its reports back in buf, for the
|
||||
// final flush to write.
|
||||
close(b.done)
|
||||
b.writers.Wait()
|
||||
|
||||
err = b.flushLocked()
|
||||
})
|
||||
|
||||
@@ -183,12 +167,11 @@ func New(
|
||||
}
|
||||
|
||||
// Append marshals v as a single JSON line and appends it to
|
||||
// the buffer. It stores nothing and returns an error if the last
|
||||
// write of a report file failed, until a write succeeds again. If the
|
||||
// line would take usedBytes past maxBytes, the oldest report files are
|
||||
// deleted to make room; it stores nothing and returns ErrFull if that
|
||||
// cannot make room. If the buffer reaches the size threshold, it is
|
||||
// drained and written to disk asynchronously.
|
||||
// the buffer. If the line would take usedBytes past maxBytes, the
|
||||
// oldest report files are deleted to make room; it stores nothing
|
||||
// and returns ErrFull if that cannot make room. If the buffer
|
||||
// reaches the size threshold, it is drained and written to disk
|
||||
// asynchronously.
|
||||
func (b *Buffer) Append(v any) error {
|
||||
line, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
@@ -199,13 +182,6 @@ func (b *Buffer) Append(v any) error {
|
||||
|
||||
b.mu.Lock()
|
||||
|
||||
if b.writeErr != nil {
|
||||
err = fmt.Errorf("report files cannot be written: %w", b.writeErr)
|
||||
b.mu.Unlock()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
b.deleteOldestFiles(lineBytes)
|
||||
|
||||
if b.usedBytes+lineBytes > b.maxBytes {
|
||||
@@ -222,12 +198,12 @@ func (b *Buffer) Append(v any) error {
|
||||
data := b.drainBuf()
|
||||
b.mu.Unlock()
|
||||
|
||||
b.writers.Go(func() {
|
||||
go func() {
|
||||
writeErr := b.writeFile(data)
|
||||
if writeErr != nil {
|
||||
b.log.Error("flush reports failed", "error", writeErr)
|
||||
}
|
||||
})
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -315,8 +291,7 @@ func (b *Buffer) drainBuf() []byte {
|
||||
}
|
||||
|
||||
// writeFile writes data, reports drained from the buffer, to a new
|
||||
// timestamped zstd-compressed JSONL file in the data directory. If the
|
||||
// write fails, it puts the reports back in the buffer.
|
||||
// timestamped zstd-compressed JSONL file in the data directory.
|
||||
func (b *Buffer) writeFile(data []byte) error {
|
||||
// The timestamp comes first, so the names sort by time; the number
|
||||
// after it tells apart files named in the same millisecond.
|
||||
@@ -325,27 +300,19 @@ func (b *Buffer) writeFile(data []byte) error {
|
||||
|
||||
size, err := b.createFile(filepath.Join(b.dataDir, name), data)
|
||||
|
||||
// The reports no longer wait to be written, so they stop counting
|
||||
// at their uncompressed size. If the write failed they are lost;
|
||||
// otherwise they count as the file, which from here on may be
|
||||
// deleted to make room.
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
b.writeErr = err
|
||||
b.usedBytes -= int64(len(data))
|
||||
|
||||
if err != nil {
|
||||
// The reports go back in front of any appended since, for the
|
||||
// next flush to write, and still count at their uncompressed
|
||||
// size.
|
||||
appended := bytes.Clone(b.buf.Bytes())
|
||||
b.buf.Reset()
|
||||
b.buf.Write(data)
|
||||
b.buf.Write(appended)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// The reports no longer wait to be written, so they stop counting
|
||||
// at their uncompressed size and count as the file, which from here
|
||||
// on may be deleted to make room.
|
||||
b.usedBytes -= int64(len(data))
|
||||
b.usedBytes += size
|
||||
b.filesBytes += size
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -15,13 +13,10 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
"sneak.berlin/go/netwatch/internal/handlers"
|
||||
"sneak.berlin/go/netwatch/internal/healthcheck"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
"sneak.berlin/go/netwatch/internal/reportbuf"
|
||||
|
||||
@@ -108,174 +103,9 @@ func TestFailedFinalFlushFailsStop(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopWaitsForPeriodicWrite holds open the write the periodic flush
|
||||
// starts, then stops the app. The stop must wait until the write is
|
||||
// released, and the report must then be in its file. The test runs in
|
||||
// a synctest bubble, whose clock moves on at once when every goroutine
|
||||
// in it is waiting, so the periodic flush does not take a minute.
|
||||
func TestStopWaitsForPeriodicWrite(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("DATA_DIR", dir)
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
buf, app := startApp(t)
|
||||
|
||||
created := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
buf.OnFileCreated(func(*os.File) {
|
||||
created <- struct{}{}
|
||||
|
||||
<-release
|
||||
})
|
||||
|
||||
err := buf.Append(map[string]string{"id": "periodic"})
|
||||
if err != nil {
|
||||
t.Fatalf("append report: %v", err)
|
||||
}
|
||||
|
||||
<-created
|
||||
|
||||
err = stopWhileHeld(t, app, release)
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
|
||||
files, err := readReportFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read report files: %v", err)
|
||||
}
|
||||
|
||||
want := `{"id":"periodic"}` + "\n"
|
||||
if len(files) != 1 || files[0] != want {
|
||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestStopWaitsForWriteAtSizeThreshold holds open the write Append
|
||||
// starts when the buffer reaches FlushSizeThreshold, then stops the
|
||||
// app. The stop must wait until the write is released, and the reports
|
||||
// must then be in its file.
|
||||
func TestStopWaitsForWriteAtSizeThreshold(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("DATA_DIR", dir)
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
buf, app := startApp(t)
|
||||
|
||||
created := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
buf.OnFileCreated(func(*os.File) {
|
||||
created <- struct{}{}
|
||||
|
||||
<-release
|
||||
})
|
||||
|
||||
appended := appendToThreshold(t, buf)
|
||||
|
||||
<-created
|
||||
|
||||
err := stopWhileHeld(t, app, release)
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
|
||||
files, err := readReportFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read report files: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 1 || files[0] != appended {
|
||||
t.Fatalf("%d report files, want one holding the %d bytes "+
|
||||
"appended", len(files), len(appended))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestFinalFlushWritesReportsOfFailedWrite holds open the write the
|
||||
// periodic flush starts, makes it fail, then stops the app. The stop
|
||||
// waits for the write, which puts its report back in the buffer, and
|
||||
// the final flush writes it, so the stop succeeds with nothing lost.
|
||||
func TestFinalFlushWritesReportsOfFailedWrite(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("DATA_DIR", dir)
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
buf, app := startApp(t)
|
||||
|
||||
created := make(chan *os.File)
|
||||
release := make(chan struct{})
|
||||
|
||||
buf.OnFileCreated(func(f *os.File) {
|
||||
created <- f
|
||||
|
||||
<-release
|
||||
})
|
||||
|
||||
err := buf.Append(map[string]string{"id": "failed"})
|
||||
if err != nil {
|
||||
t.Fatalf("append report: %v", err)
|
||||
}
|
||||
|
||||
held := <-created
|
||||
|
||||
// Closing the file under the write makes the write fail.
|
||||
_ = held.Close()
|
||||
|
||||
// The final flush's write goes through.
|
||||
buf.OnFileCreated(func(*os.File) {})
|
||||
|
||||
err = stopWhileHeld(t, app, release)
|
||||
if err != nil {
|
||||
t.Fatalf("stop: %v", err)
|
||||
}
|
||||
|
||||
files, err := readReportFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read report files: %v", err)
|
||||
}
|
||||
|
||||
want := `{"id":"failed"}` + "\n"
|
||||
if len(files) != 1 || files[0] != want {
|
||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// stopWhileHeld stops app while a report file write is held open until
|
||||
// release is closed, and fails the test if the stop returns before
|
||||
// that. It then closes release and returns the stop's error. It must
|
||||
// be called in a synctest bubble.
|
||||
func stopWhileHeld(
|
||||
t *testing.T,
|
||||
app *fxtest.App,
|
||||
release chan<- struct{},
|
||||
) error {
|
||||
t.Helper()
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
|
||||
go func() { stopped <- app.Stop(t.Context()) }()
|
||||
|
||||
// Wait returns once every other goroutine in the bubble is waiting:
|
||||
// the stop has got as far as it can while the write is held.
|
||||
synctest.Wait()
|
||||
|
||||
if len(stopped) != 0 {
|
||||
t.Error("stop returned while a report file write was held open")
|
||||
}
|
||||
|
||||
close(release)
|
||||
|
||||
return <-stopped
|
||||
}
|
||||
|
||||
// startApp starts a Buffer through fx, as main does, with the DATA_DIR
|
||||
// and DATA_DIR_MAX_BYTES the calling test has set, and returns it with
|
||||
// the app, for tests that stop the app themselves.
|
||||
func startApp(t *testing.T) (*reportbuf.Buffer, *fxtest.App) {
|
||||
// startBuffer starts a Buffer through fx, as main does, with the
|
||||
// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
|
||||
func startBuffer(t *testing.T) *reportbuf.Buffer {
|
||||
t.Helper()
|
||||
|
||||
var buf *reportbuf.Buffer
|
||||
@@ -293,16 +123,6 @@ func startApp(t *testing.T) (*reportbuf.Buffer, *fxtest.App) {
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
return buf, app
|
||||
}
|
||||
|
||||
// startBuffer starts a Buffer through fx, as main does, with the
|
||||
// DATA_DIR and DATA_DIR_MAX_BYTES the calling test has set.
|
||||
func startBuffer(t *testing.T) *reportbuf.Buffer {
|
||||
t.Helper()
|
||||
|
||||
buf, _ := startApp(t)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
@@ -687,17 +507,13 @@ func TestFileBeingWrittenIsNeverDeleted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailedWriteKeepsItsReports makes a write fail once its file is
|
||||
// created, under a cap with room for one report. The part of the file
|
||||
// written is removed, so it takes no room, and the report waits in
|
||||
// memory for the next flush, which writes it. Once written, it counts
|
||||
// as its file and no longer as a report waiting, so the cap still holds
|
||||
// exactly: one more report is taken and the one after it is refused.
|
||||
func TestFailedWriteKeepsItsReports(t *testing.T) {
|
||||
report := map[string]string{"id": "kept"}
|
||||
dir := t.TempDir()
|
||||
// TestFailedWriteStopsCounting makes a write fail once its file is
|
||||
// created. Its reports are lost, so they stop counting, and the part of
|
||||
// the file written is removed, so it takes no room.
|
||||
func TestFailedWriteStopsCounting(t *testing.T) {
|
||||
report := map[string]string{"id": "failed"}
|
||||
|
||||
t.Setenv("DATA_DIR", dir)
|
||||
t.Setenv("DATA_DIR", t.TempDir())
|
||||
t.Setenv("DATA_DIR_MAX_BYTES", strconv.Itoa(lineBytes(t, report)))
|
||||
|
||||
buf := startBuffer(t)
|
||||
@@ -728,213 +544,10 @@ func TestFailedWriteKeepsItsReports(t *testing.T) {
|
||||
// Writes from here on, the final one at stop included, succeed.
|
||||
buf.OnFileCreated(func(*os.File) {})
|
||||
|
||||
err = buf.Flush()
|
||||
if err != nil {
|
||||
t.Fatalf("flush after the failed write: %v", err)
|
||||
}
|
||||
|
||||
files, err := readReportFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read report files: %v", err)
|
||||
}
|
||||
|
||||
want := `{"id":"kept"}` + "\n"
|
||||
if len(files) != 1 || files[0] != want {
|
||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
||||
}
|
||||
|
||||
// The file is deleted to make room if need be.
|
||||
err = buf.Append(report)
|
||||
if err != nil {
|
||||
t.Fatalf("report after the written one: %v", err)
|
||||
t.Fatalf("report after the failed write: %v", err)
|
||||
}
|
||||
|
||||
err = buf.Append(report)
|
||||
if !errors.Is(err, reportbuf.ErrFull) {
|
||||
t.Fatalf("report past the cap: error = %v, want ErrFull", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailedWriteKeepsReportsAppendedDuringIt holds a write open,
|
||||
// appends a report while it is held, then makes the held write fail.
|
||||
// Both reports are kept, and the next flush writes them in the order
|
||||
// they were appended, the held write's first.
|
||||
func TestFailedWriteKeepsReportsAppendedDuringIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("DATA_DIR", dir)
|
||||
|
||||
buf := startBuffer(t)
|
||||
|
||||
created := make(chan *os.File)
|
||||
release := make(chan struct{})
|
||||
|
||||
buf.OnFileCreated(func(f *os.File) {
|
||||
created <- f
|
||||
|
||||
<-release
|
||||
})
|
||||
|
||||
err := buf.Append(map[string]string{"id": "held"})
|
||||
if err != nil {
|
||||
t.Fatalf("report of the held write: %v", err)
|
||||
}
|
||||
|
||||
flushed := make(chan error)
|
||||
|
||||
go func() { flushed <- buf.Flush() }()
|
||||
|
||||
held := <-created
|
||||
|
||||
// Errorf, not Fatalf, until the held write is released, so that a
|
||||
// failure here does not leave it held.
|
||||
err = buf.Append(map[string]string{"id": "appended"})
|
||||
if err != nil {
|
||||
t.Errorf("report appended while the write was held: %v", err)
|
||||
}
|
||||
|
||||
// Closing the file under the write makes the write fail.
|
||||
_ = held.Close()
|
||||
|
||||
close(release)
|
||||
|
||||
err = <-flushed
|
||||
if err == nil {
|
||||
t.Fatal("held write succeeded, though its file was closed under it")
|
||||
}
|
||||
|
||||
// Writes from here on, the final one at stop included, succeed.
|
||||
buf.OnFileCreated(func(*os.File) {})
|
||||
|
||||
err = buf.Flush()
|
||||
if err != nil {
|
||||
t.Fatalf("flush after the failed write: %v", err)
|
||||
}
|
||||
|
||||
files, err := readReportFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read report files: %v", err)
|
||||
}
|
||||
|
||||
want := `{"id":"held"}` + "\n" + `{"id":"appended"}` + "\n"
|
||||
if len(files) != 1 || files[0] != want {
|
||||
t.Fatalf("report files = %q, want one holding %q", files, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportsRefusedWhileWritesFail posts reports through the report
|
||||
// handler, as the page does. Once a write has failed, a report is
|
||||
// refused with 500 until a write succeeds again; the reports taken
|
||||
// before and after are stored, and the one refused is not.
|
||||
func TestReportsRefusedWhileWritesFail(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("DATA_DIR", dir)
|
||||
|
||||
buf, h := startBufferWithHandlers(t)
|
||||
|
||||
if code := postReport(t, h, "before"); code != http.StatusOK {
|
||||
t.Fatalf("report before the failed write: status = %d, want %d",
|
||||
code, http.StatusOK)
|
||||
}
|
||||
|
||||
// With the data directory gone, the write has nowhere to go.
|
||||
err := os.Remove(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("remove data dir: %v", err)
|
||||
}
|
||||
|
||||
err = buf.Flush()
|
||||
if err == nil {
|
||||
t.Fatal("flush succeeded with the data directory gone")
|
||||
}
|
||||
|
||||
if code := postReport(t, h, "refused"); code != http.StatusInternalServerError {
|
||||
t.Fatalf("report after the failed write: status = %d, want %d",
|
||||
code, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
err = os.Mkdir(dir, 0o750)
|
||||
if err != nil {
|
||||
t.Fatalf("make data dir: %v", err)
|
||||
}
|
||||
|
||||
err = buf.Flush()
|
||||
if err != nil {
|
||||
t.Fatalf("flush with the data directory back: %v", err)
|
||||
}
|
||||
|
||||
if code := postReport(t, h, "after"); code != http.StatusOK {
|
||||
t.Fatalf("report after a write succeeded: status = %d, want %d",
|
||||
code, http.StatusOK)
|
||||
}
|
||||
|
||||
err = buf.Flush()
|
||||
if err != nil {
|
||||
t.Fatalf("flush of the report after: %v", err)
|
||||
}
|
||||
|
||||
files, err := readReportFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read report files: %v", err)
|
||||
}
|
||||
|
||||
stored := strings.Join(files, "")
|
||||
|
||||
for _, clientID := range []string{"before", "after"} {
|
||||
if !strings.Contains(stored, `"clientId":"`+clientID+`"`) {
|
||||
t.Errorf("report %q not stored", clientID)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(stored, `"clientId":"refused"`) {
|
||||
t.Error("refused report stored")
|
||||
}
|
||||
}
|
||||
|
||||
// startBufferWithHandlers starts a Buffer and the Handlers that store
|
||||
// reports in it through fx, as main does, with the DATA_DIR the calling
|
||||
// test has set.
|
||||
func startBufferWithHandlers(
|
||||
t *testing.T,
|
||||
) (*reportbuf.Buffer, *handlers.Handlers) {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
buf *reportbuf.Buffer
|
||||
h *handlers.Handlers
|
||||
)
|
||||
|
||||
app := fxtest.New(t,
|
||||
fx.Provide(
|
||||
config.New,
|
||||
globals.New,
|
||||
handlers.New,
|
||||
healthcheck.New,
|
||||
logger.New,
|
||||
reportbuf.New,
|
||||
),
|
||||
fx.Populate(&buf, &h),
|
||||
)
|
||||
|
||||
app.RequireStart()
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
return buf, h
|
||||
}
|
||||
|
||||
// postReport posts a report from clientID to the report handler, as the
|
||||
// page does, and returns the status of the answer.
|
||||
func postReport(t *testing.T, h *handlers.Handlers, clientID string) int {
|
||||
t.Helper()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequestWithContext(t.Context(),
|
||||
http.MethodPost, "/api/v1/reports",
|
||||
strings.NewReader(`{"clientId":"`+clientID+`","hosts":[]}`),
|
||||
)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
h.HandleReport().ServeHTTP(rec, req)
|
||||
|
||||
return rec.Code
|
||||
}
|
||||
|
||||
// TestConcurrentAppendsStopAtCap appends from many goroutines at once
|
||||
@@ -1068,32 +681,6 @@ func TestFlushAtSizeThreshold(t *testing.T) {
|
||||
|
||||
buf := startBuffer(t)
|
||||
|
||||
appended := appendToThreshold(t, buf)
|
||||
|
||||
// Append writes the file in the background, so wait for it.
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
|
||||
for {
|
||||
files, err := readReportFiles(dir)
|
||||
if err == nil && len(files) == 1 && files[0] == appended {
|
||||
return
|
||||
}
|
||||
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%d report files (error: %v), want one holding the "+
|
||||
"%d bytes appended", len(files), err, len(appended))
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// appendToThreshold appends reports to buf until it holds
|
||||
// FlushSizeThreshold bytes, so that the last append starts writing them
|
||||
// to a report file, and returns the lines appended.
|
||||
func appendToThreshold(t *testing.T, buf *reportbuf.Buffer) string {
|
||||
t.Helper()
|
||||
|
||||
// Large reports, so the threshold takes a few hundred appends.
|
||||
pad := strings.Repeat("a", 64<<10)
|
||||
|
||||
@@ -1116,7 +703,22 @@ func appendToThreshold(t *testing.T, buf *reportbuf.Buffer) string {
|
||||
appended.WriteByte('\n')
|
||||
}
|
||||
|
||||
return appended.String()
|
||||
// Append writes the file in the background, so wait for it.
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
|
||||
for {
|
||||
files, err := readReportFiles(dir)
|
||||
if err == nil && len(files) == 1 && files[0] == appended.String() {
|
||||
return
|
||||
}
|
||||
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%d report files (error: %v), want one holding the "+
|
||||
"%d bytes appended", len(files), err, appended.Len())
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// reportFilesBytes returns the total size of the report files in dir.
|
||||
|
||||
@@ -34,16 +34,13 @@ func (s *Server) SetupRoutes() {
|
||||
s.router.Use(s.mw.Recoverer())
|
||||
s.router.Use(middleware.RequestID)
|
||||
s.router.Use(s.mw.Logging())
|
||||
// s.mw.Recoverer would write a panic's 500 only after the panic has
|
||||
// passed the request log, so it is written here, where the log sees it.
|
||||
s.router.Use(s.mw.Write500OnPanic())
|
||||
s.router.Use(s.mw.SecurityHeaders())
|
||||
s.router.Use(s.mw.CORS(s.params.Config.CORSAllowedOrigins))
|
||||
s.router.Use(s.mw.MaxBodyBytes(maxRequestBodyBytes))
|
||||
s.router.Use(middleware.Timeout(requestTimeout))
|
||||
|
||||
// Sentry reports a panic, then panics again, so that it still
|
||||
// reaches s.mw.Recoverer.
|
||||
// Sentry reports a panic, then panics again, so that s.mw.Recoverer
|
||||
// still answers 500.
|
||||
if s.params.Config.SentryDSN != "" {
|
||||
s.router.Use(sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle)
|
||||
}
|
||||
@@ -67,8 +64,6 @@ func (s *Server) SetupRoutes() {
|
||||
// other.
|
||||
if s.params.Config.MetricsUsername != "" {
|
||||
r.Use(s.mw.Metrics(registry))
|
||||
// Likewise inside the metrics.
|
||||
r.Use(s.mw.Write500OnPanic())
|
||||
}
|
||||
|
||||
r.Get("/.well-known/healthcheck", s.h.HandleHealthCheck())
|
||||
|
||||
@@ -2,10 +2,9 @@
|
||||
# script/frontend-test: run the frontend test suite on the host: the
|
||||
# unit tests in test/unit/, through the test script in package.json,
|
||||
# then the production build, which fails on broken code. make test runs
|
||||
# the same tests in the frontend-test stage of Dockerfile, and the image
|
||||
# build runs the production build in its frontend-build stage. The tests
|
||||
# print a dot each; if any fails, they run again with every test listed,
|
||||
# and the script fails even if that run passes. NODE_OPTIONS chooses the
|
||||
# the same in the frontend stage of Dockerfile. The tests print a dot
|
||||
# each; if any fails, they run again with every test listed, and the
|
||||
# script fails even if that run passes. NODE_OPTIONS chooses the
|
||||
# reporter because yarn adds its arguments after the test files, where
|
||||
# node would take a reporter option for one more file.
|
||||
set -eu
|
||||
|
||||
@@ -64,8 +64,8 @@ main() {
|
||||
# configs/nginx.conf is a template: the image renders it over its own
|
||||
# default.conf, with the same port and limit bin/entrypoint.sh uses.
|
||||
# The empty file it includes trusts no proxy, as bin/entrypoint.sh
|
||||
# writes it when TRUSTED_PROXIES is set and empty. It also includes
|
||||
# the security headers, so the page runs under the shipped policy.
|
||||
# writes it when TRUSTED_PROXIES is unset. It also includes the
|
||||
# security headers, so the page runs under the shipped policy.
|
||||
docker run -d --rm --name "$SERVER" \
|
||||
--network "$NETWORK" --network-alias netwatch \
|
||||
-e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \
|
||||
|
||||
+8
-139
@@ -13,22 +13,19 @@ import {
|
||||
humanDuration,
|
||||
latencyClass,
|
||||
latencyHex,
|
||||
log,
|
||||
measureLatency,
|
||||
Reporter,
|
||||
tick,
|
||||
} from "../../web/src/main.js";
|
||||
|
||||
// There is no page here, so the tests stand in for it. The debug log looks
|
||||
// for its panel by id and finds none, an element the page creates is a
|
||||
// plain object, and no frame the page asks for ever comes. Each element of
|
||||
// a host's row that tick or greyOutUI draws into is a plain object, made
|
||||
// the first time a test looks it up and kept in elements under its
|
||||
// selector until the next test starts. As on a page, writing its text
|
||||
// replaces its markup; the status dot greyOutUI looks for in it is not
|
||||
// there. Drawing a sparkline does nothing; it looks for the pixel ratio on
|
||||
// window and finds none. In each test, console.error and console.info
|
||||
// print nothing and keep what they are given.
|
||||
// for its panel by id and finds none. Each element of a host's row that
|
||||
// tick or greyOutUI draws into is a plain object, made the first time a
|
||||
// test looks it up and kept in elements under its selector until the next
|
||||
// test starts. As on a page, writing its text replaces its markup; the
|
||||
// status dot greyOutUI looks for in it is not there. Drawing a sparkline
|
||||
// does nothing; it looks for the pixel ratio on window and finds none. In
|
||||
// each test, console.error and console.info print nothing and keep what
|
||||
// they are given.
|
||||
const doNothing = () => {};
|
||||
let elements;
|
||||
beforeEach((t) => {
|
||||
@@ -48,11 +45,8 @@ const canvasContext = {
|
||||
arc: doNothing,
|
||||
};
|
||||
globalThis.window = {};
|
||||
globalThis.requestAnimationFrame = doNothing;
|
||||
globalThis.cancelAnimationFrame = doNothing;
|
||||
globalThis.document = {
|
||||
getElementById: () => null,
|
||||
createElement: () => ({}),
|
||||
querySelector: (selector) =>
|
||||
(elements[selector] ??= {
|
||||
getContext: () => canvasContext,
|
||||
@@ -384,131 +378,6 @@ test("a check that fails in the first round, which is discarded, writes nothing
|
||||
assert.deepEqual(consoleLines("info"), []);
|
||||
});
|
||||
|
||||
// The reports Reporter sent in this test, each as the hosts it sent.
|
||||
function reportedHosts() {
|
||||
return fetch.mock.calls.map(
|
||||
(call) => JSON.parse(call.arguments[1].body).hosts,
|
||||
);
|
||||
}
|
||||
|
||||
// In the next two tests every report is delivered, and only Target has
|
||||
// checks to report. target.reset() does to Target what the interval menu
|
||||
// does to each target.
|
||||
|
||||
// The interval changes after the first report. The second report is sent
|
||||
// before Target's first check since then, and the third after it. A fourth
|
||||
// would hold no check the first three did not, so it is not sent.
|
||||
test("checks recorded before an interval change and not yet reported go in the next report, with the status of the last check, and no check is reported twice", async (t) => {
|
||||
const state = new AppState([
|
||||
{ name: "Target", url: "https://target.test" },
|
||||
]);
|
||||
const target = state.local[0];
|
||||
const reporter = new Reporter(state, "client", CONFIG.reportInterval);
|
||||
t.mock.method(globalThis, "fetch", async () => ({ ok: true }));
|
||||
|
||||
target.pushSample(1000, { latency: 10, error: null });
|
||||
await reporter.flush();
|
||||
target.pushSample(2000, { latency: 20, error: null });
|
||||
target.pushSample(3000, { latency: null, error: "timeout" });
|
||||
target.reset();
|
||||
await reporter.flush();
|
||||
target.pushSample(4000, { latency: 40, error: null });
|
||||
await reporter.flush();
|
||||
await reporter.flush();
|
||||
|
||||
const name = "Target";
|
||||
const url = "https://target.test";
|
||||
assert.deepEqual(reportedHosts(), [
|
||||
[
|
||||
{
|
||||
name,
|
||||
url,
|
||||
status: "online",
|
||||
history: [{ t: 1000, latency: 10, error: null }],
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
name,
|
||||
url,
|
||||
status: "error",
|
||||
history: [
|
||||
{ t: 2000, latency: 20, error: null },
|
||||
{ t: 3000, latency: null, error: "timeout" },
|
||||
],
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
name,
|
||||
url,
|
||||
status: "online",
|
||||
history: [{ t: 4000, latency: 40, error: null }],
|
||||
},
|
||||
],
|
||||
]);
|
||||
// The page's history starts over.
|
||||
assert.deepEqual(target.history, [
|
||||
{ timestamp: 4000, latency: 40, error: null },
|
||||
]);
|
||||
});
|
||||
|
||||
// The interval changes twice, after 60 checks each time, before the first
|
||||
// report. Of the 120 checks the interval menu cleared, the report has the
|
||||
// last 100 (CONFIG.maxHistoryPoints).
|
||||
test("of the checks interval changes cleared before they were reported, the next report has the last 100", async (t) => {
|
||||
const state = new AppState([
|
||||
{ name: "Target", url: "https://target.test" },
|
||||
]);
|
||||
const target = state.local[0];
|
||||
const reporter = new Reporter(state, "client", CONFIG.reportInterval);
|
||||
t.mock.method(globalThis, "fetch", async () => ({ ok: true }));
|
||||
|
||||
for (let n = 1; n <= 120; n++) {
|
||||
target.pushSample(n * 1000, { latency: 10, error: null });
|
||||
if (n % 60 === 0) target.reset();
|
||||
}
|
||||
await reporter.flush();
|
||||
|
||||
const sent = reportedHosts()[0][0].history;
|
||||
assert.equal(sent.length, 100);
|
||||
assert.equal(sent[0].t, 21000);
|
||||
assert.equal(sent.at(-1).t, 120000);
|
||||
});
|
||||
|
||||
// panel stands in for the open debug log panel, and for the element in it
|
||||
// that holds its lines, whichever the page looks up by id; the lines are
|
||||
// kept in order in lines. Drawing the whole panel, as opening it does,
|
||||
// replaces every line with a new one.
|
||||
test("with the debug log open, writing more lines than it keeps adds each to the panel and leaves the lines already shown as they are", (t) => {
|
||||
const lines = [];
|
||||
const panel = {
|
||||
classList: { contains: () => false },
|
||||
children: lines,
|
||||
get firstElementChild() {
|
||||
return lines[0];
|
||||
},
|
||||
append: (line) => lines.push(line),
|
||||
removeChild: (line) => lines.splice(lines.indexOf(line), 1),
|
||||
replaceChildren: (...newLines) =>
|
||||
lines.splice(0, lines.length, ...newLines),
|
||||
};
|
||||
t.mock.method(globalThis.document, "getElementById", () => panel);
|
||||
const kept = CONFIG.maxLogEntries;
|
||||
|
||||
for (let i = 1; i <= kept; i++) log.info(`line ${i}`);
|
||||
const shown = [...lines];
|
||||
for (let i = kept + 1; i <= kept + 5; i++) log.info(`line ${i}`);
|
||||
|
||||
assert.deepEqual(
|
||||
lines.map((line) => line.textContent.match(/line \d+$/)?.[0]),
|
||||
Array.from({ length: kept }, (_, i) => `line ${i + 6}`),
|
||||
);
|
||||
// The first five lines shown were dropped, and the rest are the same
|
||||
// lines, not drawn again.
|
||||
shown.slice(5).forEach((line, i) => assert.equal(lines[i], line));
|
||||
});
|
||||
|
||||
// The page shows < > " & and ' in a row's markup as
|
||||
// < > " & and '.
|
||||
test(`a target whose name and URL hold < > " & and ' shows those characters in its row`, () => {
|
||||
|
||||
+28
-79
@@ -144,15 +144,13 @@ const WAN_HOSTS = [
|
||||
|
||||
const debugLog = [];
|
||||
|
||||
export const log = (() => {
|
||||
const log = (() => {
|
||||
function append(level, message) {
|
||||
const entry = { timestamp: new Date(), level, message };
|
||||
debugLog.push(entry);
|
||||
debugLog.push({ timestamp: new Date(), level, message });
|
||||
if (debugLog.length > CONFIG.maxLogEntries)
|
||||
debugLog.splice(0, debugLog.length - CONFIG.maxLogEntries);
|
||||
const panel = document.getElementById("debug-panel");
|
||||
if (panel && !panel.classList.contains("hidden"))
|
||||
appendDebugLogLine(entry);
|
||||
if (panel && !panel.classList.contains("hidden")) renderDebugLog();
|
||||
}
|
||||
return {
|
||||
error: (msg) => append("error", msg),
|
||||
@@ -259,14 +257,6 @@ function latencyStats(latencies) {
|
||||
};
|
||||
}
|
||||
|
||||
// The status a check's error gives its host: "error" for a timeout,
|
||||
// "offline" for any other error, and "online" for none.
|
||||
function statusFromError(error) {
|
||||
if (error === "timeout") return "error";
|
||||
if (error) return "offline";
|
||||
return "online";
|
||||
}
|
||||
|
||||
export class HostState {
|
||||
constructor(host, pinned = false) {
|
||||
this.name = host.name;
|
||||
@@ -275,10 +265,6 @@ export class HostState {
|
||||
// error }, or a round skipped while paused, { timestamp,
|
||||
// latency: null, paused: true }.
|
||||
this.history = [];
|
||||
// The last 100 (CONFIG.maxHistoryPoints) entries the interval menu
|
||||
// has cleared from the history, which the reporter may not have
|
||||
// sent yet.
|
||||
this.clearedHistory = [];
|
||||
this.lastLatency = null;
|
||||
this.status = "pending"; // 'online' | 'offline' | 'error' | 'pending'
|
||||
this.pinned = pinned;
|
||||
@@ -294,7 +280,9 @@ export class HostState {
|
||||
});
|
||||
this._trim();
|
||||
this.lastLatency = result.latency;
|
||||
this.status = statusFromError(result.error);
|
||||
if (result.error === "timeout") this.status = "error";
|
||||
else if (result.error) this.status = "offline";
|
||||
else this.status = "online";
|
||||
this.consecutiveFailures = result.error
|
||||
? this.consecutiveFailures + 1
|
||||
: 0;
|
||||
@@ -305,18 +293,6 @@ export class HostState {
|
||||
this._trim();
|
||||
}
|
||||
|
||||
// What the interval menu does to each host: the history starts over and
|
||||
// the host waits for its next check. The history's entries move to
|
||||
// clearedHistory, so the reporter still sends those it has not.
|
||||
reset() {
|
||||
this.clearedHistory = [...this.clearedHistory, ...this.history].slice(
|
||||
-CONFIG.maxHistoryPoints,
|
||||
);
|
||||
this.history = [];
|
||||
this.lastLatency = null;
|
||||
this.status = "pending";
|
||||
}
|
||||
|
||||
// The min, max, median and average latency of the checks in the history
|
||||
// that got an answer.
|
||||
historyStats() {
|
||||
@@ -453,12 +429,9 @@ function getClientId() {
|
||||
|
||||
// Build the delta report body the backend decodes, plus the new per-host
|
||||
// high-water marks. Pure function of the passed state: `hosts` is an array
|
||||
// of { name, url, status, clearedHistory, history }, `since` maps a host url
|
||||
// to the Unix-ms timestamp of the last sample already reported for it, and
|
||||
// `now` is a Date. Only non-paused samples newer than the mark are included,
|
||||
// from clearedHistory as well as history. A host whose status is pending,
|
||||
// as the interval menu leaves it until its next check, is sent with the
|
||||
// status of its last check, which is the last sample sent. Returns null
|
||||
// of { name, url, status, history }, `since` maps a host url to the Unix-ms
|
||||
// timestamp of the last sample already reported for it, and `now` is a Date.
|
||||
// Only non-paused samples newer than the mark are included. Returns null
|
||||
// when no host has an unreported sample.
|
||||
export function buildReport(hosts, clientId, now, since) {
|
||||
const reportHosts = [];
|
||||
@@ -467,7 +440,7 @@ export function buildReport(hosts, clientId, now, since) {
|
||||
const mark = since.get(host.url) ?? 0;
|
||||
const samples = [];
|
||||
let high = mark;
|
||||
for (const p of [...host.clearedHistory, ...host.history]) {
|
||||
for (const p of host.history) {
|
||||
if (p.paused) continue;
|
||||
if (p.timestamp <= mark) continue;
|
||||
samples.push({
|
||||
@@ -481,10 +454,7 @@ export function buildReport(hosts, clientId, now, since) {
|
||||
reportHosts.push({
|
||||
name: host.name,
|
||||
url: host.url,
|
||||
status:
|
||||
host.status === "pending"
|
||||
? statusFromError(samples.at(-1).error)
|
||||
: host.status,
|
||||
status: host.status,
|
||||
history: samples,
|
||||
});
|
||||
marks.set(host.url, high);
|
||||
@@ -511,7 +481,7 @@ export function buildReport(hosts, clientId, now, since) {
|
||||
// Only one report is ever in flight, and it is abandoned after half the
|
||||
// interval, so a slow POST can neither overlap the next report (which would
|
||||
// carry the same samples) nor stall reporting for good.
|
||||
export class Reporter {
|
||||
class Reporter {
|
||||
constructor(state, clientId, intervalMs) {
|
||||
this.state = state;
|
||||
this.clientId = clientId;
|
||||
@@ -1109,8 +1079,9 @@ function updateHealthBox(state) {
|
||||
|
||||
// --- Debug Log Renderer ------------------------------------------------------
|
||||
|
||||
// One line of the debug log panel, its message set as text.
|
||||
function debugLogLine(entry) {
|
||||
function renderDebugLog() {
|
||||
const el = document.getElementById("debug-log");
|
||||
if (!el) return;
|
||||
const levelColors = {
|
||||
error: "text-red-400",
|
||||
warning: "text-orange-400",
|
||||
@@ -1118,43 +1089,19 @@ function debugLogLine(entry) {
|
||||
info: "text-gray-300",
|
||||
debug: "text-gray-500",
|
||||
};
|
||||
const ts = formatUTCTimestamp(entry.timestamp);
|
||||
const lvl = entry.level.toUpperCase().padEnd(7);
|
||||
const line = document.createElement("div");
|
||||
line.className = levelColors[entry.level] || "text-gray-400";
|
||||
line.textContent = `${ts} ${lvl} ${entry.message}`;
|
||||
return line;
|
||||
}
|
||||
|
||||
// Draws every line of the debug log into the panel, when it is opened.
|
||||
function renderDebugLog() {
|
||||
const el = document.getElementById("debug-log");
|
||||
if (!el) return;
|
||||
el.replaceChildren(...debugLog.map((entry) => debugLogLine(entry)));
|
||||
el.replaceChildren(
|
||||
...debugLog.map((entry) => {
|
||||
const ts = formatUTCTimestamp(entry.timestamp);
|
||||
const lvl = entry.level.toUpperCase().padEnd(7);
|
||||
const line = document.createElement("div");
|
||||
line.className = levelColors[entry.level] || "text-gray-400";
|
||||
line.textContent = `${ts} ${lvl} ${entry.message}`;
|
||||
return line;
|
||||
}),
|
||||
);
|
||||
el.scrollTop = el.scrollHeight;
|
||||
}
|
||||
|
||||
// The request that scrolls the open panel to its newest line in the next
|
||||
// frame. One waits at most, also in a hidden tab, where frames do not run.
|
||||
let debugLogScrollFrame = 0;
|
||||
|
||||
// Adds entry to the open panel as its last line, and drops the first line
|
||||
// once the panel holds more than maxLogEntries. A check that ends while the
|
||||
// page is busy is timed only when the page is done, so the lines already
|
||||
// shown are left as they are, and the panel is scrolled once a frame
|
||||
// rather than once a line: scrolling it lays out all of its lines again.
|
||||
function appendDebugLogLine(entry) {
|
||||
const el = document.getElementById("debug-log");
|
||||
if (!el) return;
|
||||
el.append(debugLogLine(entry));
|
||||
if (el.children.length > CONFIG.maxLogEntries)
|
||||
el.removeChild(el.firstElementChild);
|
||||
cancelAnimationFrame(debugLogScrollFrame);
|
||||
debugLogScrollFrame = requestAnimationFrame(() => {
|
||||
el.scrollTop = el.scrollHeight;
|
||||
});
|
||||
}
|
||||
|
||||
// --- Sorting -----------------------------------------------------------------
|
||||
|
||||
// Sort WAN hosts: pinned hosts first (alphabetically), then unpinned hosts
|
||||
@@ -1555,7 +1502,9 @@ async function init() {
|
||||
|
||||
// Clear all host history and reset state
|
||||
for (const host of state.allHosts) {
|
||||
host.reset();
|
||||
host.history = [];
|
||||
host.lastLatency = null;
|
||||
host.status = "pending";
|
||||
}
|
||||
state.tickCount = 0;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user