diff --git a/.dockerignore b/.dockerignore index 5d051da..4f7fa2b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -72,10 +72,10 @@ **/*.sublime-* # This repository's own entries, after the shared content above: the -# frontend build, the viewport test's output, what `make build` and -# `make run` in backend/ leave behind, and log files at the root. +# frontend build, the viewport test's output, the binary `make build` +# writes and the reports `make run` stores, and log files at the root. /dist /tmp -/backend/netwatch-server -/backend/data +/netwatch-server +/data /*.log diff --git a/.gitignore b/.gitignore index ce123a5..eb652e1 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,10 @@ node_modules/ # Build output dist/ tmp/ -/backend/netwatch-server +/netwatch-server + +# The reports make run stores, in netwatch-server's default DATA_DIR +/data/ # Go test binaries and coverage output *.test diff --git a/backend/.golangci.yml b/.golangci.yml similarity index 100% rename from backend/.golangci.yml rename to .golangci.yml diff --git a/Dockerfile b/Dockerfile index 209691b..972b057 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,15 +21,15 @@ RUN yarn install --frozen-lockfile COPY . . RUN yarn eslint . -# Lint phase: golangci-lint over the backend with backend/.golangci.yml, +# Lint phase: golangci-lint over the Go packages with .golangci.yml, # and eslint through the copy from frontend-lint at the end. The # golangci/golangci-lint image ships Go and the linter. # golangci/golangci-lint:v2.14.0, 2026-09-24 FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint WORKDIR /src -COPY backend/go.mod backend/go.sum ./ +COPY go.mod go.sum ./ RUN go mod download -COPY backend/ . +COPY . . RUN golangci-lint run --config .golangci.yml ./... # Nothing is wanted from frontend-lint; the copy is what makes this # phase run it. @@ -69,7 +69,7 @@ RUN yarn build # golang:1.25.7-trixie, 2026-10-07 FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test WORKDIR /src -COPY backend/ . +COPY . . RUN --mount=type=tmpfs,target=/go/pkg/mod \ --mount=type=tmpfs,target=/root/.cache/go-build \ go test -timeout 90s -race -cover ./... || \ @@ -90,13 +90,12 @@ RUN apk add --no-cache git # A tar-stream context keeps the sender's file owners, which git refuses. RUN git config --system --add safe.directory /src WORKDIR /src -COPY backend/go.mod backend/go.sum backend/ -RUN cd backend && go mod download +COPY go.mod go.sum ./ +RUN go mod download COPY . . -# backend/script/build is the one definition of the build command: -# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=..." -# It reads VERSION from the environment. +# The same go build as in script/build, which builds the binary on the +# host. # # The version is the VERSION build argument when one is given, otherwise # `git describe --tags --always` on the .git in the build context: the @@ -112,7 +111,9 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \ exit 1 ;; \ esac; \ fi; \ - VERSION="$version" backend/script/build + CGO_ENABLED=0 go build -trimpath \ + -ldflags "-s -w -X main.Version=$version" \ + -o netwatch-server ./cmd/netwatch-server/ # Runtime stage, and the last one: a plain `docker build .` builds it # and the stages it copies from, the two phases included. @@ -128,10 +129,10 @@ RUN addgroup -g 1000 -S netwatch && \ # At start-up the nginx image renders every template here into # conf.d; bin/entrypoint.sh says how. RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/templates/netwatch.conf.template -COPY security-headers.conf /etc/nginx/security-headers.conf +COPY configs/nginx.conf /etc/nginx/templates/netwatch.conf.template +COPY configs/security-headers.conf /etc/nginx/security-headers.conf COPY --from=frontend /app/dist /usr/share/nginx/html -COPY --from=builder /src/backend/netwatch-server /usr/local/bin/netwatch-server +COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh # bin/entrypoint.sh creates DATA_DIR at start and gives it and /data to diff --git a/Makefile b/Makefile index fefa8ae..322677a 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -.PHONY: bootstrap setup dev build test lint fmt fmt-check check \ +.PHONY: bootstrap setup dev build run clean test lint fmt fmt-check check \ add-dependency tidy frontend-viewport-test docker hooks # Standard targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern (see the Entrypoints section # of README.md). test, lint, fmt, fmt-check and check cover the whole -# repo: the frontend here and the Go backend in backend/. +# repo: the frontend in web/ and the Go backend in cmd/ and internal/. bootstrap: @script/bootstrap @@ -15,10 +15,17 @@ setup: dev: @script/dev -# The frontend only; backend/Makefile's build target builds the Go server. +# The frontend into dist/ and the netwatch-server binary. build: @script/build +# Builds, then runs netwatch-server on port 8080. +run: + @script/run + +clean: + @script/clean + test: @script/test diff --git a/README.md b/README.md index d9497b4..47a9495 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,13 @@ make setup # Run the page on the Vite dev server make dev +# Build netwatch-server and run it on port 8080 +make run + # Run the tests, both linters and the format check make check -# Build the page into dist/ +# Build the page into dist/ and netwatch-server into the repo root make build # Build the image and run it @@ -25,9 +28,8 @@ docker run -p 8080:8080 netwatch ``` `make check` and `make docker` need Docker. `make dev` passes `/api` to -`http://127.0.0.1:8080`, where `make run` in `backend/` starts `netwatch-server` -with its defaults, so the reports the page posts are stored in -`backend/data/reports`. +`http://127.0.0.1:8080`, where `make run` starts `netwatch-server` with its +defaults, so the reports the page posts are stored in `data/reports`. ## Entrypoints @@ -35,49 +37,51 @@ This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) standard: normalized scripts in `script/` are the entrypoints for the development workflow, and the Makefile targets are thin shims that call them. -The Go backend in `backend/` has its own `script/` directory and shim Makefile -(see [backend/README.md](backend/README.md)). The root scripts cover both -halves, so the root `make check` fails if either one is broken. We provide: +The scripts cover both the frontend and the Go backend, so `make check` fails if +either one is broken. We provide: - `script/bootstrap` — install all dependencies (the pinned node via nvm unless one new enough for the frontend's dependencies is installed, yarn via corepack, `yarn install --frozen-lockfile`, Go `GO_VERSION` unless the `go` on - `PATH` is exactly that version, the Go modules, and gcc with the C library - headers unless gcc is installed, for the race detector in `make test` in - `backend/`), linking what it installs itself into `~/.local/bin`, which has to - be on `PATH`. It installs no Go linter and not Docker: `make test` and - `make lint` run in Docker + `PATH` is exactly that version, and the Go modules), linking what it installs + itself into `~/.local/bin`, which has to be on `PATH`. It installs no Go + linter and not Docker: `make test` and `make lint` run in Docker - `script/setup` — make a fresh clone ready for development: bootstrap plus the git pre-commit hook - `script/dev` — run the Vite dev server, which proxies `/api` to a locally running `netwatch-server` -- `script/build` — build the frontend for production into `dist/`; - `backend/script/build` builds the Go server +- `script/build` — build the frontend for production into `dist/`, then the + static `netwatch-server` binary into the repo root with its version stamped + in: `VERSION` from the environment, or when that is unset or empty, + `git describe` inside a git checkout, then `dev` +- `script/run` — build with `script/build`, then run `netwatch-server` with its + defaults (see [Settings](#settings)) +- `script/clean` — remove what `script/build` writes - `script/projectname` — print the project name (used for the Docker image tag) - `script/test` — build the `test` phase of `Dockerfile` without the cache: the frontend's unit tests and production build in its `frontend` stage, and the backend's Go tests with the race detector and coverage - `script/lint` — build the `lint` phase of `Dockerfile` without the cache: - eslint in its `frontend-lint` stage, and golangci-lint over `backend/` + eslint in its `frontend-lint` stage, and golangci-lint over the Go packages - `script/fmt` — format all files (writes): prettier over the JavaScript, CSS, - HTML and Markdown, then gofmt over `backend/`. It runs on the host, as - `script/fmt-check` does, with `~/.local/bin` put on `PATH` + HTML and Markdown, then gofmt over `cmd/` and `internal/`. It runs on the + host, as `script/fmt-check` does, with `~/.local/bin` put on `PATH` - `script/fmt-check` — check formatting (read-only): prettier, then gofmt - `script/check` — run test, lint, and fmt-check - `script/add-dependency` — add a frontend package, or move one to another version: `make add-dependency PACKAGE=@` runs `yarn add --dev`, which changes `package.json` and `yarn.lock` together, then `yarn install --frozen-lockfile` -- `script/tidy` — run `go mod tidy` in `backend/`: to add a Go module, import it - and run `make tidy`; to move one to another version, edit its `require` line - in `backend/go.mod`, then run `make tidy` +- `script/tidy` — run `go mod tidy`: to add a Go module, import it and run + `make tidy`; to move one to another version, edit its `require` line in + `go.mod`, then run `make tidy` - `script/frontend-test` — run the unit tests in `test/unit/` on the host with Node's built-in test runner, through the `test` script in `package.json`, and if any fails, run them again listing every test, and fail; then the production build. Each test run has a 90-second timeout. `make test` runs the same in Docker - `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with - prettier (writes), the markdown in `backend/` included + prettier (writes) - `script/frontend-fmt-check` — check prettier formatting (read-only) - `script/frontend-viewport-test` — responsive-layout verification of the built frontend in a containerised headless Chrome (see @@ -93,11 +97,11 @@ halves, so the root `make check` fails if either one is broken. We provide: ## Responsive layout The narrow-viewport layout lives in the `max-width: 768px` media block in -`src/styles.css`. It is verified automatically by `make frontend-viewport-test`, -which drives a digest-pinned headless Chrome against the built `dist/` and -asserts on computed layout at widths derived from that CSS — on every breakpoint -it declares and one pixel either side of it, plus a 320px floor, a desktop -baseline and two landscape sizes. See +`web/src/styles.css`. It is verified automatically by +`make frontend-viewport-test`, which drives a digest-pinned headless Chrome +against the built `dist/` and asserts on computed layout at widths derived from +that CSS — on every breakpoint it declares and one pixel either side of it, plus +a 320px floor, a desktop baseline and two landscape sizes. See [test/viewport/README.md](test/viewport/README.md) for what it covers and what it genuinely cannot. @@ -112,8 +116,11 @@ the same and nothing is stored. ## Design +The page's source is in `web/`, the Go backend's in `cmd/` and `internal/`, and +nginx's configuration in `configs/`. + The page is built with Vite and Tailwind CSS v4. Its code is all in -`src/main.js`, with a class-based architecture: +`web/src/main.js`, with a class-based architecture: - **`CONFIG`**: Configuration object (update interval, timeouts, axis ticks, etc.). The interval menu sets `updateInterval`, the one value the page writes @@ -148,10 +155,10 @@ probing. The report-building step is a pure function of host state. ### Backend -`netwatch-server`, in `backend/`, is a small Go HTTP server that stores the -reports the page posts. It keeps them in memory and writes them to `DATA_DIR` as -zstd-compressed files of JSON lines: every minute, whenever 10 MiB are waiting, -and when it stops. Its routes: +`netwatch-server` is a small Go HTTP server that stores the reports the page +posts. It keeps them in memory and writes them to `DATA_DIR` as zstd-compressed +files of JSON lines: every minute, whenever 10 MiB are waiting, and when it +stops. Its routes: - `POST /api/v1/reports` — takes a report, without credentials; each client address may send a limited number a minute, and the report files are capped in @@ -162,11 +169,133 @@ and when it stops. Its routes: `METRICS_USERNAME` and `METRICS_PASSWORD` are set; each client address may make a limited number of requests to it a minute +It is an `fx`-wired Go application: `cmd/netwatch-server/` starts it, and its +packages are in `internal/`: + +- **`config`**: Loads configuration from environment variables and config files + via Viper. +- **`handlers`**: HTTP request handlers for the API (health check, report + ingestion). +- **`reportbuf`**: The in-memory buffer of reports and the files it writes them + to. +- **`server`**: Chi-based HTTP server with middleware wiring and route + registration. +- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting + infrastructure. + In the image, the `test` phase of `Dockerfile` tests it, the `builder` stage -builds it with `backend/script/build`, and `bin/entrypoint.sh` runs it as user -`netwatch` on `127.0.0.1:8081`, behind nginx. Outside the image, `make run` in -`backend/` builds it and runs it on port 8080. Its settings, report storage and -limits are in [backend/README.md](backend/README.md). +builds it, and `bin/entrypoint.sh` runs it as user `netwatch` on +`127.0.0.1:8081`, behind nginx (see [Deployment](#deployment)). Outside the +image, `make run` builds it and runs it on port 8080. + +#### Settings + +| Variable | Default | Description | +| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- | +| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface | +| `PORT` | `8080` | HTTP listen port | +| `DATA_DIR` | `./data/reports` | Directory for compressed reports | +| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) | +| `DEBUG` | `false` | Enable debug logging | +| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution | +| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) | +| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) | +| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) | +| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) | +| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) | + +`TRUSTED_PROXIES` defaults to +`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback +entries cover a reverse proxy on the same host. A request whose direct peer is +outside this set has its forwarded headers ignored, and the direct peer is +logged and rate-limited instead. The container image does not use this default; +see [Deployment](#deployment). + +A variable set to a value the server cannot use, such as `PORT=abc`, +`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from +starting, with an error naming the variable. An empty variable counts as unset. + +#### Report storage + +Reports are written as `reports--.jsonl.zst` files in +`DATA_DIR`. The timestamp is in UTC to the millisecond, so the names sort by +time. The number starts at 1 when the server starts and goes up by one for each +file the server starts to write, so two files written in the same millisecond +still get different names. A failed write uses up its number and leaves a gap in +the numbers: its file, if it was created, is removed. The file stays, counted +toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too. +Files are created with `O_EXCL` to prevent overwrites. + +#### Report limits + +`POST /api/v1/reports` takes reports from anyone who can reach it, without +credentials, so it is bounded instead. Both refusals below answer with the same +`{"status":"error"}` body as any other error. + +- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may + send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with + `Retry-After: 60`. The minute slides: reports from the minute before still + count, fading out over the current one, so an address is sure never to be + refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60 + seconds. The page sends one report a minute from each open tab, so the default + of 60 refuses nothing from up to 30 tabs behind one address, such as a + household or an office sharing it, however their reports bunch up. Report + responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and + `X-RateLimit-Reset` headers. +- **Size cap.** The report files in `DATA_DIR` may total at most + `DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports + waiting in memory count at their uncompressed size until they are written; + those lost to a failed write stop counting, and the part of its file written + is removed. When a report would take the total past the cap, the oldest report + files are deleted to make room, and each deletion is logged with the file's + name and size; a file still being written is never deleted. A report is + refused with 507, and nothing of it is stored, only when the reports waiting + to be written fill the cap on their own, and then no file is deleted. At + start, report files past the cap, as after lowering it, are deleted the same + way. So the cap is how much of the newest reports is kept: the default of 1 + GiB is small enough for any host; set it to the space you can give `DATA_DIR`. + +#### CORS + +The page calls the API from the origin it is served from, so by default the +server sends no CORS headers, and browsers let no other origin's pages call it. +To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS` +(for example `https://netwatch.example.com`); pages from a listed origin may +`GET` and `POST` with a `Content-Type` header. Each entry must be a plain +origin, `scheme://host` with an optional `:port`, as browsers send it: no path, +not even a trailing `/`, and no `*`. Any other entry stops the server from +starting, with an error naming `CORS_ALLOWED_ORIGINS`. + +#### Metrics + +With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves +Prometheus metrics at `GET /metrics` to requests with those as their basic auth +credentials, and answers any other with 401. For each request that reaches the +health check or `POST /api/v1/reports`, those the rate limit refuses included, +the metrics record its duration and response size, labelled with its path, +method and status; they also count those requests in progress, and include Go's +runtime and process metrics. No other request is recorded: not those to +`/metrics` itself, and not those answered before they reach either route, such +as a CORS preflight, or a request refused with 404 for a path no route has, 405 +for a method its route does not take, or 413 for declaring a body length over +the 1 MiB limit. A report whose body goes over the limit without declaring its +length reaches the route, is answered 413 there, and is recorded with that +status. Clients can make up any number of paths and methods, and each would add +labels to the metrics for as long as the server runs. + +`/metrics` is rate limited, so that its password cannot be guessed quickly: each +client address, resolved through `TRUSTED_PROXIES`, may make 60 requests to it a +minute, whatever their credentials. Past that it gets 429 with +`Retry-After: 60`, and its credentials are not checked. The minute slides as it +does for reports (see [Report limits](#report-limits)), so a scraper polling +every 2 seconds or less often is never refused. This allowance is apart from the +one for reports. + +#### Sentry + +With `SENTRY_DSN` set, each panic in a handler is reported to that Sentry +project under the release `netwatch-server-` followed by the server's version. +On shutdown the server waits up to 2 seconds for Sentry to finish sending. ### Monitoring targets @@ -268,20 +397,25 @@ behind a reverse proxy. The Docker image, built from `Dockerfile` by `make docker`, is the whole service in one container: nginx serves the built frontend and passes `/api/`, -`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`, -which listens only inside the container, on `127.0.0.1:8081`. The image: +`/.well-known/healthcheck` and `/metrics` to the Go backend, `netwatch-server`. +`bin/entrypoint.sh` starts it listening only inside the container, on +`127.0.0.1:8081`, with `TRUSTED_PROXIES=127.0.0.1/32`, so it takes the client +address nginx passes on and no other. The image: - Listens on port 8080 by default (override with `PORT` env var) - Takes the client address from `X-Forwarded-For` only on requests from the - reverse proxies named in `TRUSTED_PROXIES`, and by default from none + reverse proxies named in `TRUSTED_PROXIES`, and by default from none. + `bin/entrypoint.sh` checks each entry with `netwatch-server check-cidr`, which + parses it as the backend parses its own `TRUSTED_PROXIES` - Sends access logs to stdout - Caches static assets with immutable headers - Sends the security headers `REPO_POLICIES.md` requires on every response, as - `security-headers.conf` sets them, in place of the backend's own + `configs/security-headers.conf` sets them, in place of the backend's own - Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data` volume. Before the backend starts, the image creates `DATA_DIR` and gives it - and `/data` to user `netwatch` (uid 1000), which the backend runs as, so a - host directory bind-mounted at `/data` ends up owned by uid 1000 + and `/data` to user `netwatch` (uid 1000), which the backend runs as, with + `netwatch-server prepare-data-dir`, which acts on nothing outside `/data`, so + a host directory bind-mounted at `/data` ends up owned by uid 1000 - Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx or the backend exits on its own, so the platform restarts it diff --git a/TODO.md b/TODO.md index 53f31f5..7a7e2df 100644 --- a/TODO.md +++ b/TODO.md @@ -16,12 +16,25 @@ Docker image, and the Gitea workflow `.gitea/workflows/check.yml` runs # Next Step -Decide whether the repo moves to the layout `REPO_POLICIES.md` gives, with -`backend/` no longer repeating files from the root -([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). +Run `make frontend-viewport-test` in CI as its own step; it is not part of +`make check`, as it needs Docker and takes minutes. # Completed Steps +- 2026-10-14: the repo has the layout `REPO_POLICIES.md` gives + ([#30](https://git.eeqj.de/sneak/netwatch/issues/30)). The Go module is at the + root (`go.mod`, `go.sum`, `.golangci.yml`, `cmd/netwatch-server/`, + `internal/`), so its path `sneak.berlin/go/netwatch` names the repository. The + page's source is in `web/` (`index.html`, `src/`), which `vite.config.js` + makes Vite's root; the build still goes to `dist/`. `nginx.conf` and + `security-headers.conf` are in `configs/`. `backend/` is gone: its `LICENSE` + and `.dockerignore` are deleted, its `.gitignore` entries are in the root one, + its `README.md` is folded into the root `README.md`, and its scripts into the + root `script/`: `script/build` builds the server binary after the frontend, + `script/run` and `script/clean` came from `backend/script/`, and `make test`, + `make lint`, `make fmt` and `make fmt-check` do what its `test`, `lint`, `fmt` + and `fmt-check` did. `script/bootstrap` no longer installs gcc, which only the + host Go tests needed. The image and the page are unchanged - 2026-10-07: the files shared from `sneak/prompts` are its copies at commit `dd4027b` ([#113](https://git.eeqj.de/sneak/netwatch/issues/113)), with this repository's own entries after the shared content in `.gitignore`, @@ -416,8 +429,6 @@ Decide whether the repo moves to the layout `REPO_POLICIES.md` gives, with # Future Steps -- Run `make frontend-viewport-test` in CI as its own step; it is not part of - `make check`, as it needs Docker and takes minutes - A backend test that posts a report to `POST /api/v1/reports` and checks the compressed file it is written to - A backend route that decompresses the stored reports and answers queries on diff --git a/backend/.dockerignore b/backend/.dockerignore deleted file mode 100644 index fd4f2b0..0000000 --- a/backend/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -.DS_Store diff --git a/backend/.gitignore b/backend/.gitignore deleted file mode 100644 index e48380c..0000000 --- a/backend/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/netwatch-server -*.log -*.out -*.test -.env -data/ diff --git a/backend/LICENSE b/backend/LICENSE deleted file mode 100644 index 5b83a52..0000000 --- a/backend/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 @sneak (https://sneak.berlin) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/backend/Makefile b/backend/Makefile deleted file mode 100644 index faf9ec5..0000000 --- a/backend/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -# Thin shims; the implementations live in backend/script/ (see the -# Entrypoints section of README.md). There is no check, hooks or docker -# target here: the root Makefile's check covers this directory, its -# hooks target installs the repo's only pre-commit hook, and its docker -# target builds the one image, which contains this backend. - -.PHONY: all build test lint fmt fmt-check run clean - -all: build - -build: - @script/build - -test: - @script/test - -lint: - @script/lint - -fmt: - @script/fmt - -fmt-check: - @script/fmt-check - -run: - @script/run - -clean: - @script/clean diff --git a/backend/README.md b/backend/README.md deleted file mode 100644 index c6844b8..0000000 --- a/backend/README.md +++ /dev/null @@ -1,232 +0,0 @@ -netwatch-server is an MIT-licensed Go HTTP backend by -[@sneak](https://sneak.berlin) that receives telemetry reports from the NetWatch -SPA and persists them as zstd-compressed JSONL files on disk. - -## Getting Started - -From this directory: - -```bash -# Build and run locally -make run -``` - -From the repo root, whose `Dockerfile` builds the one image that ships this -backend behind nginx (see [Container image](#container-image)): - -```bash -# Run tests, lint, and format check over the frontend and this backend -make check - -# Build the image: nginx, the frontend and this backend -make docker -docker run -p 8080:8080 netwatch -``` - -## Entrypoints - -This directory follows the same -[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) -pattern as the repo root: the targets in `backend/Makefile` are thin shims over -`backend/script/`. The root `Dockerfile` runs `build`, and the root scripts call -`fmt` and `fmt-check`: - -- `script/build` — compile the static `netwatch-server` binary with its version - stamped in. The version is `VERSION` from the environment; when that is unset - or empty, it falls back to `git describe` inside a git checkout, then to `dev` -- `script/test` — run the Go tests on the host with the race detector and - coverage; the root `make test` runs them in the `test` phase of the root - `Dockerfile`. Go's `-timeout 90s` bounds the tests, not their compile, and - `-count=1` keeps Go from reporting a stored pass. If they fail, they run again - with `-v` for the details, and the script fails. The race detector needs a C - compiler -- `script/lint` — run the root `script/lint`, which builds the `lint` phase of - the root `Dockerfile`: golangci-lint over this directory, and eslint over the - frontend. golangci-lint never runs on the host -- `script/fmt` — format the Go sources (writes) -- `script/fmt-check` — check Go formatting (read-only) -- `script/run` — build and run the server locally -- `script/clean` — remove build artifacts - -There is no `check`, `hooks` or `docker` target here: the root `make check` -covers this directory, the root `make hooks` installs the repo's only pre-commit -hook, and the root `make docker` builds the image that contains this backend. - -## Rationale - -The NetWatch frontend collects latency measurements from the browser but has no -way to persist or aggregate them. This backend provides a minimal -`POST /api/v1/reports` endpoint that buffers incoming reports in memory and -flushes them to compressed files on disk for later analysis. - -## Design - -The server is structured as an `fx`-wired Go application under -`cmd/netwatch-server/`. Internal packages in `internal/` follow standard Go -project layout: - -- **`config`**: Loads configuration from environment variables and config files - via Viper. -- **`handlers`**: HTTP request handlers for the API (health check, report - ingestion). -- **`reportbuf`**: In-memory buffer that accumulates JSONL report lines and - flushes to zstd-compressed files when the buffer reaches 10 MiB or every 60 - seconds. -- **`server`**: Chi-based HTTP server with middleware wiring and route - registration. -- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting - infrastructure. - -### Configuration - -| Variable | Default | Description | -| ---------------------- | -------------------- | -------------------------------------------------------------------------------------------------------- | -| `BIND_ADDRESS` | empty | IP address to listen on; empty listens on every interface | -| `PORT` | `8080` | HTTP listen port | -| `DATA_DIR` | `./data/reports` | Directory for compressed reports | -| `DATA_DIR_MAX_BYTES` | `1073741824` (1 GiB) | Most bytes of report files kept in `DATA_DIR`, oldest deleted first; see [Report limits](#report-limits) | -| `DEBUG` | `false` | Enable debug logging | -| `TRUSTED_PROXIES` | loopback + RFC1918 | Comma-separated CIDRs whose `X-Forwarded-For` / `X-Real-IP` headers are trusted for client IP resolution | -| `REPORTS_PER_MINUTE` | `60` | Reports each client address may send a minute; see [Report limits](#report-limits) | -| `CORS_ALLOWED_ORIGINS` | empty | Comma-separated origins whose pages may call the API; see [CORS](#cors) | -| `METRICS_USERNAME` | empty | Basic auth user name for `/metrics`; see [Metrics](#metrics) | -| `METRICS_PASSWORD` | empty | Basic auth password for `/metrics`; see [Metrics](#metrics) | -| `SENTRY_DSN` | empty | DSN of the Sentry project to send errors to; see [Sentry](#sentry) | - -`TRUSTED_PROXIES` defaults to -`127.0.0.1/32,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`. The loopback -entries cover a reverse proxy on the same host. A request whose direct peer is -outside this set has its forwarded headers ignored, and the direct peer is -logged and rate-limited instead. The container image does not use this default; -see [Container image](#container-image). - -A variable set to a value the server cannot use, such as `PORT=abc`, -`DEBUG=maybe` or a `BIND_ADDRESS` that is not an IP address, stops it from -starting, with an error naming the variable. An empty variable counts as unset. - -### Container image - -The root `Dockerfile` builds one image in which nginx listens on the public port -8080, serves the frontend, and proxies `/api/`, `/.well-known/healthcheck` and -`/metrics` to this server. The image's entrypoint, `bin/entrypoint.sh`, starts -the server as user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and -`PORT=8081`, so only nginx reaches it, and with `TRUSTED_PROXIES=127.0.0.1/32`, -so it takes the client address nginx passes on and no other. `DATA_DIR` is -`/data/reports`, on the `/data` volume; before starting the server, the -entrypoint creates it and gives it and `/data` to `netwatch` with -`netwatch-server prepare-data-dir`, which acts on nothing outside `/data`. nginx -replaces the security headers this server sets with those in the root -`security-headers.conf`, so those are what clients of the image see. - -The container's own `TRUSTED_PROXIES` goes to nginx instead: IP addresses or -CIDRs, separated by commas, of the reverse proxies in front of the container. -nginx takes the client address from `X-Forwarded-For` only on a request from one -of them. Unset or empty, nginx trusts no proxy, and the client address is the -one each request comes from, so every client behind a proxy shares one rate -limit. An entry that is not an IP address or CIDR, such as a hostname or -`1.2.3`, stops the container at start with an error naming `TRUSTED_PROXIES`: -the entrypoint checks each entry with `netwatch-server check-cidr`, which parses -it as this server parses its own `TRUSTED_PROXIES`. - -### Report storage - -Reports are written as `reports--.jsonl.zst` files in -`DATA_DIR`. The timestamp is in UTC to the millisecond, so the names sort by -time. The number starts at 1 when the server starts and goes up by one for each -file the server starts to write, so two files written in the same millisecond -still get different names. A failed write uses up its number and leaves a gap in -the numbers: its file, if it was created, is removed. The file stays, counted -toward `DATA_DIR_MAX_BYTES` from the next start, only if removing it fails too. -Each file contains one JSON object per line, compressed with zstd. Files are -created with `O_EXCL` to prevent overwrites. - -### Report limits - -`POST /api/v1/reports` takes reports from anyone who can reach it, without -credentials, so it is bounded instead. Both refusals below answer with the same -`{"status":"error"}` body as any other error. - -- **Rate limit.** Each client address, resolved through `TRUSTED_PROXIES`, may - send `REPORTS_PER_MINUTE` reports a minute; past that it gets 429 with - `Retry-After: 60`. The minute slides: reports from the minute before still - count, fading out over the current one, so an address is sure never to be - refused only while it sends at most half of `REPORTS_PER_MINUTE` in any 60 - seconds. The page sends one report a minute from each open tab, so the default - of 60 refuses nothing from up to 30 tabs behind one address, such as a - household or an office sharing it, however their reports bunch up. Report - responses also carry `X-RateLimit-Limit`, `X-RateLimit-Remaining` and - `X-RateLimit-Reset` headers. -- **Size cap.** The report files in `DATA_DIR` may total at most - `DATA_DIR_MAX_BYTES`, counting the files already there at start. Reports - waiting in memory count at their uncompressed size until they are written; - those lost to a failed write stop counting, and the part of its file written - is removed. When a report would take the total past the cap, the oldest report - files are deleted to make room, and each deletion is logged with the file's - name and size; a file still being written is never deleted. A report is - refused with 507, and nothing of it is stored, only when the reports waiting - to be written fill the cap on their own, and then no file is deleted. At - start, report files past the cap, as after lowering it, are deleted the same - way. So the cap is how much of the newest reports is kept: the default of 1 - GiB is small enough for any host; set it to the space you can give `DATA_DIR`. - -### CORS - -The page calls the API from the origin it is served from, so by default the -server sends no CORS headers, and browsers let no other origin's pages call it. -To serve the page from elsewhere, list that origin in `CORS_ALLOWED_ORIGINS` -(for example `https://netwatch.example.com`); pages from a listed origin may -`GET` and `POST` with a `Content-Type` header. Each entry must be a plain -origin, `scheme://host` with an optional `:port`, as browsers send it: no path, -not even a trailing `/`, and no `*`. Any other entry stops the server from -starting, with an error naming `CORS_ALLOWED_ORIGINS`. - -### Metrics - -With both `METRICS_USERNAME` and `METRICS_PASSWORD` set, the server serves -Prometheus metrics at `GET /metrics` to requests with those as their basic auth -credentials, and answers any other with 401. For each request that reaches the -health check or `POST /api/v1/reports`, those the rate limit refuses included, -the metrics record its duration and response size, labelled with its path, -method and status; they also count those requests in progress, and include Go's -runtime and process metrics. No other request is recorded: not those to -`/metrics` itself, and not those answered before they reach either route, such -as a CORS preflight, or a request refused with 404 for a path no route has, 405 -for a method its route does not take, or 413 for declaring a body length over -the 1 MiB limit. A report whose body goes over the limit without declaring its -length reaches the route, is answered 413 there, and is recorded with that -status. Clients can make up any number of paths and methods, and each would add -labels to the metrics for as long as the server runs. With neither set, nothing -is recorded and `/metrics` answers 404. One without the other stops the server -from starting, with an error naming both; so does a `METRICS_USERNAME` -containing `:`, which basic auth cannot carry, with an error naming it. - -`/metrics` is rate limited, so that its password cannot be guessed quickly: each -client address, resolved through `TRUSTED_PROXIES`, may make 60 requests to it a -minute, whatever their credentials. Past that it gets 429 with -`Retry-After: 60`, and its credentials are not checked. The minute slides as it -does for reports (see [Report limits](#report-limits)), so a scraper polling -every 2 seconds or less often is never refused. This allowance is apart from the -one for reports. - -### Sentry - -With `SENTRY_DSN` set, the server sends its errors to that Sentry project: each -panic in a handler is reported there, under the release `netwatch-server-` -followed by the server's version, and the request still gets 500 from the -server's panic recovery. On shutdown the server waits up to 2 seconds for Sentry -to finish sending. A DSN Sentry refuses stops the server from starting, with an -error naming `SENTRY_DSN`. With it empty, Sentry is not set up, and nothing is -sent to it. - -## TODO - -The to-do list, this backend's open work included, is [TODO.md](../TODO.md) at -the repo root. - -## License - -MIT. See [LICENSE](LICENSE). - -## Author - -[@sneak](https://sneak.berlin) diff --git a/backend/script/build b/backend/script/build deleted file mode 100755 index 4f30155..0000000 --- a/backend/script/build +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -# script/build: compile the static netwatch-server binary into the -# backend project root, with its version stamped in. -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - - # VERSION comes from the environment (the root Dockerfile passes its - # ARG VERSION in). Unset or empty, it is git describe, or "dev" where - # there is no git or no repository history. - version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}" - - CGO_ENABLED=0 go build -trimpath \ - -ldflags "-s -w -X main.Version=$version" \ - -o netwatch-server ./cmd/netwatch-server/ -} - -main "$@" diff --git a/backend/script/clean b/backend/script/clean deleted file mode 100755 index 1f4e638..0000000 --- a/backend/script/clean +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -# script/clean: remove build artifacts. -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - rm -f netwatch-server -} - -main "$@" diff --git a/backend/script/fmt b/backend/script/fmt deleted file mode 100755 index 9fa78bf..0000000 --- a/backend/script/fmt +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -# script/fmt: format the Go sources (writes). -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - go fmt ./... -} - -main "$@" diff --git a/backend/script/fmt-check b/backend/script/fmt-check deleted file mode 100755 index 57405d0..0000000 --- a/backend/script/fmt-check +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# script/fmt-check: check Go formatting (read-only). Same scope as -# script/fmt, but fails instead of writing. -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - unformatted="$(gofmt -l .)" - if [ -n "$unformatted" ]; then - echo "Files not formatted:" >&2 - echo "$unformatted" >&2 - exit 1 - fi -} - -main "$@" diff --git a/backend/script/lint b/backend/script/lint deleted file mode 100755 index 897ba78..0000000 --- a/backend/script/lint +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# script/lint: lint the whole repo as the root make lint does, by -# building the lint phase of the root Dockerfile. golangci-lint never -# runs on the host (REPO_POLICIES.md). -set -eu - -ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" - -main() { - exec "$ROOT/script/lint" -} - -main "$@" diff --git a/backend/script/test b/backend/script/test deleted file mode 100755 index 5a95186..0000000 --- a/backend/script/test +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# script/test: run the backend test suite on the host with the race -# detector and coverage. The root make test runs the same in the test -# phase of the root Dockerfile. Go's own -timeout bounds the tests and -# not their compile, so a cold build cache cannot fail it. -count=1 -# keeps Go's test result cache out of both runs, so neither can report -# a stored pass. The race detector needs cgo, and so a C compiler. If -# the tests fail, they run again with -v for the details, and the -# script fails even if that run passes. -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - go test -count=1 -timeout 90s -race -cover ./... || { - echo "--- Rerunning with -v for details ---" - go test -count=1 -timeout 90s -race -v ./... - exit 1 - } -} - -main "$@" diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index ee53bfe..ceea002 100755 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -35,8 +35,8 @@ fi # TRUSTED_PROXIES names the reverse proxies in front of the container, # as IP addresses or CIDRs separated by commas. nginx takes the client # address from X-Forwarded-For only on a request from one of them, so -# unset or empty, it trusts no one. nginx.conf includes the file written -# here, one set_real_ip_from line per entry. +# unset or empty, it trusts no one. configs/nginx.conf includes the file +# written here, one set_real_ip_from line per entry. # # nginx looks up an entry it cannot read as an address as a hostname, # and trusts what it finds (1.2.3 is found as 1.2.0.3). So each entry @@ -76,10 +76,10 @@ stop_requested="" trap 'stop_requested=yes' TERM INT # netwatch-server runs as the netwatch user and listens on loopback -# only, on a port other than the public one; nginx.conf proxies to this -# address. Its only client is nginx, so it takes the client address -# nginx passes on from 127.0.0.1 alone, whatever TRUSTED_PROXIES the -# container has. The netwatch user has no login shell, hence -s +# only, on a port other than the public one; configs/nginx.conf proxies +# to this address. Its only client is nginx, so it takes the client +# address nginx passes on from 127.0.0.1 alone, whatever TRUSTED_PROXIES +# the container has. The netwatch user has no login shell, hence -s # /bin/sh. busybox su replaces itself with the command instead of # staying on as its parent, so $! is the server's own PID. BIND_ADDRESS=127.0.0.1 PORT=8081 TRUSTED_PROXIES=127.0.0.1/32 \ @@ -88,8 +88,8 @@ backend=$! # nginx starts through the nginx image's own entrypoint, which applies # the image's start-up configuration and then replaces itself with -# nginx. Part of that start-up configuration renders nginx.conf into -# conf.d with nginx listening on PORT. NGINX_ENVSUBST_FILTER limits +# nginx. Part of that start-up configuration renders configs/nginx.conf +# into conf.d with nginx listening on PORT. NGINX_ENVSUBST_FILTER limits # that rendering to PORT: a variable nginx itself uses, such as $uri, # would otherwise be replaced by an environment variable of the same # name. diff --git a/backend/cmd/netwatch-server/main.go b/cmd/netwatch-server/main.go similarity index 100% rename from backend/cmd/netwatch-server/main.go rename to cmd/netwatch-server/main.go diff --git a/backend/cmd/netwatch-server/main_test.go b/cmd/netwatch-server/main_test.go similarity index 100% rename from backend/cmd/netwatch-server/main_test.go rename to cmd/netwatch-server/main_test.go diff --git a/nginx.conf b/configs/nginx.conf similarity index 98% rename from nginx.conf rename to configs/nginx.conf index d8b0c67..53f821c 100644 --- a/nginx.conf +++ b/configs/nginx.conf @@ -53,7 +53,7 @@ server { # netwatch-server sets the same security headers on its own # responses. Its copies are dropped so that each header goes out - # once, as security-headers.conf sets it. + # once, as configs/security-headers.conf sets it. proxy_hide_header Strict-Transport-Security; proxy_hide_header Content-Security-Policy; proxy_hide_header X-Frame-Options; diff --git a/security-headers.conf b/configs/security-headers.conf similarity index 72% rename from security-headers.conf rename to configs/security-headers.conf index 6602d6e..4f6e413 100644 --- a/security-headers.conf +++ b/configs/security-headers.conf @@ -1,5 +1,5 @@ # The security headers REPO_POLICIES.md requires on every response. -# nginx.conf includes this file, which Dockerfile copies to +# configs/nginx.conf includes this file, which Dockerfile copies to # /etc/nginx/security-headers.conf. always sends each header on error # responses too. @@ -8,11 +8,11 @@ add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" alway # Scripts and styles load only from the page's own origin. Inline ones # are blocked, style attributes in markup included, so style elements # through classes or element.style. data: images are for the favicon -# in index.html. connect-src is * because the browser checks each probe in -# src/main.js against it, and also every redirect the probe follows, -# and several of those hosts redirect to others; a list of hosts here -# would block those probes. It also covers the reports the page sends -# to its own origin. +# in web/index.html. connect-src is * because the browser checks each +# probe in web/src/main.js against it, and also every redirect the probe +# follows, and several of those hosts redirect to others; a list of +# hosts here would block those probes. It also covers the reports the +# page sends to its own origin. add_header Content-Security-Policy "default-src 'self'; connect-src *; img-src 'self' data:; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" always; add_header X-Frame-Options DENY always; diff --git a/eslint.config.js b/eslint.config.js index 1a2d756..5ff3543 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -8,7 +8,7 @@ export default defineConfig([ js.configs.recommended, { // The page, and facts.js, which the viewport test runs inside it. - files: ["src/**/*.js", "test/viewport/facts.js"], + files: ["web/src/**/*.js", "test/viewport/facts.js"], languageOptions: { globals: { ...globals.browser, diff --git a/backend/go.mod b/go.mod similarity index 100% rename from backend/go.mod rename to go.mod diff --git a/backend/go.sum b/go.sum similarity index 100% rename from backend/go.sum rename to go.sum diff --git a/backend/internal/config/config.go b/internal/config/config.go similarity index 98% rename from backend/internal/config/config.go rename to internal/config/config.go index 348380d..f92d0f1 100644 --- a/backend/internal/config/config.go +++ b/internal/config/config.go @@ -29,8 +29,8 @@ import ( const defaultTrustedProxies = "127.0.0.1/32,::1/128," + "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" -// Default limits on stored reports; backend/README.md gives the -// reasons for these values. +// Default limits on stored reports; README.md gives the reasons for +// these values. const ( defaultReportsPerMinute = 60 defaultDataDirMaxBytes = 1 << 30 // 1 GiB diff --git a/backend/internal/config/config_test.go b/internal/config/config_test.go similarity index 100% rename from backend/internal/config/config_test.go rename to internal/config/config_test.go diff --git a/backend/internal/globals/globals.go b/internal/globals/globals.go similarity index 100% rename from backend/internal/globals/globals.go rename to internal/globals/globals.go diff --git a/backend/internal/handlers/export_test.go b/internal/handlers/export_test.go similarity index 100% rename from backend/internal/handlers/export_test.go rename to internal/handlers/export_test.go diff --git a/backend/internal/handlers/handlers.go b/internal/handlers/handlers.go similarity index 100% rename from backend/internal/handlers/handlers.go rename to internal/handlers/handlers.go diff --git a/backend/internal/handlers/healthcheck.go b/internal/handlers/healthcheck.go similarity index 100% rename from backend/internal/handlers/healthcheck.go rename to internal/handlers/healthcheck.go diff --git a/backend/internal/handlers/healthcheck_test.go b/internal/handlers/healthcheck_test.go similarity index 100% rename from backend/internal/handlers/healthcheck_test.go rename to internal/handlers/healthcheck_test.go diff --git a/backend/internal/handlers/report.go b/internal/handlers/report.go similarity index 100% rename from backend/internal/handlers/report.go rename to internal/handlers/report.go diff --git a/backend/internal/handlers/report_test.go b/internal/handlers/report_test.go similarity index 100% rename from backend/internal/handlers/report_test.go rename to internal/handlers/report_test.go diff --git a/backend/internal/healthcheck/healthcheck.go b/internal/healthcheck/healthcheck.go similarity index 100% rename from backend/internal/healthcheck/healthcheck.go rename to internal/healthcheck/healthcheck.go diff --git a/backend/internal/logger/logger.go b/internal/logger/logger.go similarity index 100% rename from backend/internal/logger/logger.go rename to internal/logger/logger.go diff --git a/backend/internal/middleware/export_test.go b/internal/middleware/export_test.go similarity index 100% rename from backend/internal/middleware/export_test.go rename to internal/middleware/export_test.go diff --git a/backend/internal/middleware/middleware.go b/internal/middleware/middleware.go similarity index 100% rename from backend/internal/middleware/middleware.go rename to internal/middleware/middleware.go diff --git a/backend/internal/middleware/middleware_test.go b/internal/middleware/middleware_test.go similarity index 100% rename from backend/internal/middleware/middleware_test.go rename to internal/middleware/middleware_test.go diff --git a/backend/internal/reportbuf/datadir.go b/internal/reportbuf/datadir.go similarity index 100% rename from backend/internal/reportbuf/datadir.go rename to internal/reportbuf/datadir.go diff --git a/backend/internal/reportbuf/datadir_test.go b/internal/reportbuf/datadir_test.go similarity index 100% rename from backend/internal/reportbuf/datadir_test.go rename to internal/reportbuf/datadir_test.go diff --git a/backend/internal/reportbuf/export_test.go b/internal/reportbuf/export_test.go similarity index 100% rename from backend/internal/reportbuf/export_test.go rename to internal/reportbuf/export_test.go diff --git a/backend/internal/reportbuf/reportbuf.go b/internal/reportbuf/reportbuf.go similarity index 100% rename from backend/internal/reportbuf/reportbuf.go rename to internal/reportbuf/reportbuf.go diff --git a/backend/internal/reportbuf/reportbuf_test.go b/internal/reportbuf/reportbuf_test.go similarity index 100% rename from backend/internal/reportbuf/reportbuf_test.go rename to internal/reportbuf/reportbuf_test.go diff --git a/backend/internal/server/export_test.go b/internal/server/export_test.go similarity index 100% rename from backend/internal/server/export_test.go rename to internal/server/export_test.go diff --git a/backend/internal/server/http.go b/internal/server/http.go similarity index 100% rename from backend/internal/server/http.go rename to internal/server/http.go diff --git a/backend/internal/server/http_test.go b/internal/server/http_test.go similarity index 100% rename from backend/internal/server/http_test.go rename to internal/server/http_test.go diff --git a/backend/internal/server/routes.go b/internal/server/routes.go similarity index 100% rename from backend/internal/server/routes.go rename to internal/server/routes.go diff --git a/backend/internal/server/routes_test.go b/internal/server/routes_test.go similarity index 100% rename from backend/internal/server/routes_test.go rename to internal/server/routes_test.go diff --git a/backend/internal/server/server.go b/internal/server/server.go similarity index 100% rename from backend/internal/server/server.go rename to internal/server/server.go diff --git a/script/bootstrap b/script/bootstrap index d388af8..1bf0e7f 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -17,7 +17,7 @@ # golangci-lint is not installed: make lint runs it in Docker, as make # test runs the tests, and this script does not install Docker either. # -# Unlike the org model: Go and gcc for backend/, a newer node for eslint. +# Unlike the org model: Go for the backend, a newer node for eslint. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -272,19 +272,13 @@ main() { if missing make; then pkg_install gnumake make make make; fi if missing git; then pkg_install git git git git; fi - # The race detector in backend/'s make test needs cgo, which Go turns - # on only when it finds its C compiler, gcc on Linux. apt and apk - # ship the C library headers apart from gcc. - if missing gcc; then - pkg_install gcc "gcc libc6-dev" gcc "gcc musl-dev" - fi ensure_node ensure_yarn yarn install --frozen-lockfile ensure_go - (cd "$ROOT/backend" && go mod download) + go mod download if missing docker; then echo "bootstrap: docker not found; make test and make lint, and so" >&2 diff --git a/script/build b/script/build index 68c4bcb..710bca6 100755 --- a/script/build +++ b/script/build @@ -1,6 +1,7 @@ #!/bin/sh -# script/build: build the frontend for production into dist/. The Go -# backend is built by backend/script/build. +# script/build: build the frontend for production into dist/, then the +# static netwatch-server binary into the repo root, with its version +# stamped in. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -8,6 +9,14 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" yarn build + + # VERSION comes from the environment. Unset or empty, it is git + # describe, or "dev" where there is no git or no repository history. + version="${VERSION:-$(git describe --always --dirty 2>/dev/null || echo dev)}" + + CGO_ENABLED=0 go build -trimpath \ + -ldflags "-s -w -X main.Version=$version" \ + -o netwatch-server ./cmd/netwatch-server/ } main "$@" diff --git a/script/clean b/script/clean new file mode 100755 index 0000000..4f00626 --- /dev/null +++ b/script/clean @@ -0,0 +1,13 @@ +#!/bin/sh +# script/clean: remove what script/build writes: dist/ and the +# netwatch-server binary. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + rm -rf dist netwatch-server +} + +main "$@" diff --git a/script/fmt b/script/fmt index c18d300..f998d21 100755 --- a/script/fmt +++ b/script/fmt @@ -1,7 +1,9 @@ #!/bin/sh # script/fmt: format the whole repo (writes): prettier over the -# JavaScript, CSS, HTML and Markdown, then gofmt over the Go backend. -# The org model formats only markdown; this repo also has JS and Go. +# JavaScript, CSS, HTML and Markdown, then gofmt over the Go sources in +# cmd/ and internal/; not over the whole root, as node_modules/ holds +# .go files of its own. The org model formats only markdown; this repo +# also has JS and Go. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" @@ -12,7 +14,7 @@ main() { # ~/.local/bin, which the shell that called it may not have on PATH. PATH="$HOME/.local/bin:$PATH" "$ROOT/script/frontend-fmt" - "$ROOT/backend/script/fmt" + gofmt -l -w cmd internal } main "$@" diff --git a/script/fmt-check b/script/fmt-check index 72788e7..91dd6b3 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -12,7 +12,12 @@ main() { # ~/.local/bin, which the shell that called it may not have on PATH. PATH="$HOME/.local/bin:$PATH" "$ROOT/script/frontend-fmt-check" - "$ROOT/backend/script/fmt-check" + unformatted="$(gofmt -l cmd internal)" + if [ -n "$unformatted" ]; then + echo "Go files not formatted:" >&2 + echo "$unformatted" >&2 + exit 1 + fi } main "$@" diff --git a/script/frontend-fmt b/script/frontend-fmt index a16bcd9..ce1322d 100755 --- a/script/frontend-fmt +++ b/script/frontend-fmt @@ -1,9 +1,9 @@ #!/bin/sh # script/frontend-fmt: format the JavaScript, CSS, HTML and Markdown, -# repo-wide (writes), the markdown in backend/ included. Those are the -# languages REPO_POLICIES.md gives prettier; the YAML is left alone, as -# backend/.golangci.yml must stay the org standard byte for byte. -# Prettier does not read Go; backend/script/fmt formats the Go sources. +# repo-wide (writes). Those are the languages REPO_POLICIES.md gives +# prettier; the YAML is left alone, as .golangci.yml must stay the org +# standard byte for byte. Prettier does not read Go; script/fmt runs +# gofmt over the Go sources. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/script/frontend-viewport-test b/script/frontend-viewport-test index d26e0fe..fba6fb8 100755 --- a/script/frontend-viewport-test +++ b/script/frontend-viewport-test @@ -2,11 +2,11 @@ # script/frontend-viewport-test: verify the responsive layout of the built # frontend in a real browser engine. # -# Builds dist/, serves it with the same nginx image and the same nginx.conf -# the shipping container uses, points a containerised headless Chrome at it -# over CDP, and asserts on computed layout at every viewport width derived -# from the app's own CSS. See test/viewport/README.md for what this covers -# and what it cannot. +# Builds dist/, serves it with the same nginx image and the same +# configs/nginx.conf the shipping container uses, points a containerised +# headless Chrome at it over CDP, and asserts on computed layout at every +# viewport width derived from the app's own CSS. See +# test/viewport/README.md for what this covers and what it cannot. # # Deliberately not part of script/check: it needs Docker and takes far # longer than the 20s budget make test has to stay inside. @@ -61,18 +61,18 @@ main() { # host. docker network create --internal "$NETWORK" > /dev/null - # nginx.conf is a template: the image renders it over its own + # configs/nginx.conf is a template: the image renders it over its own # default.conf, with the same port and limit bin/entrypoint.sh uses. # The empty file it includes trusts no proxy, as bin/entrypoint.sh - # writes it when TRUSTED_PROXIES is unset. nginx.conf also includes - # the security headers, so the page runs under the shipped policy. + # writes it when TRUSTED_PROXIES is unset. It also includes the + # security headers, so the page runs under the shipped policy. docker run -d --rm --name "$SERVER" \ --network "$NETWORK" --network-alias netwatch \ -e PORT=8080 -e NGINX_ENVSUBST_FILTER='^PORT$' \ -v "$ROOT/dist:/usr/share/nginx/html:ro" \ - -v "$ROOT/nginx.conf:/etc/nginx/templates/default.conf.template:ro" \ + -v "$ROOT/configs/nginx.conf:/etc/nginx/templates/default.conf.template:ro" \ -v /dev/null:/etc/nginx/trusted-proxies.conf:ro \ - -v "$ROOT/security-headers.conf:/etc/nginx/security-headers.conf:ro" \ + -v "$ROOT/configs/security-headers.conf:/etc/nginx/security-headers.conf:ro" \ "$SERVER_IMAGE" > /dev/null # The image's own entrypoint already exposes CDP on 9222 and passes diff --git a/backend/script/run b/script/run similarity index 53% rename from backend/script/run rename to script/run index 5ea87ba..824af6c 100755 --- a/backend/script/run +++ b/script/run @@ -1,5 +1,6 @@ #!/bin/sh -# script/run: build and run netwatch-server locally. +# script/run: build with script/build, then run netwatch-server locally +# with its defaults: port 8080, reports stored in data/reports. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" diff --git a/script/tidy b/script/tidy index fadf4cd..b948505 100755 --- a/script/tidy +++ b/script/tidy @@ -1,12 +1,12 @@ #!/bin/sh -# script/tidy: run go mod tidy in backend/, which adds the modules the -# Go sources import, drops those they no longer do, and updates go.sum. +# script/tidy: run go mod tidy, which adds the modules the Go sources +# import, drops those they no longer do, and updates go.sum. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { - cd "$ROOT/backend" + cd "$ROOT" go mod tidy } diff --git a/test/unit/main.test.js b/test/unit/main.test.js index d47bafe..360b1a7 100644 --- a/test/unit/main.test.js +++ b/test/unit/main.test.js @@ -1,4 +1,4 @@ -// Unit tests for src/main.js, run with Node's built-in test runner by the +// Unit tests for web/src/main.js, run with Node's built-in test runner by the // test script in package.json. Importing the module does not start the // page. @@ -15,7 +15,7 @@ import { latencyHex, measureLatency, tick, -} from "../../src/main.js"; +} from "../../web/src/main.js"; // There is no page here, so the tests stand in for it. The debug log looks // for its panel by id and finds none. Each element of a host's row that diff --git a/test/viewport/README.md b/test/viewport/README.md index 8d7f2de..998189d 100644 --- a/test/viewport/README.md +++ b/test/viewport/README.md @@ -7,11 +7,11 @@ make frontend-viewport-test ``` It builds `dist/`, serves it from the same digest-pinned `nginx` image and the -same `nginx.conf` the shipping container uses, drives a digest-pinned headless -Chrome against it over CDP, and asserts on computed layout at every viewport -width derived from the app's own CSS. Screenshots land in `tmp/viewport/` -alongside a `results.json`; they are artifacts for a human to look at when -something fails, not the evidence. The assertions are the evidence. +same `configs/nginx.conf` the shipping container uses, drives a digest-pinned +headless Chrome against it over CDP, and asserts on computed layout at every +viewport width derived from the app's own CSS. Screenshots land in +`tmp/viewport/` alongside a `results.json`; they are artifacts for a human to +look at when something fails, not the evidence. The assertions are the evidence. The target is deliberately outside `make check`: it takes minutes, and `make test` has to stay under 60 seconds. @@ -19,10 +19,10 @@ The target is deliberately outside `make check`: it takes minutes, and ## How the widths are chosen Not from a list of phone models. `viewports.js` parses the `@media` conditions -out of `src/styles.css` and scans `src/main.js` and `index.html` for Tailwind -responsive prefixes, then tests every breakpoint it finds at one pixel below it, -exactly on it, and one pixel above it. A generic 375px "phone" test sails -straight past an off-by-one at a media query boundary; `max-width: 768px` +out of `web/src/styles.css` and scans `web/src/main.js` and `web/index.html` for +Tailwind responsive prefixes, then tests every breakpoint it finds at one pixel +below it, exactly on it, and one pixel above it. A generic 375px "phone" test +sails straight past an off-by-one at a media query boundary; `max-width: 768px` matches _at_ 768, and the sweep pins down which side of that line each layout is on. @@ -108,7 +108,7 @@ function of viewport width and CSS, and is covered above. Complementary layers, not two stacks. The unit tests in `test/unit/`, which `make test` runs with Node's built-in test runner, exercise the functions -`src/main.js` exports in-process with no browser. This harness exercises +`web/src/main.js` exports in-process with no browser. This harness exercises rendered layout in a real engine and is the only thing here that can see a media query. Neither replaces the other; assertions about computed styles and element geometry belong here, assertions about functions belong in `test/unit/`. diff --git a/test/viewport/harness.js b/test/viewport/harness.js index 4dd95fa..d5a90c8 100644 --- a/test/viewport/harness.js +++ b/test/viewport/harness.js @@ -51,7 +51,8 @@ const PROBE_DELAYS_MS = [2, 45, 123, 456, 780]; // offline row rendering is exercised too. const UNREACHABLE_MODULUS = 7; // The gateway candidate that "answers", so gateway detection succeeds and -// the Local Gateway row renders. Matches GATEWAY_CANDIDATES in src/main.js. +// the Local Gateway row renders. Matches GATEWAY_CANDIDATES in +// web/src/main.js. const RESPONSIVE_GATEWAY = "http://192.168.1.1"; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/test/viewport/viewports.js b/test/viewport/viewports.js index 23a20b2..31579ca 100644 --- a/test/viewport/viewports.js +++ b/test/viewport/viewports.js @@ -161,26 +161,27 @@ const ANCHOR_VIEWPORTS = [ export function deriveViewports(root) { const conditions = [ ...mediaConditionsFromCss( - readFileSync(join(root, "src/styles.css"), "utf8"), - "src/styles.css", + readFileSync(join(root, "web/src/styles.css"), "utf8"), + "web/src/styles.css", ), ...mediaConditionsFromMarkup([ { - path: "src/main.js", - text: readFileSync(join(root, "src/main.js"), "utf8"), + path: "web/src/main.js", + text: readFileSync(join(root, "web/src/main.js"), "utf8"), }, { - path: "index.html", - text: readFileSync(join(root, "index.html"), "utf8"), + path: "web/index.html", + text: readFileSync(join(root, "web/index.html"), "utf8"), }, ]), ]; if (conditions.length === 0) { throw new Error( - "no responsive breakpoints found in src/styles.css, src/main.js or " + - "index.html — either the responsive layout was deleted or this " + - "derivation has stopped matching the sources", + "no responsive breakpoints found in web/src/styles.css, " + + "web/src/main.js or web/index.html — either the responsive " + + "layout was deleted or this derivation has stopped matching " + + "the sources", ); } diff --git a/vite.config.js b/vite.config.js index 0f362bd..317bd86 100644 --- a/vite.config.js +++ b/vite.config.js @@ -6,6 +6,10 @@ const commitHash = execSync("git rev-parse --short HEAD").toString().trim(); const commitFull = execSync("git rev-parse HEAD").toString().trim(); export default defineConfig({ + // The page's source is in web/; the build still goes to dist/ at the + // repo root, which Dockerfile and the viewport test serve. Vite + // empties only an outDir inside root unless told to. + root: "web", plugins: [tailwindcss()], server: { // Proxy /api to a locally running netwatch-server so `yarn dev` @@ -19,6 +23,8 @@ export default defineConfig({ __COMMIT_FULL__: JSON.stringify(commitFull), }, build: { + outDir: "../dist", + emptyOutDir: true, target: "esnext", minify: "esbuild", cssMinify: true, diff --git a/index.html b/web/index.html similarity index 90% rename from index.html rename to web/index.html index ddb6af0..a1e31c4 100644 --- a/index.html +++ b/web/index.html @@ -9,7 +9,7 @@ type="image/svg+xml" href="data:image/svg+xml,📡" /> - diff --git a/src/main.js b/web/src/main.js similarity index 100% rename from src/main.js rename to web/src/main.js diff --git a/src/styles.css b/web/src/styles.css similarity index 100% rename from src/styles.css rename to web/src/styles.css