diff --git a/.dockerignore b/.dockerignore index 033ab7b..2b5e166 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,80 @@ -node_modules -dist -tmp -.DS_Store -*.log +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. + +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a +# password in a remote URL or the token the CI checkout step stores there. +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config + +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. .claude -# .git is sent so the build can stamp the version, without its config. -.git/config +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +**/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* + +# This repository's own host-built artifacts, after the shared content +# above: the frontend build, the viewport test's output, and what +# `make build` and `make run` in backend/ leave behind. +/dist +/tmp +/backend/netwatch-server +/backend/data diff --git a/.editorconfig b/.editorconfig index 2fe0ce0..b8d5a99 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,8 @@ insert_final_newline = true [Makefile] indent_style = tab + +# This repository's own entries, after the shared content above. + +[*.go] +indent_style = tab diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 08c2ebc..ee73864 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,7 +6,4 @@ jobs: steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - # script/cibuild bootstraps, runs every check and builds the - # image. script/bootstrap links what it installs into - # ~/.local/bin, so that has to be on PATH for the rest. - - run: PATH="$HOME/.local/bin:$PATH" script/cibuild + - run: script/cibuild diff --git a/.gitignore b/.gitignore index e676a31..ce123a5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,18 +11,51 @@ Thumbs.db .vscode/ *.sublime-* +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + # Node node_modules/ -# Environment / secrets -.env -.env.* -*.pem -*.key +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation after these lines, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] + +# This repository's own entries, after the shared content above. # Build output dist/ tmp/ +/backend/netwatch-server + +# Go test binaries and coverage output +*.test +*.out # Logs *.log diff --git a/.prettierignore b/.prettierignore index ddbaa34..23d67fc 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,7 +1,2 @@ -dist/ node_modules/ -tmp/ yarn.lock -.claude/ -# The org standard file, copied verbatim; backend/script/lint checks its sha256. -backend/.golangci.yml diff --git a/Dockerfile b/Dockerfile index 19e1591..c2c9f5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,95 +2,116 @@ # passes /api/, /.well-known/healthcheck and /metrics to netwatch-server, # the Go backend, which runs in the same container on loopback only. # bin/entrypoint.sh starts and watches both. - -# Lint stage — fast feedback on formatting and lint issues. The -# golangci/golangci-lint image ships Go, gofmt, make and the linter, so -# nothing is installed here. The root make lint builds this stage alone. -# golangci/golangci-lint:v2.12.2 (2026-08-10) -FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint - -WORKDIR /src -COPY backend/go.mod backend/go.sum ./ -RUN go mod download -COPY backend/ . -RUN make fmt-check -RUN make lint - -# Backend build stage -# golang:1.25-alpine (2026-02-27) -FROM golang:1.25-alpine@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder - -# gcc and musl-dev are for make test: its race detector needs cgo, which -# Go turns on by itself once a C compiler is present. make build still -# sets CGO_ENABLED=0, so the binary stays static. -RUN apk add --no-cache gcc git make musl-dev - -WORKDIR /src - -# Force BuildKit to run the lint stage before proceeding. BuildKit runs -# stages in parallel by default; without this no-op copy a lint failure -# would not gate compilation. -COPY --from=lint /src/go.sum /dev/null - -COPY backend/go.mod backend/go.sum ./ -RUN go mod download -COPY backend/ . - -RUN make test - -# make build is a shim around backend/script/build, the one definition -# of the build command: -# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=..." -# That script reads VERSION from the environment, so it is handed over -# there rather than as a make variable. # -# The version is the VERSION build argument when one is given, otherwise -# `git describe --tags --always` of the repo's .git: the tag on a tagged -# commit, tag-N-gHASH on a commit after one, the short commit when no -# tag is reachable. A version that still comes out empty, dev or unknown -# fails the build. .git goes to /git, not /src/.git, where go build would -# find it and record VCS details of a work tree holding only backend/. -COPY .git /git -ARG VERSION -RUN version="${VERSION:-$(git --git-dir=/git describe --tags --always)}"; \ - case "$version" in ""|dev|unknown) \ - echo "version is '$version' although .git is present" >&2; \ - exit 1 ;; \ - esac; \ - VERSION="$version" make build +# The lint and test phases are the gates: `make lint` (script/lint) +# builds the lint stage alone and `make test` (script/test) the test +# stage alone, and the builder stage depends on both, so the image +# cannot be built unless they pass. Each covers the frontend as well, +# through a copy from a node stage. Inside them each tool is invoked +# directly, never through make or script/, whose lint and test are +# themselves docker builds. -# Frontend lint stage — eslint over the JavaScript, as the lint stage -# above lints the Go. The root make lint builds this stage alone too. +# Frontend lint stage: eslint with the rules in eslint.config.js. The +# lint phase below runs it. # node:22-alpine as of 2026-02-22 FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend-lint WORKDIR /app COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile COPY . . -RUN script/frontend-lint +RUN yarn eslint . -# Frontend stage +# Lint phase: golangci-lint over the backend with backend/.golangci.yml, +# and eslint through the copy from frontend-lint at the end. The +# golangci/golangci-lint image ships Go and the linter. +# golangci/golangci-lint:v2.14.0, 2026-09-24 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint +WORKDIR /src +COPY backend/go.mod backend/go.sum ./ +RUN go mod download +COPY backend/ . +RUN golangci-lint run --config .golangci.yml ./... +# Nothing is wanted from frontend-lint; the copy is what makes this +# phase run it. +COPY --from=frontend-lint /app/yarn.lock /dev/null + +# Frontend stage: the unit tests in test/unit/, then the production +# build into dist/, which the runtime stage serves. The test phase below +# runs it. The tests print a dot each; if any fails, they run again with +# every test listed, and the step fails even if that run passes. +# NODE_OPTIONS chooses the reporter because yarn adds its arguments +# after the test files, where node would take a reporter option for one +# more file. # node:22-alpine as of 2026-02-22 FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS frontend WORKDIR /app - -# Force BuildKit to run the frontend-lint stage before proceeding, as -# the builder stage does with the lint stage: without this no-op copy an -# eslint failure would not gate the image. -COPY --from=frontend-lint /app/yarn.lock /dev/null - COPY package.json yarn.lock ./ RUN yarn install --frozen-lockfile -RUN apk add --no-cache git make +# vite.config.js reads the commit for the page's footer with git. +RUN apk add --no-cache git COPY . . -# make frontend-check runs the frontend tests and format check; its test -# step runs the unit tests, then the production yarn build, so this both -# produces dist/ and gates the image on test and formatting regressions. -# This node stage has neither Go nor Docker; the frontend-lint, lint and -# builder stages above gate the rest. -RUN make frontend-check +RUN NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || \ + { echo "--- Rerunning with every test listed for details ---"; \ + NODE_OPTIONS=--test-reporter=spec timeout 90 yarn --silent run test; \ + exit 1; } +RUN yarn build -# Runtime stage +# Test phase: the backend's tests with the race detector and coverage, +# and the frontend's through the copy from the frontend stage at the +# end. -race needs cgo and so a C compiler, which the Debian Go image +# ships and the alpine one does not. -timeout 90s is a backstop above +# the 60-second cap on the suite. The rerun with -v only shows details: +# the step fails however it ends, because the first run already failed. +# golang:1.25.7-trixie, 2026-10-07 +FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test +WORKDIR /src +COPY backend/go.mod backend/go.sum ./ +RUN go mod download +COPY backend/ . +RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } +# Nothing is wanted from the frontend stage; the copy is what makes +# this phase run its tests. +COPY --from=frontend /app/yarn.lock /dev/null + +# Backend build stage. Nothing is wanted from the two phases; the copies +# are what make BuildKit build them first, so this stage cannot run +# unless lint and test passed. +# golang:1.25-alpine (2026-02-27) +FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder +COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null +RUN apk add --no-cache git +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src +WORKDIR /src +COPY backend/go.mod backend/go.sum backend/ +RUN cd backend && go mod download +COPY . . + +# backend/script/build is the one definition of the build command: +# CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.Version=..." +# It reads VERSION from the environment. +# +# The version is the VERSION build argument when one is given, otherwise +# `git describe --tags --always` on the .git in the build context: the +# tag on a tagged commit, tag-N-gHASH on a commit after one, the short +# commit when no tag is reachable. With .git present, a version that is +# still empty, dev or unknown fails the build: git is missing or could +# not read the checkout. +ARG VERSION +RUN version="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$version" in ""|dev|unknown) \ + echo "version is '$version' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + VERSION="$version" backend/script/build + +# Runtime stage, and the last one: a plain `docker build .` builds it +# and the stages it copies from, the two phases included. # nginx:stable-alpine as of 2026-02-22 FROM nginx@sha256:15e96e59aa3b0aada3a121296e3bce117721f42d88f5f64217ef4b18f458c6ab @@ -106,7 +127,7 @@ RUN rm /etc/nginx/conf.d/default.conf COPY nginx.conf /etc/nginx/templates/netwatch.conf.template COPY security-headers.conf /etc/nginx/security-headers.conf COPY --from=frontend /app/dist /usr/share/nginx/html -COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server +COPY --from=builder /src/backend/netwatch-server /usr/local/bin/netwatch-server COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh # bin/entrypoint.sh creates DATA_DIR at start and gives it and /data to diff --git a/Makefile b/Makefile index 29d7600..fefa8ae 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: bootstrap setup dev build test lint fmt fmt-check check \ - add-dependency tidy frontend-check frontend-viewport-test docker hooks + add-dependency tidy frontend-viewport-test docker hooks # Standard targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern (see the Entrypoints section @@ -42,14 +42,8 @@ add-dependency: tidy: @script/tidy -# The frontend tests and format check, for Dockerfile's frontend stage, -# which has neither Go nor Docker. Use check everywhere else. -frontend-check: - @script/frontend-check - # Responsive-layout verification in a containerised browser. Kept out of -# check: it needs Docker and takes minutes, where make test has to stay -# under 20 seconds. +# check: it takes minutes. frontend-viewport-test: @script/frontend-viewport-test diff --git a/README.md b/README.md index c3a11e0..94a6c75 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ halves, so the root `make check` fails if either one is broken. We provide: corepack, `yarn install --frozen-lockfile`, the pinned Go unless one at least as new as `backend/go.mod` asks for is installed, the Go modules, and gcc with the C library headers unless gcc is installed, for the race detector in - `make test`), linking what it installs itself into `~/.local/bin`, which has - to be on `PATH`. It installs no Go linter and not Docker: `make lint` runs - both linters in Docker + `make test` in `backend/`), linking what it installs itself into + `~/.local/bin`, which has to be on `PATH`. It installs no Go linter and not + Docker: `make test` and `make lint` run in Docker - `script/setup` — make a fresh clone ready for development: bootstrap plus the git pre-commit hook - `script/dev` — run the Vite dev server, which proxies `/api` to a locally @@ -54,11 +54,14 @@ halves, so the root `make check` fails if either one is broken. We provide: - `script/build` — build the frontend for production into `dist/`; `backend/script/build` builds the Go server - `script/projectname` — print the project name (used for the Docker image tag) -- `script/test` — run `script/frontend-test`, then `backend/script/test`, the +- `script/test` — build the `test` phase of `Dockerfile` without the cache: the + frontend's unit tests and production build in its `frontend` stage, and the backend's Go tests with the race detector and coverage -- `script/lint` — run eslint, then golangci-lint, both in Docker, by building - the `frontend-lint` and `lint` stages of `Dockerfile` without the cache -- `script/fmt` — format all files (writes): prettier, then gofmt over `backend/` +- `script/lint` — build the `lint` phase of `Dockerfile` without the cache: + eslint in its `frontend-lint` stage, and golangci-lint over `backend/` +- `script/fmt` — format all files (writes): prettier over the JavaScript, CSS, + HTML and Markdown, then gofmt over `backend/`. It runs on the host, as + `script/fmt-check` does, with `~/.local/bin` put on `PATH` - `script/fmt-check` — check formatting (read-only): prettier, then gofmt - `script/check` — run test, lint, and fmt-check - `script/add-dependency` — add a frontend package, or move one to another @@ -68,23 +71,18 @@ halves, so the root `make check` fails if either one is broken. We provide: - `script/tidy` — run `go mod tidy` in `backend/`: to add a Go module, import it and run `make tidy`; to move one to another version, edit its `require` line in `backend/go.mod`, then run `make tidy` -- `script/frontend-test` — run the unit tests in `test/unit/` with Node's - built-in test runner, through the `test` script in `package.json`, and if any - fails, run them again listing every test, and fail; then the production build. - Each run has a 30-second timeout -- `script/frontend-lint` — run eslint with the rules in `eslint.config.js`; it - runs inside the `frontend-lint` stage of `Dockerfile`, which `make lint` - builds -- `script/frontend-fmt` — format everything prettier understands (writes), the - markdown in `backend/` included +- `script/frontend-test` — run the unit tests in `test/unit/` on the host with + Node's built-in test runner, through the `test` script in `package.json`, and + if any fails, run them again listing every test, and fail; then the production + build. Each test run has a 90-second timeout. `make test` runs the same in + Docker +- `script/frontend-fmt` — format the JavaScript, CSS, HTML and Markdown with + prettier (writes), the markdown in `backend/` included - `script/frontend-fmt-check` — check prettier formatting (read-only) -- `script/frontend-check` — run `script/frontend-test` and - `script/frontend-fmt-check`, for the frontend stage of `Dockerfile`, which has - neither Go nor Docker - `script/frontend-viewport-test` — responsive-layout verification of the built frontend in a containerised headless Chrome (see [test/viewport/README.md](test/viewport/README.md)). Not part of - `script/check`: it needs Docker and takes minutes. + `script/check`: it takes minutes. - `script/docker` — build the image from `Dockerfile` without the build cache, tagged `netwatch` via `script/projectname` - `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`, @@ -164,11 +162,11 @@ and when it stops. Its routes: `METRICS_USERNAME` and `METRICS_PASSWORD` are set; each client address may make a limited number of requests to it a minute -In the image, the `builder` stage of `Dockerfile` tests it and builds it with -`backend/script/build`, and `bin/entrypoint.sh` runs it as user `netwatch` on -`127.0.0.1:8081`, behind nginx. Outside the image, `make run` in `backend/` -builds it and runs it on port 8080. Its settings, report storage and limits are -in [backend/README.md](backend/README.md). +In the image, the `test` phase of `Dockerfile` tests it, the `builder` stage +builds it with `backend/script/build`, and `bin/entrypoint.sh` runs it as user +`netwatch` on `127.0.0.1:8081`, behind nginx. Outside the image, `make run` in +`backend/` builds it and runs it on port 8080. Its settings, report storage and +limits are in [backend/README.md](backend/README.md). ### Monitoring targets diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..20382d1 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-10-04 --- This document covers repository structure, tooling, and workflow standards. Code @@ -60,17 +60,28 @@ style conventions are in separate documents: prerequisite since nvm requires bash. yarn is then pinned via `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific - pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in - `script/precommit`, not in the hook itself. Model scripts are at + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -89,87 +100,198 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + When a check is added or changed, prove it works by planting a defect it must + catch and watching the run fail on it, then revert the defect. A green run + alone shows neither that the check ran nor that it covers what it should. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Debian Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make fmt-check - RUN make lint + RUN golangci-lint run --config .golangci.yml ./... - # Build stage - # golang:1.x-alpine, YYYY-MM-DD - FROM golang@sha256:... AS builder + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. + # golang:1.x, YYYY-MM-DD + FROM golang@sha256:... AS test WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make test + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + RUN apk add --no-cache git + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . - # Runtime stage + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one FROM alpine@sha256:... COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them: + + ```dockerfile + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. A separate + workflow limited to `main` by a `branches` list under `on: push` cannot be + checked by review: to try a change to it, add the feature branch to that list + and push, then remove the branch from the list again before merging. Keep any + job in it that publishes behind `if: github.ref_name == 'main'`, so the run + from the feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -189,14 +311,21 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -209,11 +338,26 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. + + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. That matters on a developer's machine, where this + target runs and the directory lasts from one run to the next. The `test` + phase of the `Dockerfile` needs no `-count=1`: its base image holds no + result for this repo's tests and nothing before its `go test` step runs a + test, so there is nothing to replay. `--no-cache` (above) is what makes that + step run on an unchanged tree. + Python example: ```makefile @@ -239,10 +383,84 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.claude`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -258,9 +476,56 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -374,12 +639,14 @@ style conventions are in separate documents: settings. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) @@ -406,3 +673,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/TODO.md b/TODO.md index 91c6e04..af11818 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,22 @@ Decide whether the repo moves to the layout `REPO_POLICIES.md` gives, with # Completed Steps +- 2026-10-07: the files shared from `sneak/prompts` are its copies at commit + `dd4027b` ([#113](https://git.eeqj.de/sneak/netwatch/issues/113)), with this + repository's own entries after the shared content in `.gitignore`, + `.dockerignore` and `.editorconfig`. `make lint` and `make test` each build + one phase of `Dockerfile` without the cache: `lint` runs golangci-lint v2.14.0 + over `backend/` and, through the `frontend-lint` stage, eslint; `test` runs + the Go tests on the Debian Go image and, through the `frontend` stage, the + frontend's unit tests and build. The builder stage waits on both, and without + a `VERSION` build argument takes the version from `git describe` on the `.git` + in the build context. Neither linter nor the tests run on the host for + `make check`; `backend/script/lint` runs the root one, and + `backend/.golangci.yml` is no longer checked against a sha256. prettier + formats only the JavaScript, CSS, HTML and Markdown, so the shared + `.golangci.yml` stays as fetched. `script/fmt` and `script/fmt-check` put + `~/.local/bin` on `PATH`, which the shared workflow no longer does. + `script/frontend-lint` and `script/frontend-check` are gone - 2026-10-04: the page's footer no longer says "IPv4 only" ([#111](https://git.eeqj.de/sneak/netwatch/issues/111)): each check is a `fetch`, the browser picks IPv4 or IPv6 for each WAN host, and the local diff --git a/backend/.golangci.yml b/backend/.golangci.yml index a7a74c2..1b73eb9 100644 --- a/backend/.golangci.yml +++ b/backend/.golangci.yml @@ -17,6 +17,7 @@ linters: disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) - godot # Requires comments to end with periods - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go diff --git a/backend/README.md b/backend/README.md index 47f74ee..c6844b8 100644 --- a/backend/README.md +++ b/backend/README.md @@ -28,20 +28,21 @@ docker run -p 8080:8080 netwatch This directory follows the same [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) pattern as the repo root: the targets in `backend/Makefile` are thin shims over -`backend/script/`. The root `Dockerfile` runs them, and the root scripts call -`test`, `fmt` and `fmt-check`: +`backend/script/`. The root `Dockerfile` runs `build`, and the root scripts call +`fmt` and `fmt-check`: - `script/build` — compile the static `netwatch-server` binary with its version stamped in. The version is `VERSION` from the environment; when that is unset or empty, it falls back to `git describe` inside a git checkout, then to `dev` -- `script/test` — run the Go tests with the race detector and coverage. Go's - `-timeout 30s` bounds the tests, not their compile. If they fail, they run - again with `-v` for the details, and the script fails. The race detector needs - a C compiler -- `script/lint` — check `.golangci.yml` against its pinned sha256, then run - golangci-lint. It runs inside the golangci-lint image of the lint stage of the - root `Dockerfile`; from a checkout, run `make lint` at the repo root, which - builds that stage +- `script/test` — run the Go tests on the host with the race detector and + coverage; the root `make test` runs them in the `test` phase of the root + `Dockerfile`. Go's `-timeout 90s` bounds the tests, not their compile, and + `-count=1` keeps Go from reporting a stored pass. If they fail, they run again + with `-v` for the details, and the script fails. The race detector needs a C + compiler +- `script/lint` — run the root `script/lint`, which builds the `lint` phase of + the root `Dockerfile`: golangci-lint over this directory, and eslint over the + frontend. golangci-lint never runs on the host - `script/fmt` — format the Go sources (writes) - `script/fmt-check` — check Go formatting (read-only) - `script/run` — build and run the server locally diff --git a/backend/internal/middleware/middleware_test.go b/backend/internal/middleware/middleware_test.go index c916cf2..bbfa7c1 100644 --- a/backend/internal/middleware/middleware_test.go +++ b/backend/internal/middleware/middleware_test.go @@ -343,7 +343,7 @@ func TestLoggingCutsRequestStringsToBound(t *testing.T) { http.MethodGet, "/"+long, http.NoBody) req.Header.Set("User-Agent", long) req.Header.Set("Referer", long) - req.Header.Set("X-Request-Id", long) + req.Header.Set("X-Request-ID", long) handler.ServeHTTP(httptest.NewRecorder(), req) diff --git a/backend/script/lint b/backend/script/lint index b447e9e..897ba78 100755 --- a/backend/script/lint +++ b/backend/script/lint @@ -1,50 +1,13 @@ #!/bin/sh -# script/lint: run golangci-lint over the backend. This runs inside the -# lint stage of the root Dockerfile, whose digest-pinned golangci-lint -# image provides the linter; nothing installs golangci-lint on the host. -# From a checkout, run `make lint` at the repo root, which builds that -# stage. -# -# .golangci.yml is standardized org-wide and must never be edited here -# (REPO_POLICIES.md). Its last silent drift replaced the v2 schema with -# v1 keys, which left every threshold in the file inert while the build -# stayed green. So the file is first checked against the canonical -# copy's sha256: a local comparison, no network, nothing unpinned. +# script/lint: lint the whole repo as the root make lint does, by +# building the lint phase of the root Dockerfile. golangci-lint never +# runs on the host (REPO_POLICIES.md). set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -# The sha256 of the org standard .golangci.yml. When that file changes in -# sneak/prompts and is copied here again, this changes with it. -GOLANGCI_CONFIG_SHA256="a79b63a254602a5318db5d0e9a06bc71b84bf0c1d896305229d8bfed1d1b1776" +ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)" main() { - cd "$ROOT" - if [ ! -f .golangci.yml ]; then - echo "backend/.golangci.yml is missing. Copy the org standard verbatim" >&2 - echo "from https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml" >&2 - exit 1 - fi - actual="$(sha256sum .golangci.yml | cut -d' ' -f1)" - if [ -z "$actual" ]; then - echo "sha256sum is missing or printed no hash, so" >&2 - echo "backend/.golangci.yml could not be checked." >&2 - exit 1 - fi - if [ "$actual" != "$GOLANGCI_CONFIG_SHA256" ]; then - echo "backend/.golangci.yml does not match GOLANGCI_CONFIG_SHA256" >&2 - echo "in backend/script/lint." >&2 - echo " expected $GOLANGCI_CONFIG_SHA256" >&2 - echo " actual $actual" >&2 - echo "Compare it with the org standard," >&2 - echo "https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml" >&2 - echo "- If they differ, it was edited here: restore the org standard" >&2 - echo " verbatim. Do not edit it." >&2 - echo "- If they are the same, the org standard changed: set" >&2 - echo " GOLANGCI_CONFIG_SHA256 in backend/script/lint to the actual hash." >&2 - exit 1 - fi - golangci-lint run ./... + exec "$ROOT/script/lint" } main "$@" diff --git a/backend/script/test b/backend/script/test index a4063e3..5a95186 100755 --- a/backend/script/test +++ b/backend/script/test @@ -1,18 +1,21 @@ #!/bin/sh -# script/test: run the backend test suite with the race detector and -# coverage. Go's own -timeout bounds the tests and not their compile, -# so a cold build cache cannot fail it. The race detector needs cgo, -# and so a C compiler. If the tests fail, they run again with -v for -# the details, and the script fails even if that run passes. +# script/test: run the backend test suite on the host with the race +# detector and coverage. The root make test runs the same in the test +# phase of the root Dockerfile. Go's own -timeout bounds the tests and +# not their compile, so a cold build cache cannot fail it. -count=1 +# keeps Go's test result cache out of both runs, so neither can report +# a stored pass. The race detector needs cgo, and so a C compiler. If +# the tests fail, they run again with -v for the details, and the +# script fails even if that run passes. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - go test -timeout 30s -race -cover ./... || { + go test -count=1 -timeout 90s -race -cover ./... || { echo "--- Rerunning with -v for details ---" - go test -timeout 30s -race -v ./... + go test -count=1 -timeout 90s -race -v ./... exit 1 } } diff --git a/eslint.config.js b/eslint.config.js index 2aab933..1a2d756 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,5 +1,5 @@ // eslint's recommended rules over every JavaScript file in the repo. -// script/frontend-lint runs it, in the frontend-lint stage of Dockerfile. +// make lint runs it, in the frontend-lint stage of Dockerfile. import js from "@eslint/js"; import globals from "globals"; import { defineConfig } from "eslint/config"; diff --git a/script/bootstrap b/script/bootstrap index d708c81..00494d3 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -15,8 +15,8 @@ # hook find it once that directory is on PATH. Nothing in ~/.local/bin # that this script did not create is ever replaced. # -# golangci-lint is not installed: make lint runs it in Docker, which -# this script does not install either. +# golangci-lint is not installed: make lint runs it in Docker, as make +# test runs the tests, and this script does not install Docker either. # # Unlike the org model: Go and gcc for backend/, a newer node for eslint. set -eu @@ -256,9 +256,9 @@ main() { if missing make; then pkg_install gnumake make make make; fi if missing git; then pkg_install git git git git; fi - # The race detector in make test needs cgo, which Go turns on only - # when it finds its C compiler, gcc on Linux. apt and apk ship the C - # library headers apart from gcc. + # The race detector in backend/'s make test needs cgo, which Go turns + # on only when it finds its C compiler, gcc on Linux. apt and apk + # ship the C library headers apart from gcc. if missing gcc; then pkg_install gcc "gcc libc6-dev" gcc "gcc musl-dev" fi @@ -271,8 +271,8 @@ main() { (cd "$ROOT/backend" && go mod download) if missing docker; then - echo "bootstrap: docker not found; make lint, and so make check" >&2 - echo " and the pre-commit hook, need it to run the linters" >&2 + echo "bootstrap: docker not found; make test and make lint, and so" >&2 + echo " make check and the pre-commit hook, need it" >&2 fi if [ -n "$path_hint" ] && [ -d "$BIN_DIR" ]; then echo "bootstrap: add $BIN_DIR to the front of your PATH, e.g." >&2 diff --git a/script/check b/script/check index 3e1778c..92875f7 100755 --- a/script/check +++ b/script/check @@ -1,6 +1,8 @@ #!/bin/sh # script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" diff --git a/script/fmt b/script/fmt index 0b3eb52..c18d300 100755 --- a/script/fmt +++ b/script/fmt @@ -1,6 +1,6 @@ #!/bin/sh -# script/fmt: format the whole repo (writes): prettier over everything -# it understands, then gofmt over the Go backend. +# script/fmt: format the whole repo (writes): prettier over the +# JavaScript, CSS, HTML and Markdown, then gofmt over the Go backend. # The org model formats only markdown; this repo also has JS and Go. set -eu @@ -8,6 +8,9 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" + # script/bootstrap links the node, yarn and gofmt it installs into + # ~/.local/bin, which the shell that called it may not have on PATH. + PATH="$HOME/.local/bin:$PATH" "$ROOT/script/frontend-fmt" "$ROOT/backend/script/fmt" } diff --git a/script/fmt-check b/script/fmt-check index bee557b..72788e7 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -8,6 +8,9 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" + # script/bootstrap links the node, yarn and gofmt it installs into + # ~/.local/bin, which the shell that called it may not have on PATH. + PATH="$HOME/.local/bin:$PATH" "$ROOT/script/frontend-fmt-check" "$ROOT/backend/script/fmt-check" } diff --git a/script/frontend-check b/script/frontend-check deleted file mode 100755 index fad36b0..0000000 --- a/script/frontend-check +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -# script/frontend-check: run the frontend tests and format check only. -# This exists for the frontend stage of Dockerfile, a node image with -# neither Go nor Docker; the Dockerfile's frontend-lint stage runs the -# frontend linter, and its lint and builder stages gate the backend. -# Everywhere else, use script/check, which covers the whole repo. Must -# not modify any files. -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - "$ROOT/script/frontend-test" - "$ROOT/script/frontend-fmt-check" -} - -main "$@" diff --git a/script/frontend-fmt b/script/frontend-fmt index 523497a..a16bcd9 100755 --- a/script/frontend-fmt +++ b/script/frontend-fmt @@ -1,6 +1,8 @@ #!/bin/sh -# script/frontend-fmt: format the frontend and every other file prettier -# understands, repo-wide (writes), the markdown in backend/ included. +# script/frontend-fmt: format the JavaScript, CSS, HTML and Markdown, +# repo-wide (writes), the markdown in backend/ included. Those are the +# languages REPO_POLICIES.md gives prettier; the YAML is left alone, as +# backend/.golangci.yml must stay the org standard byte for byte. # Prettier does not read Go; backend/script/fmt formats the Go sources. set -eu @@ -8,7 +10,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - yarn prettier --write . + yarn prettier --write '**/*.{js,css,html,md}' } main "$@" diff --git a/script/frontend-fmt-check b/script/frontend-fmt-check index 1501fce..7cc718a 100755 --- a/script/frontend-fmt-check +++ b/script/frontend-fmt-check @@ -7,7 +7,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - yarn prettier --check . + yarn prettier --check '**/*.{js,css,html,md}' } main "$@" diff --git a/script/frontend-lint b/script/frontend-lint deleted file mode 100755 index caa01d4..0000000 --- a/script/frontend-lint +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# script/frontend-lint: run eslint over the frontend. This runs inside -# the frontend-lint stage of Dockerfile, the digest-pinned node image -# with the packages from yarn.lock. From a checkout, run `make lint`, -# which builds that stage. -set -eu - -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -main() { - cd "$ROOT" - yarn eslint . -} - -main "$@" diff --git a/script/frontend-test b/script/frontend-test index bf34e69..5a8c965 100755 --- a/script/frontend-test +++ b/script/frontend-test @@ -1,7 +1,8 @@ #!/bin/sh -# script/frontend-test: run the frontend test suite: the unit tests in -# test/unit/, through the test script in package.json, then the -# production build, which fails on broken code. The tests print a dot +# script/frontend-test: run the frontend test suite on the host: the +# unit tests in test/unit/, through the test script in package.json, +# then the production build, which fails on broken code. make test runs +# the same in the frontend stage of Dockerfile. The tests print a dot # each; if any fails, they run again with every test listed, and the # script fails even if that run passes. NODE_OPTIONS chooses the # reporter because yarn adds its arguments after the test files, where @@ -12,9 +13,9 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - NODE_OPTIONS=--test-reporter=dot timeout 30 yarn --silent run test || { + NODE_OPTIONS=--test-reporter=dot timeout 90 yarn --silent run test || { echo "--- Rerunning with every test listed for details ---" - NODE_OPTIONS=--test-reporter=spec timeout 30 yarn --silent run test + NODE_OPTIONS=--test-reporter=spec timeout 90 yarn --silent run test exit 1 } timeout 30 yarn build diff --git a/script/lint b/script/lint index d0192a2..2d8b075 100755 --- a/script/lint +++ b/script/lint @@ -1,23 +1,23 @@ #!/bin/sh -# script/lint: lint the whole repo: eslint over the frontend, then the Go -# linter over backend/. +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. # -# No linter runs on the host: this builds the frontend-lint and lint -# stages of Dockerfile, the digest-pinned node and golangci-lint images. -# The first runs eslint; the second runs the backend's fmt-check and -# lint targets. --no-cache makes each linter really run every time -# rather than reuse an earlier result, and each stage is built for its -# checks alone, so no image is kept. +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - timeout 300 docker build --no-cache --target frontend-lint \ - --output type=cacheonly . - timeout 300 docker build --no-cache --target lint \ - --output type=cacheonly . + docker build --no-cache \ + --target lint \ + -t "$("$SCRIPT_DIR/projectname")-lint" . } main "$@" diff --git a/script/test b/script/test index d48d3ca..cd239f2 100755 --- a/script/test +++ b/script/test @@ -1,17 +1,19 @@ #!/bin/sh -# script/test: run the test suite for the whole repo: the frontend at -# the repo root, then the Go backend in backend/. Each half has its own -# 30-second limit, and there is none around both: from a cold Go build -# cache, compiling the backend's tests with the race detector can take -# 30 seconds on its own, and Go's -timeout leaves the compile out. +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - script/frontend-test - backend/script/test + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . } main "$@" diff --git a/test/viewport/README.md b/test/viewport/README.md index 8e89805..8d7f2de 100644 --- a/test/viewport/README.md +++ b/test/viewport/README.md @@ -13,8 +13,8 @@ width derived from the app's own CSS. Screenshots land in `tmp/viewport/` alongside a `results.json`; they are artifacts for a human to look at when something fails, not the evidence. The assertions are the evidence. -The target is deliberately outside `make check`: it needs Docker and takes -minutes, and `make test` has to stay under 20 seconds. +The target is deliberately outside `make check`: it takes minutes, and +`make test` has to stay under 60 seconds. ## How the widths are chosen