diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 06b80af..08c2ebc 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,5 +6,7 @@ jobs: steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - # script/cibuild builds the image, whose stages run every check. - - run: script/cibuild + # script/cibuild bootstraps, runs every check and builds the + # image. script/bootstrap links what it installs into + # ~/.local/bin, so that has to be on PATH for the rest. + - run: PATH="$HOME/.local/bin:$PATH" script/cibuild diff --git a/Dockerfile b/Dockerfile index 417fc97..d13e53e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,8 +77,9 @@ COPY --from=frontend /app/dist /usr/share/nginx/html COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh +# bin/entrypoint.sh creates DATA_DIR at start and gives it and /data to +# the netwatch user, whatever is mounted there. ENV DATA_DIR=/data/reports -RUN mkdir -p /data/reports && chown -R netwatch:netwatch /data VOLUME /data # The default public port; PORT changes it. diff --git a/README.md b/README.md index 415457e..acbb62d 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,12 @@ The Go backend in `backend/` has its own `script/` directory and shim Makefile (see [backend/README.md](backend/README.md)). The root scripts cover both halves, so the root `make check` fails if either one is broken. We provide: -- `script/bootstrap` — install all dependencies (pinned node via nvm if needed, - yarn via corepack, `yarn install --frozen-lockfile`, the pinned Go unless one - at least as new as `backend/go.mod` asks for is installed, and the Go - modules), linking what it installs itself into `~/.local/bin`, which has to be - on `PATH`. It installs no Go linter and not Docker: `make lint` runs the - linter in Docker +- `script/bootstrap` — install all dependencies (the pinned node via nvm unless + one new enough for the frontend's dependencies is installed, yarn via + corepack, `yarn install --frozen-lockfile`, the pinned Go unless one at least + as new as `backend/go.mod` asks for is installed, and the Go modules), linking + what it installs itself into `~/.local/bin`, which has to be on `PATH`. It + installs no Go linter and not Docker: `make lint` runs the linter in Docker - `script/setup` — make a fresh clone ready for development: bootstrap plus the git pre-commit hook - `script/projectname` — print the project name (used for the Docker image tag) @@ -65,7 +65,8 @@ halves, so the root `make check` fails if either one is broken. We provide: `script/check`: it needs Docker and takes minutes. - `script/docker` — build the image from `Dockerfile` without the build cache, tagged `netwatch` via `script/projectname` -- `script/cibuild` — CI entrypoint: builds the image +- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`, + then builds the image as `script/docker` does, without the build cache - `script/precommit` — run by the git pre-commit hook; runs `script/check` - `script/install-precommit` — install the git pre-commit hook @@ -191,8 +192,9 @@ only inside the container, on `127.0.0.1:8081`. The image: - Sends the security headers `REPO_POLICIES.md` requires on every response, as `security-headers.conf` sets them, in place of the backend's own - Stores reports in `DATA_DIR`, `/data/reports` by default, on the `/data` - volume. The backend runs as user `netwatch` (uid 1000), so a directory - bind-mounted at `/data` must be writable by uid 1000 + volume. Before the backend starts, the image creates `DATA_DIR` and gives it + and `/data` to user `netwatch` (uid 1000), which the backend runs as, so a + host directory bind-mounted at `/data` ends up owned by uid 1000 - Writes buffered reports to disk on `docker stop`, and exits non-zero if nginx or the backend exits on its own, so the platform restarts it @@ -202,16 +204,6 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app for netwatch needs: - **Port:** container port `8080`. - **Volume:** container path `/data`; the reports are kept in `/data/reports`. -- **First run:** upaas bind-mounts the host directory it is given and does not - create it, and the backend, which runs as uid 1000, does not start unless it - can write there. Create the directory, owned by uid 1000, before the first - deploy: - - ```bash - mkdir -p /path/to/data - chown 1000:1000 /path/to/data - ``` - - **Environment variables:** none is required. An empty one counts as unset, and one set to a value netwatch cannot use stops the container at start, with the reason in its log. diff --git a/TODO.md b/TODO.md index e37242e..8b17dd2 100644 --- a/TODO.md +++ b/TODO.md @@ -23,6 +23,30 @@ latest run passes. # Completed Steps +- 2026-09-29: the container sets up its own data directory (issue #75): + `bin/entrypoint.sh`, still as root, creates `DATA_DIR` if missing and gives it + and `/data` to the `netwatch` user with mode 750 before starting the backend + as that user, so an empty host directory owned by root, or one holding files + from another uid, works with no step on the host. It stops the start instead + when a symbolic link is on the path to `DATA_DIR`, since root would change + whatever the link points to. The `README.md` first-run step that created and + chowned the host directory is gone, and the image no longer sets that + ownership at build time +- 2026-09-29: CI can no longer pass on checks that did not run (issue #37): + `script/cibuild` is now the org model, byte for byte. It runs + `script/bootstrap` and `script/check`, then builds the image with `--no-cache` + and the version from `git describe` as the `VERSION` build argument, where it + used to be a plain `docker build .` whose check steps could come from the + build cache. The workflow puts `~/.local/bin`, where bootstrap links what it + installs, on the step's `PATH`, and bootstrap now installs its pinned node + when the installed one is older than the frontend's dependencies need +- 2026-09-29: `backend/.golangci.yml` re-vendored from `sneak/prompts` (issue + #41): `gomodguard`, deprecated in golangci-lint v2.12.0, is disabled and its + successor `gomodguard_v2` enabled with the org block list, so lint runs print + no deprecation warning. The new file also turns `depguard` on with its + `test-support` rule, which keeps `net/http/httptest` out of non-test code; + netwatch adds no entries of its own to that rule. `backend/script/lint` checks + the new sha256 - 2026-09-29: nginx sends the security headers `REPO_POLICIES.md` requires on every response (issue #18), including errors, `/assets/` and what it passes on from the backend, whose own copies it drops so each header goes out once. They @@ -191,9 +215,3 @@ latest run passes. (main always green policy) - Decide what to do with untracked resume.sh: commit it, gitignore it, or delete it -- Upstream fix needed in `sneak/prompts`: the org-standard `.golangci.yml` - enables `gomodguard`, which golangci-lint v2.12.2 reports as deprecated since - v2.12.0 and replaced by `gomodguard_v2`, so every backend lint run prints a - deprecation warning. The file is standardized and must never be edited in this - repo, so nothing can be done here beyond tracking it — tracked at - diff --git a/backend/.golangci.yml b/backend/.golangci.yml index 26b1610..a7a74c2 100644 --- a/backend/.golangci.yml +++ b/backend/.golangci.yml @@ -10,14 +10,20 @@ run: linters: default: all + enable: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields - - depguard # Dependency allow/block lists - godot # Requires comments to end with periods - - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 settings: lll: line-length: 88 @@ -28,6 +34,64 @@ linters: max-complexity: 15 dupl: threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. + rules: + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 diff --git a/backend/README.md b/backend/README.md index ad29c08..9914eb0 100644 --- a/backend/README.md +++ b/backend/README.md @@ -104,7 +104,8 @@ this server. The image's entrypoint, `bin/entrypoint.sh`, starts the server as user `netwatch` (uid 1000) with `BIND_ADDRESS=127.0.0.1` and `PORT=8081`, so only nginx reaches it, and with `TRUSTED_PROXIES=127.0.0.1/32`, so it takes the client address nginx passes on and no other. `DATA_DIR` is `/data/reports`, on -the `/data` volume, which `netwatch` owns. nginx replaces the security headers +the `/data` volume; the entrypoint creates it and gives it and `/data` to +`netwatch` before starting the server. nginx replaces the security headers this server sets with those in the root `security-headers.conf`, so those are what clients of the image see. diff --git a/backend/script/lint b/backend/script/lint index 256f805..720b747 100755 --- a/backend/script/lint +++ b/backend/script/lint @@ -14,7 +14,7 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -GOLANGCI_CONFIG_SHA256="021cc83f4e6fc7c31b95b34b846723dfcf20b66b7baeea1dc40406e643346bcb" +GOLANGCI_CONFIG_SHA256="a79b63a254602a5318db5d0e9a06bc71b84bf0c1d896305229d8bfed1d1b1776" main() { cd "$ROOT" diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 4458f9e..6f3bdf8 100755 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -61,6 +61,29 @@ for proxy in $(printf '%s' "$TRUSTED_PROXIES" | tr ',' ' '); do echo "set_real_ip_from $cidr;" done > /etc/nginx/trusted-proxies.conf +# netwatch-server keeps its report files in DATA_DIR, on the /data +# volume, which may be a host directory owned by root or by another +# uid. Both are given to the netwatch user here, with the mode the +# server gives a directory it creates, so the host directory needs no +# preparing. +# +# chown and chmod, run as root, change whatever a symbolic link on the +# path points to, anywhere in the container, and the netwatch user can +# put one in /data. So the start stops unless readlink -f, which +# follows every link on a path, gives /data and DATA_DIR back as they +# are. It also writes a path in full, so a DATA_DIR with '.', '..' or +# an extra '/' in it is refused too. +export DATA_DIR="${DATA_DIR:-/data/reports}" +mkdir -p "$DATA_DIR" || exit 1 +if [ "$(readlink -f /data)" != /data ] || + [ "$(readlink -f "$DATA_DIR")" != "$DATA_DIR" ]; then + echo "entrypoint: DATA_DIR must be a full path with no '.', '..'," \ + "extra '/' or symbolic link on it or on /data, not '$DATA_DIR'" >&2 + exit 1 +fi +chown -R netwatch:netwatch /data "$DATA_DIR" || exit 1 +chmod 750 /data "$DATA_DIR" || exit 1 + # A stop signal is only noted here; the loop below acts on it. stop_requested="" trap 'stop_requested=yes' TERM INT diff --git a/script/bootstrap b/script/bootstrap index 66e5873..d807ee3 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,12 +3,12 @@ # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes nothing is present. Node is -# used directly if installed; otherwise it is installed at a pinned -# version via nvm (installing nvm itself first, from a hash-verified -# release archive, never curl | sh). Go, with its gofmt, is used -# directly if it is at least the version backend/go.mod asks for; -# otherwise the pinned Go release is installed from its hash-verified -# archive. +# used directly if it is at least NODE_MIN_VERSION; otherwise it is +# installed at a pinned version via nvm (installing nvm itself first, +# from a hash-verified release archive, never curl | sh). Go, with its +# gofmt, is used directly if it is at least the version backend/go.mod +# asks for; otherwise the pinned Go release is installed from its +# hash-verified archive. # # What this script installs outside the system package manager lives # under $HOME and is linked into ~/.local/bin, where make and the git @@ -23,6 +23,10 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" # Pinned versions, 2026-07-07 NODE_VERSION="22.17.0" +# The oldest node the frontend's dependencies accept: the "engines" +# field of puppeteer-core 25.5.0, the most demanding of them, asks for +# 22.12.0 or newer, 2026-09-29. An older installed node is not used. +NODE_MIN_VERSION="22.12.0" NVM_VERSION="0.40.3" # sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" @@ -136,8 +140,22 @@ ensure_nvm() { rm -rf "$tmp" } +# node_ok: the node on PATH is at least NODE_MIN_VERSION. node itself +# compares the two: major, then minor, then patch. +node_ok() { + if missing node; then return 1; fi + node -e ' + const have = process.versions.node.split(".").map(Number); + const want = process.argv[1].split(".").map(Number); + for (let i = 0; i < 3; i++) { + if (have[i] !== want[i]) process.exit(have[i] > want[i] ? 0 : 1); + } + ' "$NODE_MIN_VERSION" +} + +# ensure_node: unless node_ok, install NODE_VERSION and link its node. ensure_node() { - if ! missing node; then return 0; fi + if node_ok; then return 0; fi ensure_nvm nvm_sh "nvm install $NODE_VERSION" link_bin "$HOME/.nvm/versions/node/v$NODE_VERSION/bin/node" node diff --git a/script/cibuild b/script/cibuild index d860bf5..688299f 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,15 +1,29 @@ #!/bin/sh -# script/cibuild: run the CI build: build the one image from Dockerfile, -# whose stages run the checks as build steps (the backend's fmt-check, -# lint and tests, and the frontend's test, lint and fmt-check). This is -# the only build step the Gitea workflow runs. +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - timeout 300 docker build . + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@"