From e45bc578b27d74724350041b7fb87d080ce16be0 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 7 Jul 2026 02:14:16 +0200 Subject: [PATCH] scripts-to-rule-them-all (#10) Reviewed-on: https://git.eeqj.de/sneak/netwatch/pulls/10 Co-authored-by: sneak Co-committed-by: sneak --- .gitea/workflows/check.yml | 2 +- Makefile | 28 ++- README.md | 23 ++ REPO_POLICIES.md | 468 ++++++++++++++++++++++++++++++------- TODO.md | 64 ++--- script/bootstrap | 143 ++++++++++++ script/check | 14 ++ script/cibuild | 13 ++ script/docker | 14 ++ script/fmt | 12 + script/fmt-check | 13 ++ script/install-precommit | 16 ++ script/lint | 12 + script/precommit | 12 + script/projectname | 12 + script/setup | 13 ++ script/test | 13 ++ 17 files changed, 748 insertions(+), 124 deletions(-) create mode 100755 script/bootstrap create mode 100755 script/check create mode 100755 script/cibuild create mode 100755 script/docker create mode 100755 script/fmt create mode 100755 script/fmt-check create mode 100755 script/install-precommit create mode 100755 script/lint create mode 100755 script/precommit create mode 100755 script/projectname create mode 100755 script/setup create mode 100755 script/test diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 81ac7d9..860d9dd 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,5 +6,5 @@ jobs: steps: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - run: docker build . + - run: script/cibuild - run: docker build -f Dockerfile.backend . diff --git a/Makefile b/Makefile index a269a68..353f708 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,35 @@ -.PHONY: dev test lint fmt fmt-check check docker +.PHONY: bootstrap setup dev test lint fmt fmt-check check docker hooks + +# Standard targets are thin shims; the implementations live in script/ +# per the scripts-to-rule-them-all pattern (see the Entrypoints section +# of README.md). + +bootstrap: + @script/bootstrap + +setup: + @script/setup dev: yarn dev test: - timeout 30 yarn build + @script/test lint: - yarn prettier --check . + @script/lint fmt: - yarn prettier --write . + @script/fmt fmt-check: - yarn prettier --check . + @script/fmt-check -check: test lint fmt-check +check: + @script/check docker: - timeout 300 docker build -t netwatch . + @script/docker + +hooks: + @script/install-precommit diff --git a/README.md b/README.md index a9eb122..d4dbdd4 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,29 @@ docker build -t netwatch . docker run -p 8080:8080 netwatch ``` +## Entrypoints + +This repository adheres to the +[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) +standard: normalized scripts in `script/` are the entrypoints for the +development workflow, and the Makefile targets are thin shims that call them. We +provide: + +- `script/bootstrap` — install all dependencies (pinned node via nvm if needed, + yarn via corepack, `yarn install --frozen-lockfile`) +- `script/setup` — make a fresh clone ready for development: bootstrap plus the + git pre-commit hook +- `script/projectname` — print the project name (used for the Docker image tag) +- `script/test` — run the production build as the test (no unit tests yet) +- `script/lint` — run prettier in check mode +- `script/fmt` — format all files (writes) +- `script/fmt-check` — check formatting (read-only) +- `script/check` — run test, lint, and fmt-check +- `script/docker` — build the Docker image tagged via `script/projectname` +- `script/cibuild` — CI entrypoint: plain `docker build .` +- `script/precommit` — run by the git pre-commit hook; runs `script/check` +- `script/install-precommit` — install the git pre-commit hook + ## Rationale When debugging network issues, it's useful to have a persistent at-a-glance view diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 2d40cb4..bc2f161 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,108 +1,408 @@ -# Development Policies +--- +title: Repository Policies +last_modified: 2026-07-06 +--- -- Docker image references by tag are server-mutable, therefore using them is an - RCE vulnerability. All docker image references must use cryptographic hashes - to securely specify the exact image that is expected. +This document covers repository structure, tooling, and workflow standards. Code +style conventions are in separate documents: -- Correspondingly, `go install` commands using things like '@latest' are also - dangerous RCE. Whenever writing scripts or tools, ALWAYS specify go install - targets using commit hashes which are cryptographically secure. +- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md) + (general, bash, Docker) +- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md) +- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md) +- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md) +- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md) -- Every repo with software in it must have a Makefile in the root. Each such - Makefile should support `make test` (runs the project-specific tests), - `make lint`, `make fmt` (writes), `make fmt-check` (readonly), and - `make check` (has `test`, `lint`, and `fmt-check` as prereqs), `make docker` - (builds docker image). +--- -- Every repo should have a Dockerfile. If the repo contains non-server software, - the Dockerfile should bring up a development environment and `make check` - (i.e. the docker build should fail if the branch is not green). +- Cross-project documentation (such as this file) must include + `last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync + with the authoritative source as policies evolve. -- Platform-specific standard formatting should be used. `black` for python, - `prettier` for js/css/etc, `go fmt` for go. The only changes to default - settings should be to specify four-space indents where applicable (i.e. - everything except `go fmt`). +- **ALL external references must be pinned by cryptographic hash.** This + includes Docker base images, Go modules, npm packages, GitHub Actions, and + anything else fetched from a remote source. Version tags (`@v4`, `@latest`, + `:3.21`, etc.) are server-mutable and therefore remote code execution + vulnerabilities. The ONLY acceptable way to reference an external dependency + is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm + integrity hash in lockfile, GitHub Actions `@`). No exceptions. + This also means never `curl | bash` to install tools like pyenv, nvm, rustup, + etc. Instead, download a specific release archive from GitHub, verify its hash + (hardcoded in the Dockerfile or script), and only then install. Unverified + install scripts are arbitrary remote code execution. This is the single most + important rule in this document. Double-check every external reference in + every file before committing. There are zero exceptions to this rule. -- If local testing is possible (it is not always), `make check` should be a - pre-commit hook. If it is not possible, `make lint && make fmt-check` should - be a pre-commit hook. +- Every repo with software must have a root `Makefile` with these targets: + `make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes), + `make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`), + `make docker`, and `make hooks` (installs pre-commit hook). A model Makefile + is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`. -- If a working `make test` takes more than 20 seconds, that's a bug that needs - fixing. In fact, there should be a timeout specified in the `Makefile` that - fails it automatically if it takes >30s. +- Repos follow the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + pattern: the implementation of each Makefile target lives in an executable + script in `script/` (`script/bootstrap`, `script/setup`, `script/test`, + `script/lint`, `script/fmt`, `script/fmt-check`, `script/check`, + `script/docker`), and the Makefile targets are thin shims that call them. The + scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in + minimal containers (e.g. alpine images have no bash); locate the repo root + with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From + the standard's canonical set we use `bootstrap`, `setup` (make the repo ready + for development after a fresh clone: runs `bootstrap`, then + `install-precommit`, plus any repo-specific initialization), `test`, and + `cibuild`. `script/bootstrap` installs all dependencies idempotently and + assumes nothing is present: base tools come from nix, apt, brew, or apk + (detected in that order; apt runs noninteractive). For node it uses the + installed node if present; otherwise it installs a PINNED node version via + nvm, first installing nvm itself if missing — from a hash-verified GitHub + release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. yarn is then pinned via + `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; + always exact versions. `script/cibuild` runs the CI build: it changes to the + repo root and runs `docker build .`; the Gitea workflow calls it. Four further + scripts are our own extensions to the standard: `script/check` runs + `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is + what the git pre-commit hook runs, and it calls `script/check`; + `script/install-precommit` installs the git pre-commit hook (the `make hooks` + target shims to it); and `script/projectname` (literally that filename) simply + outputs the project's name. Scripts that need the name call + `script/projectname` — e.g. `script/docker` assembles its image tag from it — + so those scripts stay byte-identical across all repos. Repo-type-specific + pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in + `script/precommit`, not in the hook itself. Model scripts are at + `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README + must document the provided scripts in an **Entrypoints** section (see the + README requirements below). -- Docker builds should time out in 5 minutes or less. +- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.) + instead of invoking the underlying tools directly. The Makefile is the single + source of truth for how these operations are run. + +- The Makefile is authoritative documentation for how the repo is used. Beyond + the required targets above, it should have targets for every common operation: + running a local development server (`make run`, `make dev`), re-initializing + or migrating the database (`make db-reset`, `make migrate`), building + artifacts (`make build`), generating code, seeding data, or anything else a + developer would do regularly. If someone checks out the repo and types + `make`, they should see every meaningful operation available. A new + contributor should be able to understand the entire development workflow by + reading the Makefile. + +- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` + as a build step so the build fails if the branch is not green. For non-server + repos, the Dockerfile should bring up a development environment and run + `make check`. For server repos, `make check` should run as an early build + stage before the final image is assembled. Dockerfiles install development + prerequisites by running `script/bootstrap` rather than duplicating installs + inline; COPY `script/` and the dependency manifests (`package.json` + + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap + layer stays cached until dependencies change. + +- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go + repos use a multistage build where linting runs in an independent stage based + on the `golangci/golangci-lint` image (pinned by hash). This stage runs + `make fmt-check` and `make lint` before the full build begins. The build stage + then declares an explicit dependency on the lint stage via + `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete + linting before proceeding to compilation and tests. This ensures lint failures + surface in seconds rather than minutes, without blocking on dependency + download or compilation in the build stage. + + The standard pattern for a Go repo Dockerfile is: + + ```dockerfile + # Lint stage — fast feedback on formatting and lint issues + # golangci/golangci-lint:v2.x.x, YYYY-MM-DD + FROM golangci/golangci-lint@sha256:... AS lint + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN make fmt-check + RUN make lint + + # Build stage + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + WORKDIR /src + + # Force BuildKit to run the lint stage before proceeding + COPY --from=lint /src/go.sum /dev/null + + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN make test + + ARG VERSION=dev + RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage + FROM alpine@sha256:... + COPY --from=builder /app /usr/local/bin/app + ENTRYPOINT ["app"] + ``` + + Key points: + - The lint stage uses the `golangci/golangci-lint` image directly (it + includes both Go and the linter), so there is no need to install the + linter separately. + - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates + a stage dependency. BuildKit runs stages in parallel by default; without + this line, the build stage would not wait for lint to finish and a lint + failure might not fail the overall build. + - If the project uses `//go:embed` directives that reference build artifacts + (e.g. a web frontend compiled in a separate stage), the lint stage must + create placeholder files so the embed directives resolve. Example: + `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. + The lint stage should not depend on the actual build output — it exists to + fail fast. + - If the project requires CGO or system libraries for linting (e.g. + `vips-dev`), install them in the lint stage with `apk add`. + - The build stage runs `make test` after compilation setup. Tests run in the + build stage, not the lint stage, because they may require compiled + artifacts or heavier dependencies. + +- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that + runs `script/cibuild` (which runs `docker build .`) on push. Since the + Dockerfile already runs `make check`, a successful build implies all checks + pass. + +- Use platform-standard formatters: `black` for Python, `prettier` for + JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with + two exceptions: four-space indents (except Go), and `proseWrap: always` for + Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown, + HTML, CSS) should also have `.prettierrc` and `.prettierignore`. + +- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local + testing is not possible in the repo, `script/precommit` may skip `script/test` + and run only `script/lint` and `script/fmt-check`. The hook is installed by + `script/install-precommit`; the Makefile must provide a `make hooks` target + that shims to it. + +- All repos with software must have tests that run via the platform-standard + test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful + tests exist yet, add the most minimal test possible — e.g. importing the + module under test to verify it compiles/parses. There is no excuse for + `make test` to be a no-op. + +- `make test` must complete in under 20 seconds. Add a 30-second timeout in the + Makefile. + +- **`make test` should use the conditional verbose rerun pattern.** Run tests + without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to + show full output. This keeps CI logs and `docker build` output clean on + success (just package/suite summaries) while providing full diagnostic detail + on failure (every test case, every assertion). The general shell pattern: + + ```makefile + test: + @ || \ + { echo "--- Rerunning with -v for details ---"; \ + ; exit 1; } + ``` + + Go example: + + ```makefile + test: + @go test -timeout 30s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 30s -race -v ./...; exit 1; } + ``` + + Python example: + + ```makefile + test: + @python -m pytest || \ + { echo "--- Rerunning with -v for details ---"; \ + python -m pytest -v; exit 1; } + ``` + + The `exit 1` ensures the target always fails after a rerun — the first run + already proved the tests are broken, so the build must not pass even if a + flaky test happens to succeed on the second attempt. The rerun exists solely + for diagnostic output. + +- Docker builds must complete in under 5 minutes. + +- `make check` must not modify any files in the repo. Tests may use temporary + directories. - `main` must always pass `make check`, no exceptions. -- Do all changes on a feature branch. You can do whatever you want on a feature - branch. +- Never commit secrets. `.env` files, credentials, API keys, and private keys + must be in `.gitignore`. No exceptions. -- We have a standardized `.golangci.yml` which we reuse and is _NEVER_ to be - modified by an agent, only manually by the user. It can be copied from - `~/dev/upaas/.golangci.yml` if it exists at that location. +- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), + editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. + Fetch the standard `.gitignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up + a new repo. -- When specifying images or packages by hash in Dockerfiles or - `docker-compose.yml`, put a comment above the line and show the version and - date at which it was current. +- **No build artifacts in version control.** Code-derived data (compiled + bundles, minified output, generated assets) must never be committed to the + repository if it can be avoided. The build process (e.g. Dockerfile, Makefile) + should generate these at build time. Notable exception: Go protobuf generated + files (`.pb.go`) ARE committed because repos need to work with `go get`, which + downloads code but does not execute code generation. -- For javascript, always use `yarn` over `npm`. +- Never use `git add -A` or `git add .`. Always stage files explicitly by name. -- Whenever writing dates, ALWAYS write YYYY-MM-DD (ISO 8601). +- Never force-push to `main`. -- Simple projects should be configured with environment variables, as is - standard for Dockerized applications. +- Make all changes on a feature branch. You can do whatever you want on a + feature branch. -- Dockerized web services should listen on the default HTTP port of 8080 unless - overridden with the `PORT` environment variable. +- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only + manually by the user. Fetch from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. -- The `README.md` is a project's primary documentation. It should contain at a - minimum the following sections: - - Description - - Include a short and complete description of the functionality and - purpose of the software as the first line in the readme. It must - include: - - the name - - the purpose - - the category (web server, SPA, command line tool, etc) - - the license - - the author - - eg: "µPaaS is an MIT-licensed Go web application by @sneak that - receives git-frontend webhooks and interacts with a Docker server - to build and deploy applications in realtime as certain branches - are updated." - - Getting Started - - a code block with copy-pasteable installation/use sections - - Rationale - - why does this exist? - - Design - - how is the program structured? - - TODO - - This is your TODO list for the project - update it meticulously, even - in between commits. Whenever planning, put your todo list in the - README so that a separate agent with new context can pick up where you - left off. - - License - - GPL or MIT or WTFPL - ask the user when beginning a new project and - include a LICENSE file in the root and in a section in the README. - - Author - - @sneak (link `@sneak` to `https://sneak.berlin`). +- When pinning images or packages by hash, add a comment above the reference + with the version and date (YYYY-MM-DD). -- When beginning a new project, initialize a git repo and make the first commit - simply the first version of the README.md in the root of the repo. +- Use `yarn`, not `npm`. -- For Go packages, the module root is `sneak.berlin/go/...`, such as - `sneak.berlin/go/dnswatcher`. +- Write all dates as YYYY-MM-DD (ISO 8601). -- We use SemVer always. +- Simple projects should be configured with environment variables. -- If no tag `1.0.0` or greater exists in the repository, modify the existing - migrations and assume no installed base or existing databases. If `>=1.0.0`, - database changes add new migration files. +- Dockerized web services listen on port 8080 by default, overridable with + `PORT`. -- New repos must have at a minimum the following files: - - `README.md`, `.git`, `.gitignore` - - `POLICIES.md` (copy from `~/Documents/_PROMPTS/POLICIES.md`) +- **HTTP/web services must be hardened for production internet exposure before + tagging 1.0.** This means full compliance with security best practices + including, without limitation, all of the following: + - **Security headers** on every response: + - `Strict-Transport-Security` (HSTS) with `max-age` of at least one year + and `includeSubDomains`. + - `Content-Security-Policy` (CSP) with a restrictive default policy + (`default-src 'self'` as a baseline, tightened per-resource as + needed). Never use `unsafe-inline` or `unsafe-eval` unless + unavoidable, and document the reason. + - `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required). + Prefer the `frame-ancestors` CSP directive as the primary control. + - `X-Content-Type-Options: nosniff`. + - `Referrer-Policy: strict-origin-when-cross-origin` (or stricter). + - `Permissions-Policy` restricting access to browser features the + application does not use (camera, microphone, geolocation, etc.). + - **Request and response limits:** + - Maximum request body size enforced on all endpoints (e.g. Go + `http.MaxBytesReader`). Choose a sane default per-route; never accept + unbounded input. + - Maximum response body size where applicable (e.g. paginated APIs). + - `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend + against slowloris attacks. + - `WriteTimeout` on the `http.Server`. + - `IdleTimeout` on the `http.Server`. + - Per-handler execution time limits via `context.WithTimeout` or + chi/stdlib `middleware.Timeout`. + - **Authentication and session security:** + - Rate limiting on password-based authentication endpoints. API keys are + high-entropy and not susceptible to brute force, so they are exempt. + - CSRF tokens on all state-mutating HTML forms. API endpoints + authenticated via `Authorization` header (Bearer token, API key) are + exempt because the browser does not attach these automatically. + - Passwords stored using bcrypt, scrypt, or argon2 — never plain-text, + MD5, or SHA. + - Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or + `Strict`) attributes. + - **Reverse proxy awareness:** + - True client IP detection when behind a reverse proxy + (`X-Forwarded-For`, `X-Real-IP`). The application must accept + forwarded headers only from a configured set of trusted proxy + addresses — never trust `X-Forwarded-For` unconditionally. + - **CORS:** + - Authenticated endpoints must restrict `Access-Control-Allow-Origin` to + an explicit allowlist of known origins. Wildcard (`*`) is acceptable + only for public, unauthenticated read-only APIs. + - **Error handling:** + - Internal errors must never leak stack traces, SQL queries, file paths, + or other implementation details to the client. Return generic error + messages in production; detailed errors only when `DEBUG` is enabled. + - **TLS:** + - Services never terminate TLS directly. They are always deployed behind + a TLS-terminating reverse proxy. The service itself listens on plain + HTTP. However, HSTS headers and `Secure` cookie flags must still be + set by the application so that the browser enforces HTTPS end-to-end. + + This list is non-exhaustive. Apply defense-in-depth: if a standard security + hardening measure exists for HTTP services and is not listed here, it is + still expected. When in doubt, harden. + +- `README.md` is the primary documentation. Required sections: + - **Description**: First line must include the project name, purpose, + category (web server, SPA, CLI tool, etc.), license, and author. Example: + "µPaaS is an MIT-licensed Go web application by @sneak that receives + git-frontend webhooks and deploys applications via Docker in realtime." + - **Getting Started**: Copy-pasteable install/usage code block. + - **Entrypoints**: Opens by stating that the repo adheres to the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + standard (with that link), then documents each provided `script/` + entrypoint and its purpose. + - **Rationale**: Why does this exist? + - **Design**: How is the program structured? + - **TODO**: Update meticulously, even between commits. When planning, put + the todo list in the README so a new agent can pick up where the last one + left off. + - **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a + `LICENSE` file in the repo root and a License section in the README. + - **Author**: [@sneak](https://sneak.berlin). + +- First commit of a new repo should contain only `README.md`. + +- Go module root: `sneak.berlin/go/`. Always run `go mod tidy` before + committing. + +- Use SemVer. + +- Database migrations live in `internal/db/migrations/` and must be embedded in + the binary. + - `000_migration.sql` — contains ONLY the creation of the migrations + tracking table itself. Nothing else. + - `001_schema.sql` — the full application schema. + - **Pre-1.0.0:** never add additional migration files (002, 003, etc.). + There is no installed base to migrate. Edit `001_schema.sql` directly. + - **Post-1.0.0:** add new numbered migration files for each schema change. + Never edit existing migrations after release. + +- All repos should have an `.editorconfig` enforcing the project's indentation + settings. + +- Avoid putting files in the repo root unless necessary. Root should contain + only project-level config files (`README.md`, `Makefile`, `Dockerfile`, + `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and + language-specific config). Everything else goes in a subdirectory. Canonical + subdirectory names: + - `bin/` — executable scripts and tools + - `cmd/` — Go command entrypoints + - `configs/` — configuration templates and examples + - `deploy/` — deployment manifests (k8s, compose, terraform) + - `docs/` — documentation and markdown (README.md stays in root) + - `internal/` — Go internal packages + - `internal/db/migrations/` — database migrations + - `pkg/` — Go library packages + - `share/` — systemd units, data files + - `static/` — static assets (images, fonts, etc.) + - `web/` — web frontend source + +- When setting up a new repo, files from the `prompts` repo may be used as + templates. Fetch them from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/`. + +- New repos must contain at minimum: + - `README.md`, `.git`, `.gitignore`, `.editorconfig` + - `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo) + - `Makefile` + - `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`, + `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, + `install-precommit`) - `Dockerfile`, `.dockerignore` - - for go: `go.mod`, `go.sum`, `.golangci.yml` - - for js: `package.json` + - `.gitea/workflows/check.yml` + - Go: `go.mod`, `go.sum`, `.golangci.yml` + - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` + - Python: `pyproject.toml` diff --git a/TODO.md b/TODO.md index 9e280f9..587c805 100644 --- a/TODO.md +++ b/TODO.md @@ -1,47 +1,47 @@ # Workflow -* branch (from `main`) -* do the work in Next Step -* move Next Step to the top of Completed Steps -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* merge to `main` if the branch is not protected, otherwise open a PR -* push +- branch (from `main`) +- do the work in Next Step +- move Next Step to the top of Completed Steps +- move the top item of Future Steps into Next Step +- commit (`TODO.md` changes in the same commit as the work) +- merge to `main` if the branch is not protected, otherwise open a PR +- push # Status -pre-1.0. No git tags. Backend work in flight on feat/reportbuf-storage -(dirty: src/main.js). Frontend is functional; backend is new and unmerged. +pre-1.0. No git tags. Backend work in flight on feat/reportbuf-storage (dirty: +src/main.js). Frontend is functional; backend is new and unmerged. # Next Step -Land feat/reportbuf-storage: finish the in-progress src/main.js change, -get make check green, and merge the branch to main. The branch adds the -backend (buffered zstd-compressed report storage), the CI workflow, and -backend repo standard files, so merging it also closes most compliance -gaps. +Land feat/reportbuf-storage: finish the in-progress src/main.js change, get make +check green, and merge the branch to main. The branch adds the backend (buffered +zstd-compressed report storage), the CI workflow, and backend repo standard +files, so merging it also closes most compliance gaps. # Completed Steps -- 2026-02-27: backend with buffered zstd-compressed report storage; CI - workflow and backend repo standard files; backend Dockerfile fixed (Go - 1.25, golangci-lint) and moved to repo root (feat/reportbuf-storage, - unmerged) -- 2026-02-26: host row layout redesigned with CSS grid; overflow and - spacing fixes; nginx config extracted; port hardcoded to 8080 -- 2026-02-26: debug log panel, median stats, recovery probe, Docker - build fix, S3 Singapore endpoint added -- 2026-02-23: summary box redesign, host pinning, local and UTC clocks, - checks counter -- 2026-02-23: hosts sorted by latency; GET instead of HEAD for latency; - timeout derived from interval; Hetzner regional endpoints; 3s interval +- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile + shims, README Entrypoints section +- 2026-02-27: backend with buffered zstd-compressed report storage; CI workflow + and backend repo standard files; backend Dockerfile fixed (Go 1.25, + golangci-lint) and moved to repo root (feat/reportbuf-storage, unmerged) +- 2026-02-26: host row layout redesigned with CSS grid; overflow and spacing + fixes; nginx config extracted; port hardcoded to 8080 +- 2026-02-26: debug log panel, median stats, recovery probe, Docker build fix, + S3 Singapore endpoint added +- 2026-02-23: summary box redesign, host pinning, local and UTC clocks, checks + counter +- 2026-02-23: hosts sorted by latency; GET instead of HEAD for latency; timeout + derived from interval; Hetzner regional endpoints; 3s interval - 2026-01-29: initial NetWatch network latency monitor # Future Steps -- Compliance top-up as one small commit: add .editorconfig and add the - hooks target to the Makefile -- After merge, confirm .gitea/workflows/check.yml is on main and CI is - green (main always green policy) -- Decide what to do with untracked resume.sh: commit it, gitignore it, - or delete it +- Compliance top-up as one small commit: add .editorconfig and add the hooks + target to the Makefile +- After merge, confirm .gitea/workflows/check.yml is on main and CI is green + (main always green policy) +- Decide what to do with untracked resume.sh: commit it, gitignore it, or delete + it diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..4df1d8c --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,143 @@ +#!/bin/sh +# script/bootstrap: install all dependencies needed to build and develop +# this repo. Idempotent: every install is guarded by a check so already +# installed tools are skipped. Base tooling comes from nix, apt, brew, +# or apk (detected in that order); assumes nothing is present. Node is +# used directly if installed; otherwise it is installed at a pinned +# version via nvm (installing nvm itself first, from a hash-verified +# release archive, never curl | sh). +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Pinned versions, 2026-07-07 +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + +PKGMGR="" +SUDO="" +APT_UPDATED="" + +detect_pkgmgr() { + [ -n "$PKGMGR" ] && return 0 + if command -v nix-env >/dev/null 2>&1; then + PKGMGR="nix" + elif command -v apt-get >/dev/null 2>&1; then + PKGMGR="apt" + elif command -v brew >/dev/null 2>&1; then + PKGMGR="brew" + elif command -v apk >/dev/null 2>&1; then + PKGMGR="apk" + else + echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2 + exit 1 + fi + if [ "$PKGMGR" = "apt" ]; then + export DEBIAN_FRONTEND=noninteractive + if [ "$(id -u)" != "0" ]; then + SUDO="sudo" + fi + fi +} + +# pkg_install +pkg_install() { + detect_pkgmgr + case "$PKGMGR" in + nix) nix-env -iA "nixpkgs.$1" ;; + apt) + if [ -z "$APT_UPDATED" ]; then + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update + APT_UPDATED=1 + fi + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" + ;; + brew) brew install "$3" ;; + apk) apk add --no-cache "$4" ;; + esac +} + +missing() { + ! command -v "$1" >/dev/null 2>&1 +} + +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +ensure_node() { + if ! missing node; then return 0; fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" +} + +ensure_yarn() { + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + elif [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + else + yarn install --frozen-lockfile + fi +} + +main() { + cd "$ROOT" + + if missing make; then pkg_install gnumake make make make; fi + if missing git; then pkg_install git git git git; fi + + ensure_node + ensure_yarn + install_js_deps + + echo "bootstrap complete" +} + +main "$@" diff --git a/script/check b/script/check new file mode 100755 index 0000000..3e1778c --- /dev/null +++ b/script/check @@ -0,0 +1,14 @@ +#!/bin/sh +# script/check: run all checks (test, lint, fmt-check). Our own +# extension to scripts-to-rule-them-all. Must not modify any files. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/test" + "$SCRIPT_DIR/lint" + "$SCRIPT_DIR/fmt-check" +} + +main "$@" diff --git a/script/cibuild b/script/cibuild new file mode 100755 index 0000000..966f51d --- /dev/null +++ b/script/cibuild @@ -0,0 +1,13 @@ +#!/bin/sh +# script/cibuild: run the CI build. The Dockerfile runs make check, so +# a successful build implies all checks pass. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build . +} + +main "$@" diff --git a/script/docker b/script/docker new file mode 100755 index 0000000..aa9387f --- /dev/null +++ b/script/docker @@ -0,0 +1,14 @@ +#!/bin/sh +# script/docker: build the Docker image tagged with the project name. +# The tag comes from script/projectname. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + timeout 300 docker build -t "$("$SCRIPT_DIR/projectname")" . +} + +main "$@" diff --git a/script/fmt b/script/fmt new file mode 100755 index 0000000..e7d63e2 --- /dev/null +++ b/script/fmt @@ -0,0 +1,12 @@ +#!/bin/sh +# script/fmt: format all files (writes). +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + yarn prettier --write . +} + +main "$@" diff --git a/script/fmt-check b/script/fmt-check new file mode 100755 index 0000000..07ad5ea --- /dev/null +++ b/script/fmt-check @@ -0,0 +1,13 @@ +#!/bin/sh +# script/fmt-check: check formatting (read-only). Same scope as +# script/fmt, but fails instead of writing. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + yarn prettier --check . +} + +main "$@" diff --git a/script/install-precommit b/script/install-precommit new file mode 100755 index 0000000..723bae1 --- /dev/null +++ b/script/install-precommit @@ -0,0 +1,16 @@ +#!/bin/sh +# script/install-precommit: install the git pre-commit hook that runs +# script/precommit. Our own extension to scripts-to-rule-them-all. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + hook=".git/hooks/pre-commit" + printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook" + chmod +x "$hook" + echo "pre-commit hook installed: runs script/precommit" +} + +main "$@" diff --git a/script/lint b/script/lint new file mode 100755 index 0000000..054682a --- /dev/null +++ b/script/lint @@ -0,0 +1,12 @@ +#!/bin/sh +# script/lint: run the linter (prettier in check mode). +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + yarn prettier --check . +} + +main "$@" diff --git a/script/precommit b/script/precommit new file mode 100755 index 0000000..c0a7867 --- /dev/null +++ b/script/precommit @@ -0,0 +1,12 @@ +#!/bin/sh +# script/precommit: run by the git pre-commit hook; fails the commit if +# checks fail. Our own extension to scripts-to-rule-them-all. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/check" +} + +main "$@" diff --git a/script/projectname b/script/projectname new file mode 100755 index 0000000..1e097a7 --- /dev/null +++ b/script/projectname @@ -0,0 +1,12 @@ +#!/bin/sh +# script/projectname: output the name of this project. Our own +# extension to scripts-to-rule-them-all. Other scripts that need the +# name (e.g. script/docker) call this, so they can stay identical +# across all repos. +set -eu + +main() { + echo "netwatch" +} + +main "$@" diff --git a/script/setup b/script/setup new file mode 100755 index 0000000..4cc5b6b --- /dev/null +++ b/script/setup @@ -0,0 +1,13 @@ +#!/bin/sh +# script/setup: set up the repo for development after a fresh clone: +# installs dependencies and the git pre-commit hook. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/install-precommit" +} + +main "$@" diff --git a/script/test b/script/test new file mode 100755 index 0000000..4e98401 --- /dev/null +++ b/script/test @@ -0,0 +1,13 @@ +#!/bin/sh +# script/test: run the test suite. This repo has no unit tests; the +# production build serves as the test (fails on broken code). +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + timeout 30 yarn build +} + +main "$@"