diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index eedd413..f600f2c 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -6,4 +6,4 @@ jobs: steps: # actions/checkout v4.2.2, 2026-03-16 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - run: docker build . + - run: script/cibuild diff --git a/Makefile b/Makefile index cab3f5b..70df7e8 100644 --- a/Makefile +++ b/Makefile @@ -13,18 +13,24 @@ GOLDFLAGS += -X main.Version=$(VERSION) GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD) GOFLAGS := -ldflags "$(GOLDFLAGS)" -.PHONY: docker default run ci test check lint fmt fmt-check hooks fixme +.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check hooks fixme default: fmt test +bootstrap: + @script/bootstrap + +setup: + @script/setup + run: ./bin/mfer ./$< ./$< gen ci: test -test: $(SOURCEFILES) mfer/mf.pb.go - go test -v --timeout 10s ./... +test: + @script/test $(PROTOC_GEN_GO): test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 @@ -32,14 +38,14 @@ $(PROTOC_GEN_GO): fixme: @grep -nir fixme . | grep -v Makefile -check: test lint fmt-check +check: + @script/check -fmt-check: mfer/mf.pb.go - sh -c 'test -z "$$(gofmt -l .)"' +fmt-check: + @script/fmt-check hooks: - echo '#!/bin/sh\nmake check' > .git/hooks/pre-commit - chmod +x .git/hooks/pre-commit + @script/install-precommit devprereqs: which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v2.0.2 @@ -54,17 +60,14 @@ bin/mfer: $(SOURCEFILES) mfer/mf.pb.go clean: rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage -fmt: mfer/mf.pb.go - gofumpt -l -w mfer internal cmd - golangci-lint run --fix - -prettier -w *.json - -prettier -w *.md +fmt: + @script/fmt lint: - golangci-lint run - sh -c 'test -z "$$(gofmt -l .)"' + @script/lint -docker: sneak-mfer.$(ARCH).tzst.dockerimage +docker: + @script/docker sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer . diff --git a/README.md b/README.md index 259a9b0..ea469ce 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,39 @@ software. A compatible javascript library is planned. # Build Status -CI runs via `docker build .` which executes `make check` (formatting, -linting, tests). The `main` branch must always be green. +CI runs via `script/cibuild` (`docker build .`), which executes `make +check` (formatting, linting, tests). The `main` branch must always be +green. + +# Entrypoints + +This repository adheres to the +[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) +standard: normalized scripts in `script/` are the entrypoints for the +development workflow, and the Makefile targets are thin shims that call +them. We provide: + +- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go + module download), idempotently +- `script/setup` — make a fresh clone ready for development: runs + `script/bootstrap`, then `script/install-precommit` +- `script/projectname` — output the project name (`mfer`); used by other + scripts such as `script/docker` +- `script/test` — run the test suite (`go test`), regenerating the + protobuf code first if it is stale +- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness +- `script/fmt` — format all code and docs (writes): `gofumpt`, + `golangci-lint run --fix`, and prettier for JSON/Markdown +- `script/fmt-check` — check formatting without writing +- `script/check` — run `script/test`, `script/lint`, and + `script/fmt-check` +- `script/docker` — build the Docker image tagged with the project name +- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile + runs the checks) +- `script/precommit` — pre-commit checks: `go mod tidy` verification, + then `script/check` +- `script/install-precommit` — install the git pre-commit hook that + runs `script/precommit` # Participation @@ -126,6 +157,7 @@ The manifest file would do several important things: # Open Questions - Should the manifest file include checksums of individual file chunks, or just for the whole assembled file? + - If so, should the chunksize be fixed or dynamic? - Should the manifest signature format be GnuPG signatures, or those from diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 5f8e062..bc2f161 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-02-22 +last_modified: 2026-07-06 --- This document covers repository structure, tooling, and workflow standards. Code @@ -34,10 +34,46 @@ style conventions are in separate documents: every file before committing. There are zero exceptions to this rule. - Every repo with software must have a root `Makefile` with these targets: - `make test`, `make lint`, `make fmt` (writes), `make fmt-check` (read-only), - `make check` (prereqs: `test`, `lint`, `fmt-check`), `make docker`, and - `make hooks` (installs pre-commit hook). A model Makefile is at - `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`. + `make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes), + `make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`), + `make docker`, and `make hooks` (installs pre-commit hook). A model Makefile + is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`. + +- Repos follow the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + pattern: the implementation of each Makefile target lives in an executable + script in `script/` (`script/bootstrap`, `script/setup`, `script/test`, + `script/lint`, `script/fmt`, `script/fmt-check`, `script/check`, + `script/docker`), and the Makefile targets are thin shims that call them. The + scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in + minimal containers (e.g. alpine images have no bash); locate the repo root + with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From + the standard's canonical set we use `bootstrap`, `setup` (make the repo ready + for development after a fresh clone: runs `bootstrap`, then + `install-precommit`, plus any repo-specific initialization), `test`, and + `cibuild`. `script/bootstrap` installs all dependencies idempotently and + assumes nothing is present: base tools come from nix, apt, brew, or apk + (detected in that order; apt runs noninteractive). For node it uses the + installed node if present; otherwise it installs a PINNED node version via + nvm, first installing nvm itself if missing — from a hash-verified GitHub + release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. yarn is then pinned via + `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; + always exact versions. `script/cibuild` runs the CI build: it changes to the + repo root and runs `docker build .`; the Gitea workflow calls it. Four further + scripts are our own extensions to the standard: `script/check` runs + `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is + what the git pre-commit hook runs, and it calls `script/check`; + `script/install-precommit` installs the git pre-commit hook (the `make hooks` + target shims to it); and `script/projectname` (literally that filename) simply + outputs the project's name. Scripts that need the name call + `script/projectname` — e.g. `script/docker` assembles its image tag from it — + so those scripts stay byte-identical across all repos. Repo-type-specific + pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in + `script/precommit`, not in the hook itself. Model scripts are at + `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README + must document the provided scripts in an **Entrypoints** section (see the + README requirements below). - Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.) instead of invoking the underlying tools directly. The Makefile is the single @@ -57,11 +93,83 @@ style conventions are in separate documents: as a build step so the build fails if the branch is not green. For non-server repos, the Dockerfile should bring up a development environment and run `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. + stage before the final image is assembled. Dockerfiles install development + prerequisites by running `script/bootstrap` rather than duplicating installs + inline; COPY `script/` and the dependency manifests (`package.json` + + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap + layer stays cached until dependencies change. + +- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go + repos use a multistage build where linting runs in an independent stage based + on the `golangci/golangci-lint` image (pinned by hash). This stage runs + `make fmt-check` and `make lint` before the full build begins. The build stage + then declares an explicit dependency on the lint stage via + `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete + linting before proceeding to compilation and tests. This ensures lint failures + surface in seconds rather than minutes, without blocking on dependency + download or compilation in the build stage. + + The standard pattern for a Go repo Dockerfile is: + + ```dockerfile + # Lint stage — fast feedback on formatting and lint issues + # golangci/golangci-lint:v2.x.x, YYYY-MM-DD + FROM golangci/golangci-lint@sha256:... AS lint + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN make fmt-check + RUN make lint + + # Build stage + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + WORKDIR /src + + # Force BuildKit to run the lint stage before proceeding + COPY --from=lint /src/go.sum /dev/null + + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN make test + + ARG VERSION=dev + RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage + FROM alpine@sha256:... + COPY --from=builder /app /usr/local/bin/app + ENTRYPOINT ["app"] + ``` + + Key points: + - The lint stage uses the `golangci/golangci-lint` image directly (it + includes both Go and the linter), so there is no need to install the + linter separately. + - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates + a stage dependency. BuildKit runs stages in parallel by default; without + this line, the build stage would not wait for lint to finish and a lint + failure might not fail the overall build. + - If the project uses `//go:embed` directives that reference build artifacts + (e.g. a web frontend compiled in a separate stage), the lint stage must + create placeholder files so the embed directives resolve. Example: + `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. + The lint stage should not depend on the actual build output — it exists to + fail fast. + - If the project requires CGO or system libraries for linting (e.g. + `vips-dev`), install them in the lint stage with `apk add`. + - The build stage runs `make test` after compilation setup. Tests run in the + build stage, not the lint stage, because they may require compiled + artifacts or heavier dependencies. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `docker build .` on push. Since the Dockerfile already runs `make check`, - a successful build implies all checks pass. + runs `script/cibuild` (which runs `docker build .`) on push. Since the + Dockerfile already runs `make check`, a successful build implies all checks + pass. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -69,9 +177,11 @@ style conventions are in separate documents: Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown, HTML, CSS) should also have `.prettierrc` and `.prettierignore`. -- Pre-commit hook: `make check` if local testing is possible, otherwise - `make lint && make fmt-check`. The Makefile should provide a `make hooks` - target to install the pre-commit hook. +- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local + testing is not possible in the repo, `script/precommit` may skip `script/test` + and run only `script/lint` and `script/fmt-check`. The hook is installed by + `script/install-precommit`; the Makefile must provide a `make hooks` target + that shims to it. - All repos with software must have tests that run via the platform-standard test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful @@ -82,6 +192,42 @@ style conventions are in separate documents: - `make test` must complete in under 20 seconds. Add a 30-second timeout in the Makefile. +- **`make test` should use the conditional verbose rerun pattern.** Run tests + without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to + show full output. This keeps CI logs and `docker build` output clean on + success (just package/suite summaries) while providing full diagnostic detail + on failure (every test case, every assertion). The general shell pattern: + + ```makefile + test: + @ || \ + { echo "--- Rerunning with -v for details ---"; \ + ; exit 1; } + ``` + + Go example: + + ```makefile + test: + @go test -timeout 30s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 30s -race -v ./...; exit 1; } + ``` + + Python example: + + ```makefile + test: + @python -m pytest || \ + { echo "--- Rerunning with -v for details ---"; \ + python -m pytest -v; exit 1; } + ``` + + The `exit 1` ensures the target always fails after a rerun — the first run + already proved the tests are broken, so the build must not pass even if a + flaky test happens to succeed on the second attempt. The rerun exists solely + for diagnostic output. + - Docker builds must complete in under 5 minutes. - `make check` must not modify any files in the repo. Tests may use temporary @@ -98,6 +244,13 @@ style conventions are in separate documents: `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up a new repo. +- **No build artifacts in version control.** Code-derived data (compiled + bundles, minified output, generated assets) must never be committed to the + repository if it can be avoided. The build process (e.g. Dockerfile, Makefile) + should generate these at build time. Notable exception: Go protobuf generated + files (`.pb.go`) ARE committed because repos need to work with `go get`, which + downloads code but does not execute code generation. + - Never use `git add -A` or `git add .`. Always stage files explicitly by name. - Never force-push to `main`. @@ -121,12 +274,76 @@ style conventions are in separate documents: - Dockerized web services listen on port 8080 by default, overridable with `PORT`. +- **HTTP/web services must be hardened for production internet exposure before + tagging 1.0.** This means full compliance with security best practices + including, without limitation, all of the following: + - **Security headers** on every response: + - `Strict-Transport-Security` (HSTS) with `max-age` of at least one year + and `includeSubDomains`. + - `Content-Security-Policy` (CSP) with a restrictive default policy + (`default-src 'self'` as a baseline, tightened per-resource as + needed). Never use `unsafe-inline` or `unsafe-eval` unless + unavoidable, and document the reason. + - `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required). + Prefer the `frame-ancestors` CSP directive as the primary control. + - `X-Content-Type-Options: nosniff`. + - `Referrer-Policy: strict-origin-when-cross-origin` (or stricter). + - `Permissions-Policy` restricting access to browser features the + application does not use (camera, microphone, geolocation, etc.). + - **Request and response limits:** + - Maximum request body size enforced on all endpoints (e.g. Go + `http.MaxBytesReader`). Choose a sane default per-route; never accept + unbounded input. + - Maximum response body size where applicable (e.g. paginated APIs). + - `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend + against slowloris attacks. + - `WriteTimeout` on the `http.Server`. + - `IdleTimeout` on the `http.Server`. + - Per-handler execution time limits via `context.WithTimeout` or + chi/stdlib `middleware.Timeout`. + - **Authentication and session security:** + - Rate limiting on password-based authentication endpoints. API keys are + high-entropy and not susceptible to brute force, so they are exempt. + - CSRF tokens on all state-mutating HTML forms. API endpoints + authenticated via `Authorization` header (Bearer token, API key) are + exempt because the browser does not attach these automatically. + - Passwords stored using bcrypt, scrypt, or argon2 — never plain-text, + MD5, or SHA. + - Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or + `Strict`) attributes. + - **Reverse proxy awareness:** + - True client IP detection when behind a reverse proxy + (`X-Forwarded-For`, `X-Real-IP`). The application must accept + forwarded headers only from a configured set of trusted proxy + addresses — never trust `X-Forwarded-For` unconditionally. + - **CORS:** + - Authenticated endpoints must restrict `Access-Control-Allow-Origin` to + an explicit allowlist of known origins. Wildcard (`*`) is acceptable + only for public, unauthenticated read-only APIs. + - **Error handling:** + - Internal errors must never leak stack traces, SQL queries, file paths, + or other implementation details to the client. Return generic error + messages in production; detailed errors only when `DEBUG` is enabled. + - **TLS:** + - Services never terminate TLS directly. They are always deployed behind + a TLS-terminating reverse proxy. The service itself listens on plain + HTTP. However, HSTS headers and `Secure` cookie flags must still be + set by the application so that the browser enforces HTTPS end-to-end. + + This list is non-exhaustive. Apply defense-in-depth: if a standard security + hardening measure exists for HTTP services and is not listed here, it is + still expected. When in doubt, harden. + - `README.md` is the primary documentation. Required sections: - **Description**: First line must include the project name, purpose, category (web server, SPA, CLI tool, etc.), license, and author. Example: "µPaaS is an MIT-licensed Go web application by @sneak that receives git-frontend webhooks and deploys applications via Docker in realtime." - **Getting Started**: Copy-pasteable install/usage code block. + - **Entrypoints**: Opens by stating that the repo adheres to the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + standard (with that link), then documents each provided `script/` + entrypoint and its purpose. - **Rationale**: Why does this exist? - **Design**: How is the program structured? - **TODO**: Update meticulously, even between commits. When planning, put @@ -144,8 +361,14 @@ style conventions are in separate documents: - Use SemVer. - Database migrations live in `internal/db/migrations/` and must be embedded in - the binary. Pre-1.0.0: modify existing migrations (no installed base assumed). - Post-1.0.0: add new migration files. + the binary. + - `000_migration.sql` — contains ONLY the creation of the migrations + tracking table itself. Nothing else. + - `001_schema.sql` — the full application schema. + - **Pre-1.0.0:** never add additional migration files (002, 003, etc.). + There is no installed base to migrate. Edit `001_schema.sql` directly. + - **Post-1.0.0:** add new numbered migration files for each schema change. + Never edit existing migrations after release. - All repos should have an `.editorconfig` enforcing the project's indentation settings. @@ -175,6 +398,9 @@ style conventions are in separate documents: - `README.md`, `.git`, `.gitignore`, `.editorconfig` - `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo) - `Makefile` + - `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`, + `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, + `install-precommit`) - `Dockerfile`, `.dockerignore` - `.gitea/workflows/check.yml` - Go: `go.mod`, `go.sum`, `.golangci.yml` diff --git a/TODO.md b/TODO.md index 99f978d..6100b0d 100644 --- a/TODO.md +++ b/TODO.md @@ -1,12 +1,12 @@ # Workflow -* branch (from `main`) -* do the work in Next Step -* move Next Step to the top of Completed Steps -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* merge to `main` if the branch is not protected, otherwise open a PR -* push +- branch (from `main`) +- do the work in Next Step +- move Next Step to the top of Completed Steps +- move the top item of Future Steps into Next Step +- commit (`TODO.md` changes in the same commit as the work) +- merge to `main` if the branch is not protected, otherwise open a PR +- push # Status @@ -23,6 +23,8 @@ commit the uncommitted work (32 modified Go files, new untracked # Completed Steps +- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, + Makefile shims, README Entrypoints section - 2026-07-03: aligned repo tooling, docs, and config with standardized policies (7d9a138, on chore/align-repo-policies, unmerged) - 2026-06-28: moved to standardized repo policies (#56, on main) @@ -42,64 +44,64 @@ commit the uncommitted work (32 modified Go files, new untracked - Compliance (fold of TODO.md audit 2026-07-02; verify which items the in-flight branch already closes, then check off): - - Add .editorconfig (canonical copy from sneak/prompts) - - Add standardized .golangci.yml (present untracked on the branch; - user-owned, copy verbatim) - - Make .gitignore cover secrets (.env, *.key, *.pem), OS files - (.DS_Store), and editor files (*.swp, *~) - - Make fmt-check/lint verify with gofumpt, not gofmt -l, so - `make check` matches what `make fmt` writes - - Add README "Getting Started" section with copy-pasteable - install/usage block - - Move FORMAT.md from repo root to docs/ and update the AGENTS.md - reference - - Pin Makefile-installed Go tools (protoc-gen-go@v1.28.1, - golangci-lint@v2.0.2) by module hash, not mutable tag - - Set `make test` timeout to 30s (currently 10s) - - Add explicit README "Rationale" heading (content exists under other - names); name the author in the README Description first line - - Reconcile root-level AGENTS.md with directory-hygiene policy (keep - or relocate) - - Add a `make build` target - - Rewrite `make hooks` to use printf or a heredoc instead of - non-portable `echo '...\n...'` + - Add .editorconfig (canonical copy from sneak/prompts) + - Add standardized .golangci.yml (present untracked on the branch; + user-owned, copy verbatim) + - Make .gitignore cover secrets (.env, _.key, _.pem), OS files + (.DS_Store), and editor files (_.swp, _~) + - Make fmt-check/lint verify with gofumpt, not gofmt -l, so + `make check` matches what `make fmt` writes + - Add README "Getting Started" section with copy-pasteable + install/usage block + - Move FORMAT.md from repo root to docs/ and update the AGENTS.md + reference + - Pin Makefile-installed Go tools (protoc-gen-go@v1.28.1, + golangci-lint@v2.0.2) by module hash, not mutable tag + - Set `make test` timeout to 30s (currently 10s) + - Add explicit README "Rationale" heading (content exists under other + names); name the author in the README Description first line + - Reconcile root-level AGENTS.md with directory-hygiene policy (keep + or relocate) + - Add a `make build` target + - Rewrite `make hooks` to use printf or a heredoc instead of + non-portable `echo '...\n...'` - Answer the 14 owner design questions in the README 1.0 roadmap: - - Format: simplify MFFileChecksum; store file mode; drop atime; - specify path normalization rules; version byte after magic; - length-prefix after magic - - Signatures: hash covers compressed or uncompressed data; sign raw - bytes vs hex canonical string; detached .mf.sig support; GPG - subprocess vs pure-Go crypto - - Implementation: deterministic manifests by default; consolidate - duplicate scanner/checker implementations; export the manifest - type; canonical Go module path for 1.0 + - Format: simplify MFFileChecksum; store file mode; drop atime; + specify path normalization rules; version byte after magic; + length-prefix after magic + - Signatures: hash covers compressed or uncompressed data; sign raw + bytes vs hex canonical string; detached .mf.sig support; GPG + subprocess vs pure-Go crypto + - Implementation: deterministic manifests by default; consolidate + duplicate scanner/checker implementations; export the manifest + type; canonical Go module path for 1.0 - Format and correctness: - - Resolve proto go_package vs go.mod module path inconsistency - - Specify and validate path invariants (UTF-8, forward-slash, - relative, no .., no leading /) - - Remove or deprecate atime; reserve mode field; add version byte - (all pending design answers) - - Write a standalone format specification document + - Resolve proto go_package vs go.mod module path inconsistency + - Specify and validate path invariants (UTF-8, forward-slash, + relative, no .., no leading /) + - Remove or deprecate atime; reserve mode field; add version byte + (all pending design answers) + - Write a standalone format specification document - Library: - - Delete internal/scanner and internal/checker; consolidate on the - mfer/ package versions (pending design answer) - - Add decompression size limit via io.LimitReader in - deserializeInner() - - Fix errors.Is dead code in checker; make AddFile verify - totalRead == size - - Export manifest type or define a public interface (pending) - - Replace GPG subprocess with pure-Go crypto (pending); add timeouts - to remaining subprocess calls + - Delete internal/scanner and internal/checker; consolidate on the + mfer/ package versions (pending design answer) + - Add decompression size limit via io.LimitReader in + deserializeInner() + - Fix errors.Is dead code in checker; make AddFile verify + totalRead == size + - Export manifest type or define a public interface (pending) + - Replace GPG subprocess with pure-Go crypto (pending); add timeouts + to remaining subprocess calls - CLI: - - Kebab-case primary flag names; fix fetch URL construction with - url.JoinPath; add http.Client timeout and retry with backoff to - fetch; rate-limit Checker progress output; add --deterministic - flag or default; wire top-level --version properly + - Kebab-case primary flag names; fix fetch URL construction with + url.JoinPath; add http.Client timeout and retry with backoff to + fetch; rate-limit Checker progress output; add --deterministic + flag or default; wire top-level --version properly - Testing: - - Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch + - Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch - Documentation: - - Promote docs/FORMAT.md as primary spec reference; audit error - messages; document the signature scheme fully + - Promote docs/FORMAT.md as primary spec reference; audit error + messages; document the signature scheme fully - Release: - - Finalize module path, bump version constant, SemVer --version - output, tag v1.0.0 + - Finalize module path, bump version constant, SemVer --version + output, tag v1.0.0 diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..c40c5e5 --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,155 @@ +#!/bin/sh +# script/bootstrap: install all dependencies needed to build and develop +# this repo. Idempotent: every install is guarded by a check so already +# installed tools are skipped. Base tooling comes from nix, apt, brew, +# or apk (detected in that order); assumes NOTHING is present (not git, +# make, node, yarn, go, or python). Node is used directly if installed; +# otherwise a pinned version is installed via nvm (installing nvm +# itself first, from a hash-verified release archive, never curl | sh). +# +# Uncomment the language sections in main() that apply to this repo. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions. +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + +PKGMGR="" +SUDO="" + +detect_pkgmgr() { + [ -n "$PKGMGR" ] && return 0 + if command -v nix-env >/dev/null 2>&1; then + PKGMGR="nix" + elif command -v apt-get >/dev/null 2>&1; then + PKGMGR="apt" + elif command -v brew >/dev/null 2>&1; then + PKGMGR="brew" + elif command -v apk >/dev/null 2>&1; then + PKGMGR="apk" + else + echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2 + exit 1 + fi + if [ "$PKGMGR" = "apt" ]; then + export DEBIAN_FRONTEND=noninteractive + if [ "$(id -u)" != "0" ]; then + SUDO="sudo" + fi + fi +} + +# pkg_install +pkg_install() { + detect_pkgmgr + case "$PKGMGR" in + nix) nix-env -iA "nixpkgs.$1" ;; + apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;; + brew) brew install "$3" ;; + apk) apk add --no-cache "$4" ;; + esac +} + +missing() { + ! command -v "$1" >/dev/null 2>&1 +} + +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash, so install it too + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +ensure_node() { + if ! missing node; then return 0; fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" +} + +ensure_yarn() { + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + elif [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + else + yarn install --frozen-lockfile + fi +} + +main() { + cd "$ROOT" + + # Base tooling (every repo) + if missing git; then pkg_install git git git git; fi + if missing make; then pkg_install gnumake make make make; fi + + # ---- JS / docs repos ---- + # ensure_node + # ensure_yarn + # install_js_deps + + # ---- Go repos ---- + if missing go; then pkg_install go golang go go; fi + # golangci-lint: packaged in nix, brew, and apk. On apt there is no + # package: download a specific release archive from GitHub and + # verify its hash (verify_sha256), never curl | sh. + if missing golangci-lint; then + pkg_install golangci-lint golangci-lint golangci-lint golangci-lint + fi + go mod download + + # ---- Python repos ---- + # if missing python3; then pkg_install python3 python3 python3 python3; fi + # python3 -m venv .venv + # ./.venv/bin/pip install -e '.[dev]' + + echo "bootstrap complete" +} + +main "$@" diff --git a/script/check b/script/check new file mode 100755 index 0000000..cc046f7 --- /dev/null +++ b/script/check @@ -0,0 +1,15 @@ +#!/bin/sh +# script/check: run all checks (test, lint, fmt-check). Our own +# extension to scripts-to-rule-them-all. Must not modify any files. +# Generic: usually needs no adaptation. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/test" + "$SCRIPT_DIR/lint" + "$SCRIPT_DIR/fmt-check" +} + +main "$@" diff --git a/script/cibuild b/script/cibuild new file mode 100755 index 0000000..3da5857 --- /dev/null +++ b/script/cibuild @@ -0,0 +1,14 @@ +#!/bin/sh +# script/cibuild: run the CI build. The Dockerfile runs script/check +# (via make check), so a successful build implies all checks pass. +# Generic: needs no adaptation. The Gitea workflow runs this on push. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build . +} + +main "$@" diff --git a/script/docker b/script/docker new file mode 100755 index 0000000..2884e41 --- /dev/null +++ b/script/docker @@ -0,0 +1,15 @@ +#!/bin/sh +# script/docker: build the Docker image tagged with the project name. +# Identical in all repos; the tag comes from script/projectname. +# Generic: needs no adaptation. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build -t "$("$SCRIPT_DIR/projectname")" . +} + +main "$@" diff --git a/script/fmt b/script/fmt new file mode 100755 index 0000000..8607e7f --- /dev/null +++ b/script/fmt @@ -0,0 +1,27 @@ +#!/bin/sh +# script/fmt: format all files (writes). +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale +# (mirrors the old Makefile prerequisite; the generated file is +# committed, so this is normally a no-op). +ensure_pb() { + if [ ! -f mfer/mf.pb.go ] || + [ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then + (cd mfer && go generate .) + fi +} + +main() { + cd "$ROOT" + ensure_pb + gofumpt -l -w mfer internal cmd + golangci-lint run --fix + # prettier is best-effort, as in the old Makefile (- prefix) + prettier -w *.json || true + prettier -w *.md || true +} + +main "$@" diff --git a/script/fmt-check b/script/fmt-check new file mode 100755 index 0000000..6c73704 --- /dev/null +++ b/script/fmt-check @@ -0,0 +1,28 @@ +#!/bin/sh +# script/fmt-check: check formatting (read-only). Same scope as +# script/fmt, but fails instead of writing. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale +# (mirrors the old Makefile prerequisite; the generated file is +# committed, so this is normally a no-op). +ensure_pb() { + if [ ! -f mfer/mf.pb.go ] || + [ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then + (cd mfer && go generate .) + fi +} + +main() { + cd "$ROOT" + ensure_pb + if [ -n "$(gofmt -l .)" ]; then + echo "gofmt: files need formatting:" >&2 + gofmt -l . >&2 + exit 1 + fi +} + +main "$@" diff --git a/script/install-precommit b/script/install-precommit new file mode 100755 index 0000000..3519de6 --- /dev/null +++ b/script/install-precommit @@ -0,0 +1,17 @@ +#!/bin/sh +# script/install-precommit: install the git pre-commit hook that runs +# script/precommit. Our own extension to scripts-to-rule-them-all. +# Generic: needs no adaptation. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + hook=".git/hooks/pre-commit" + printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit + chmod +x .git/hooks/pre-commit + echo "pre-commit hook installed: runs script/precommit" +} + +main "$@" diff --git a/script/lint b/script/lint new file mode 100755 index 0000000..071cb67 --- /dev/null +++ b/script/lint @@ -0,0 +1,17 @@ +#!/bin/sh +# script/lint: run the linter. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + golangci-lint run + if [ -n "$(gofmt -l .)" ]; then + echo "gofmt: files need formatting:" >&2 + gofmt -l . >&2 + exit 1 + fi +} + +main "$@" diff --git a/script/precommit b/script/precommit new file mode 100755 index 0000000..fc2c7e7 --- /dev/null +++ b/script/precommit @@ -0,0 +1,20 @@ +#!/bin/sh +# script/precommit: run by the git pre-commit hook; fails the commit if +# checks fail. Our own extension to scripts-to-rule-them-all. Go repo +# extras run first: go mod tidy and go fmt, failing the commit if they +# change go.mod or go.sum. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + go mod tidy + go fmt ./... + git diff --exit-code -- go.mod go.sum || + { echo "go mod tidy changed files; stage and retry" >&2; exit 1; } + "$SCRIPT_DIR/check" +} + +main "$@" diff --git a/script/projectname b/script/projectname new file mode 100755 index 0000000..2651ec3 --- /dev/null +++ b/script/projectname @@ -0,0 +1,12 @@ +#!/bin/sh +# script/projectname: output the name of this project. Our own +# extension to scripts-to-rule-them-all. Other scripts that need the +# name (e.g. script/docker) call this, so they can stay identical +# across all repos. +set -eu + +main() { + echo "mfer" +} + +main "$@" diff --git a/script/setup b/script/setup new file mode 100755 index 0000000..53327ba --- /dev/null +++ b/script/setup @@ -0,0 +1,14 @@ +#!/bin/sh +# script/setup: set up the repo for development after a fresh clone: +# installs dependencies (script/bootstrap) and the git pre-commit hook. +# Add any repo-specific initialization (db init, .env template) here. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/install-precommit" +} + +main "$@" diff --git a/script/test b/script/test new file mode 100755 index 0000000..090605c --- /dev/null +++ b/script/test @@ -0,0 +1,23 @@ +#!/bin/sh +# script/test: run the test suite. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale +# (mirrors the old Makefile prerequisite; the generated file is +# committed, so this is normally a no-op). +ensure_pb() { + if [ ! -f mfer/mf.pb.go ] || + [ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then + (cd mfer && go generate .) + fi +} + +main() { + cd "$ROOT" + ensure_pb + go test -v --timeout 10s ./... +} + +main "$@"