diff --git a/.dockerignore b/.dockerignore index ec18f8e..671d547 100644 --- a/.dockerignore +++ b/.dockerignore @@ -61,3 +61,6 @@ # This repo's own host-built binary (make build). /bin/mfer + +# The protoc script/bootstrap unpacks for script/generate. +/bin/protoc diff --git a/.gitignore b/.gitignore index 92f3769..9683e83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /bin/mfer +/bin/protoc/ /tmp /node_modules/ diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..fc37597 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +22.17.0 diff --git a/Dockerfile b/Dockerfile index b5c26f9..1db5ba3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY . . # Go half of fmt-check only: this image has no node, so no prettier. The # markdown half runs in the mdfmt stage below. The image has no gofumpt -# either; script/gofumpt builds the version it pins with `go run`. +# either; script/gofumpt builds the version bin/tools/go.mod pins. RUN script/gofumpt --check # The linter directly, not `make lint`: script/lint builds this stage, and # there is no docker inside this build. @@ -18,7 +18,7 @@ RUN golangci-lint run --config .golangci.yml ./... # Markdown/JSON format stage — prettier needs node, which the Go images # do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node -# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins. +# 22.17.0 and yarn 1.22.22, the versions .nvmrc and script/bootstrap pin. FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt WORKDIR /src @@ -31,7 +31,7 @@ COPY . . RUN script/prettier --check # Build stage — tests and compilation -# golang:1.23 (2026-03-14) +# golang:1.23.12, 2026-03-14 FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder # Force BuildKit to run the lint and mdfmt stages by creating stage dependencies diff --git a/README.md b/README.md index 632c2d6..98d5446 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,13 @@ standard: normalized scripts in `script/` are the entrypoints for the development workflow, and the Makefile targets are thin shims that call them. We provide: -- `script/bootstrap` — install all dependencies (Go, Go module download, and - node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`), - idempotently; golangci-lint is not installed, it runs only in Docker +- `script/bootstrap` — install all dependencies, idempotently: Go and the + modules of both `go.mod` and `bin/tools/go.mod`; node (the version `.nvmrc` + names, through nvm when there is no node on `PATH`) and yarn, plus the + prettier version pinned in `package.json`/`yarn.lock`; and `protoc` 33.4, + unpacked into `bin/protoc` from its release archive once the archive matches + the sha256 the script holds for this platform. golangci-lint is not installed, + it runs only in Docker - `script/setup` — make a fresh clone ready for development: runs `script/bootstrap`, then `script/install-precommit` - `script/projectname` — output the project name (`mfer`); used by other scripts @@ -82,14 +86,10 @@ provide: - `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from `mfer/mf.proto` and record the hash of that `mfer/mf.proto` in `mfer/mf.proto.sha256`; the only thing that regenerates the committed - `mfer/mf.pb.go`. It needs the exact versions that wrote the committed file, - and refuses to run with any other: `protoc` 33.4 (unpack - `protoc-33.4-.zip` from - [its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4) - and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11 - (`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which - installs it in `$(go env GOPATH)/bin`; `script/generate` adds that directory - to `PATH`) + `mfer/mf.pb.go`. It runs the `protoc` that `script/bootstrap` unpacks into + `bin/protoc`, refusing any version but 33.4, and the `protoc-gen-go` that + `bin/tools/go.mod` pins, which `go tool` builds from source checked against + the hashes in `bin/tools/go.sum` - `script/fuzz` — fuzz the manifest parser for one minute; run by hand (`make fuzz`), never by CI, while `script/test` runs its committed seed corpus as ordinary tests @@ -99,14 +99,17 @@ provide: - `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and `script/prettier --write` - `script/gofumpt` — run `gofumpt` over every Go file in the repository in the - given mode, `--write` or `--check`, at the one version it pins (built on - demand by `go run`, so nothing installs it); `script/fmt`, `script/fmt-check` + given mode, `--write` or `--check`, at the version `bin/tools/go.mod` pins + (built on demand by `go tool` from source checked against the hashes in + `bin/tools/go.sum`, so nothing installs it); `script/fmt`, `script/fmt-check` and the Docker lint stage all go through it, so they cannot disagree about Go formatting - `script/prettier` — run prettier over the repository's canonical file set (Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or - `--check`; the single definition of that file set, so `script/fmt` and - `script/fmt-check` cannot disagree about it + `--check`, with the prettier version `yarn.lock` pins, run by the node on + `PATH` or else the one `script/bootstrap` installed through nvm; the single + definition of that file set, so `script/fmt` and `script/fmt-check` cannot + disagree about it - `script/fmt-check` — check formatting without writing: `script/gofumpt --check` plus `script/prettier --check` - `script/check` — run `script/test`, `script/lint`, and `script/fmt-check` diff --git a/bin/tools/go.mod b/bin/tools/go.mod new file mode 100644 index 0000000..d8aa330 --- /dev/null +++ b/bin/tools/go.mod @@ -0,0 +1,22 @@ +// The developer tools this repo runs with `go tool`: gofumpt for +// script/gofumpt and protoc-gen-go for script/generate. Kept out of the mfer +// module so they add nothing to what mfer's users download. `go tool` builds +// exactly the source whose hashes go.sum here records. +module sneak.berlin/go/mfer/bin/tools + +go 1.26.0 + +tool ( + google.golang.org/protobuf/cmd/protoc-gen-go + mvdan.cc/gofumpt +) + +require ( + golang.org/x/mod v0.40.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/tools v0.49.0 // indirect + // protoc-gen-go v1.36.11, 2026-10-04 + google.golang.org/protobuf v1.36.11 // indirect + // gofumpt v0.12.0, 2026-10-04 + mvdan.cc/gofumpt v0.12.0 // indirect +) diff --git a/bin/tools/go.sum b/bin/tools/go.sum new file mode 100644 index 0000000..0c1d6ad --- /dev/null +++ b/bin/tools/go.sum @@ -0,0 +1,22 @@ +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +mvdan.cc/gofumpt v0.12.0 h1:1Lbudkz2kpM9Cjz2pL4M19u7q+GaEhCTNf7N9mfpcho= +mvdan.cc/gofumpt v0.12.0/go.mod h1:SmBHHrljiZu/uoypeKup3rFzP6eoC9UwCp2iH5E3jZA= diff --git a/mfer/mf.go b/mfer/mf.go deleted file mode 100644 index 1bb16da..0000000 --- a/mfer/mf.go +++ /dev/null @@ -1,3 +0,0 @@ -package mfer - -//go:generate protoc ./mf.proto --go_out=paths=source_relative:. diff --git a/package.json b/package.json index cca61f6..1a23c56 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,5 @@ { "name": "mfer", - "version": "0.1.0", "private": true, "description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.", "license": "WTFPL", diff --git a/script/bootstrap b/script/bootstrap index 1ee831f..7d61b77 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -13,11 +13,15 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" # Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions. -NODE_VERSION="22.17.0" +# The node version is in .nvmrc, where script/prettier reads it too. +NODE_VERSION="$(cat "$ROOT/.nvmrc")" NVM_VERSION="0.40.3" # sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" YARN_VERSION="1.22.22" +# protoc v33.4, 2026-10-04, for script/generate. The sha256 of each +# platform's release archive is in ensure_protoc. +PROTOC_VERSION="33.4" PKGMGR="" SUDO="" @@ -74,9 +78,11 @@ verify_sha256() { fi } -# nvm is a bash script; run a command in a bash with nvm loaded +# nvm is a bash script; run a command in a bash with nvm loaded. +# --no-use: otherwise loading nvm here switches to the version .nvmrc +# names, and fails silently while that version is not installed yet. nvm_sh() { - bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" + bash -c ". \"\$HOME/.nvm/nvm.sh\" --no-use && $*" } ensure_nvm() { @@ -122,6 +128,48 @@ install_js_deps() { fi } +# Unpack protoc's release archive for this platform into bin/protoc, after +# checking the archive's sha256, unless bin/protoc already holds the pinned +# version. +ensure_protoc() { + dir="$ROOT/bin/protoc" + if [ "$("$dir/bin/protoc" --version 2>/dev/null)" = \ + "libprotoc $PROTOC_VERSION" ]; then + return 0 + fi + case "$(uname -s) $(uname -m)" in + "Linux x86_64") + platform="linux-x86_64" + sha256="c0040ea9aef08fdeb2c74ca609b18d5fdbfc44ea0042fcfbfb38860d35f7dd66" + ;; + "Linux aarch64" | "Linux arm64") + platform="linux-aarch_64" + sha256="15aa988f4a6090636525ec236a8e4b3aab41eef402751bd5bb2df6afd9b7b5a5" + ;; + "Darwin x86_64") + platform="osx-x86_64" + sha256="a49bec10d039e902d3b43e49938c42526f90011467609864fa6386ac4014da58" + ;; + "Darwin arm64") + platform="osx-aarch_64" + sha256="726297dcfed58592fd35620a5a6246ae020c39e88f3fd4cb1827df7bcf3dfcf1" + ;; + *) + echo "bootstrap: no protoc archive pinned for $(uname -s) $(uname -m)" >&2 + exit 1 + ;; + esac + if missing curl; then pkg_install curl curl curl curl; fi + if missing unzip; then pkg_install unzip unzip unzip unzip; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/protoc.zip" \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" + verify_sha256 "$tmp/protoc.zip" "$sha256" + rm -rf "$dir" + unzip -q "$tmp/protoc.zip" -d "$dir" + rm -rf "$tmp" +} + main() { cd "$ROOT" @@ -132,8 +180,10 @@ main() { # ---- JS / docs repos ---- # This is a Go repo, but node and yarn are required anyway: prettier # formats the Markdown and JSON, and script/fmt-check verifies it. - # The version is pinned by package.json/yarn.lock, whose integrity - # hashes --frozen-lockfile enforces. + # The version is pinned by package.json/yarn.lock: yarn checks every + # package it fetches against its yarn.lock integrity hash, and + # --frozen-lockfile fails instead of rewriting a yarn.lock that no + # longer matches package.json. ensure_node ensure_yarn install_js_deps @@ -142,6 +192,10 @@ main() { if missing go; then pkg_install go golang go go; fi # No golangci-lint: script/lint runs it in Docker only. go mod download + # gofumpt and protoc-gen-go: bin/tools/go.mod pins them, and + # script/gofumpt and script/generate build them from there. + (cd "$ROOT/bin/tools" && go mod download) + ensure_protoc # ---- Python repos ---- # if missing python3; then pkg_install python3 python3 python3 python3; fi diff --git a/script/generate b/script/generate index 4ee62ff..6d23b0b 100755 --- a/script/generate +++ b/script/generate @@ -4,26 +4,17 @@ # regenerates mf.pb.go: it is committed, so building and checking need no # protoc. A test fails while mf.proto no longer matches the recorded hash. # -# Needs exactly the protoc and protoc-gen-go versions named in the header of -# the committed mf.pb.go (README.md says how to install them). Another -# version writes a different mf.pb.go, so the script refuses to run. +# Runs the protoc that script/bootstrap unpacks into bin/protoc, and the +# protoc-gen-go that bin/tools/go.mod pins. Another version of either +# writes a different mf.pb.go. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -# protoc 33.4 names itself v6.33.4 in the mf.pb.go header. +# The protoc version script/bootstrap installs. protoc 33.4 names itself +# v6.33.4 in the mf.pb.go header. PROTOC_VERSION="33.4" -PROTOC_GEN_GO_VERSION="v1.36.11" - -# require_version -require_version() { - actual="$("$1" --version 2>/dev/null || true)" - if [ "$actual" != "$2" ]; then - echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2 - echo " README.md says how to install it." >&2 - exit 1 - fi -} +PROTOC="$ROOT/bin/protoc/bin/protoc" # sha256 : print " ", with sha256sum, or with shasum # where there is no sha256sum. @@ -39,16 +30,24 @@ sha256() { } main() { + # A bin/protoc left from before the pin moved fails here, until + # script/bootstrap replaces it. + actual="$("$PROTOC" --version 2>/dev/null || true)" + if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then + echo "generate: needs protoc $PROTOC_VERSION in bin/protoc," \ + "found: ${actual:-none}; run script/bootstrap" >&2 + exit 1 + fi + # `go tool -n` builds protoc-gen-go from bin/tools and prints where the + # binary is, without running it. + plugin="$(cd "$ROOT/bin/tools" && go tool -n protoc-gen-go)" + cd "$ROOT/mfer" - # `go install` puts protoc-gen-go in $(go env GOPATH)/bin, which is - # often not on PATH. - PATH="$PATH:$(go env GOPATH)/bin" - require_version protoc "libprotoc $PROTOC_VERSION" - require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION" # Hashed before regenerating, so a missing hash tool stops the script # before it changes anything. Regenerating leaves mf.proto as it is. proto_hash="$(sha256 mf.proto)" - go generate . + "$PROTOC" --plugin=protoc-gen-go="$plugin" \ + --go_out=paths=source_relative:. ./mf.proto echo "$proto_hash" >mf.proto.sha256 } diff --git a/script/gofumpt b/script/gofumpt index 1622d46..0422ac5 100755 --- a/script/gofumpt +++ b/script/gofumpt @@ -10,10 +10,10 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -# gofumpt v0.12.0, 2026-10-04. `go run` fetches and builds exactly this -# version, so neither a developer machine nor the lint image needs -# gofumpt installed. -GOFUMPT="mvdan.cc/gofumpt@v0.12.0" +# The gofumpt version is the one bin/tools/go.mod pins. `go tool` run in +# bin/tools builds it from source checked against the hashes in +# bin/tools/go.sum, so neither a developer machine nor the lint image needs +# it installed. usage() { echo "usage: script/gofumpt --write|--check" >&2 @@ -22,15 +22,15 @@ usage() { main() { [ "$#" -eq 1 ] || usage - cd "$ROOT" - # Every Go file in the repo. gofumpt holds generated files, such as - # mfer/mf.pb.go, to gofmt's rules only. + cd "$ROOT/bin/tools" + # Every Go file in the repo, from $ROOT down. gofumpt holds generated + # files, such as mfer/mf.pb.go, to gofmt's rules only. case "$1" in - --write) go run "$GOFUMPT" -l -w . ;; + --write) go tool gofumpt -l -w "$ROOT" ;; --check) # Own line: a failing command inside `[ -n "$(...)" ]` does # not trip `set -e`, so a gofumpt that never ran would pass. - unformatted="$(go run "$GOFUMPT" -l .)" + unformatted="$(go tool gofumpt -l "$ROOT")" if [ -n "$unformatted" ]; then echo "gofumpt: files need formatting (run make fmt):" >&2 echo "$unformatted" >&2 diff --git a/script/prettier b/script/prettier index b2f66a3..9b1079c 100755 --- a/script/prettier +++ b/script/prettier @@ -18,24 +18,9 @@ usage() { exit 2 } -# Prefer the version pinned by package.json/yarn.lock so that CI and -# developer machines format identically. Fall back to a prettier on PATH, -# but say so, because a different version formats differently. -find_prettier() { - if [ -x "$ROOT/node_modules/.bin/prettier" ]; then - printf '%s\n' "$ROOT/node_modules/.bin/prettier" - return 0 - fi - if command -v prettier >/dev/null 2>&1; then - echo "prettier: node_modules/.bin/prettier is absent; using the" \ - "prettier on PATH, which may be a different version than the" \ - "one pinned in package.json. Run script/bootstrap to install" \ - "the pinned version." >&2 - command -v prettier - return 0 - fi - return 1 -} +# Only the prettier yarn installed from yarn.lock, never one on PATH: a +# different version formats differently. +PRETTIER="$ROOT/node_modules/.bin/prettier" main() { [ "$#" -eq 1 ] || usage @@ -46,10 +31,26 @@ main() { cd "$ROOT" - if ! prettier_bin="$(find_prettier)"; then - echo "prettier: not found." >&2 - echo " Install it with: script/bootstrap" >&2 - echo " (installs the version pinned in package.json/yarn.lock)" >&2 + # Where there is no node on PATH, script/bootstrap installs the version + # .nvmrc names through nvm, which keeps it in this directory. + if ! command -v node >/dev/null 2>&1; then + PATH="$HOME/.nvm/versions/node/v$(cat .nvmrc)/bin:$PATH" + fi + if ! command -v node >/dev/null 2>&1; then + echo "prettier: node is missing; run script/bootstrap" >&2 + exit 1 + fi + + # node_modules keeps the old prettier after package.json moves to a new + # one, until script/bootstrap runs again, so compare the two. + if ! installed="$("$PRETTIER" --version 2>/dev/null)"; then + echo "prettier: not installed; run script/bootstrap" >&2 + exit 1 + fi + pinned="$(node -p 'require("./package.json").devDependencies.prettier')" + if [ "$installed" != "$pinned" ]; then + echo "prettier: package.json pins $pinned but $installed is" \ + "installed; run script/bootstrap" >&2 exit 1 fi @@ -62,8 +63,7 @@ main() { # patterns always match at least one tracked file (README.md, # package.json), so an empty match means the glob broke, and prettier # erroring out is exactly what we want rather than a vacuous pass. - "$prettier_bin" "$mode" "**/*.md" - "$prettier_bin" "$mode" "**/*.json" + "$PRETTIER" "$mode" "**/*.md" "**/*.json" } main "$@"