Pin the remaining developer tool installs (closes #68) #155

Merged
clawbot merged 1 commits from issue-68-pin-tools into next 2026-10-04 20:48:52 +02:00
13 changed files with 182 additions and 81 deletions
Showing only changes of commit 239cccd45a - Show all commits
+3
View File
@@ -61,3 +61,6 @@
# This repo's own host-built binary (make build).
/bin/mfer
# The protoc script/bootstrap unpacks for script/generate.
/bin/protoc
+1
View File
@@ -1,4 +1,5 @@
/bin/mfer
/bin/protoc/
/tmp
/node_modules/
+1
View File
@@ -0,0 +1 @@
22.17.0
+3 -3
View File
@@ -10,7 +10,7 @@ COPY . .
# Go half of fmt-check only: this image has no node, so no prettier. The
# markdown half runs in the mdfmt stage below. The image has no gofumpt
# either; script/gofumpt builds the version it pins with `go run`.
# either; script/gofumpt builds the version bin/tools/go.mod pins.
RUN script/gofumpt --check
# The linter directly, not `make lint`: script/lint builds this stage, and
# there is no docker inside this build.
@@ -18,7 +18,7 @@ RUN golangci-lint run --config .golangci.yml ./...
# Markdown/JSON format stage — prettier needs node, which the Go images
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins.
# 22.17.0 and yarn 1.22.22, the versions .nvmrc and script/bootstrap pin.
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
WORKDIR /src
@@ -31,7 +31,7 @@ COPY . .
RUN script/prettier --check
# Build stage — tests and compilation
# golang:1.23 (2026-03-14)
# golang:1.23.12, 2026-03-14
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
+18 -15
View File
@@ -67,9 +67,13 @@ standard: normalized scripts in `script/` are the entrypoints for the
development workflow, and the Makefile targets are thin shims that call them. We
provide:
- `script/bootstrap` — install all dependencies (Go, Go module download, and
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
idempotently; golangci-lint is not installed, it runs only in Docker
- `script/bootstrap` — install all dependencies, idempotently: Go and the
modules of both `go.mod` and `bin/tools/go.mod`; node (the version `.nvmrc`
names, through nvm when there is no node on `PATH`) and yarn, plus the
prettier version pinned in `package.json`/`yarn.lock`; and `protoc` 33.4,
unpacked into `bin/protoc` from its release archive once the archive matches
the sha256 the script holds for this platform. golangci-lint is not installed,
it runs only in Docker
- `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit`
- `script/projectname` — output the project name (`mfer`); used by other scripts
@@ -82,14 +86,10 @@ provide:
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file,
and refuses to run with any other: `protoc` 33.4 (unpack
`protoc-33.4-<platform>.zip` from
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
installs it in `$(go env GOPATH)/bin`; `script/generate` adds that directory
to `PATH`)
`mfer/mf.pb.go`. It runs the `protoc` that `script/bootstrap` unpacks into
`bin/protoc`, refusing any version but 33.4, and the `protoc-gen-go` that
`bin/tools/go.mod` pins, which `go tool` builds from source checked against
the hashes in `bin/tools/go.sum`
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
as ordinary tests
@@ -99,14 +99,17 @@ provide:
- `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and
`script/prettier --write`
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
given mode, `--write` or `--check`, at the one version it pins (built on
demand by `go run`, so nothing installs it); `script/fmt`, `script/fmt-check`
given mode, `--write` or `--check`, at the version `bin/tools/go.mod` pins
(built on demand by `go tool` from source checked against the hashes in
`bin/tools/go.sum`, so nothing installs it); `script/fmt`, `script/fmt-check`
and the Docker lint stage all go through it, so they cannot disagree about Go
formatting
- `script/prettier` — run prettier over the repository's canonical file set
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
`--check`; the single definition of that file set, so `script/fmt` and
`script/fmt-check` cannot disagree about it
`--check`, with the prettier version `yarn.lock` pins, run by the node on
`PATH` or else the one `script/bootstrap` installed through nvm; the single
definition of that file set, so `script/fmt` and `script/fmt-check` cannot
disagree about it
- `script/fmt-check` — check formatting without writing:
`script/gofumpt --check` plus `script/prettier --check`
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
+22
View File
@@ -0,0 +1,22 @@
// The developer tools this repo runs with `go tool`: gofumpt for
// script/gofumpt and protoc-gen-go for script/generate. Kept out of the mfer
// module so they add nothing to what mfer's users download. `go tool` builds
// exactly the source whose hashes go.sum here records.
module sneak.berlin/go/mfer/bin/tools
go 1.26.0
tool (
google.golang.org/protobuf/cmd/protoc-gen-go
mvdan.cc/gofumpt
)
require (
golang.org/x/mod v0.40.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/tools v0.49.0 // indirect
// protoc-gen-go v1.36.11, 2026-10-04
google.golang.org/protobuf v1.36.11 // indirect
// gofumpt v0.12.0, 2026-10-04
mvdan.cc/gofumpt v0.12.0 // indirect
)
+22
View File
@@ -0,0 +1,22 @@
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
mvdan.cc/gofumpt v0.12.0 h1:1Lbudkz2kpM9Cjz2pL4M19u7q+GaEhCTNf7N9mfpcho=
mvdan.cc/gofumpt v0.12.0/go.mod h1:SmBHHrljiZu/uoypeKup3rFzP6eoC9UwCp2iH5E3jZA=
-3
View File
@@ -1,3 +0,0 @@
package mfer
//go:generate protoc ./mf.proto --go_out=paths=source_relative:.
-1
View File
@@ -1,6 +1,5 @@
{
"name": "mfer",
"version": "0.1.0",
"private": true,
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
"license": "WTFPL",
+59 -5
View File
@@ -13,11 +13,15 @@ set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
NODE_VERSION="22.17.0"
# The node version is in .nvmrc, where script/prettier reads it too.
NODE_VERSION="$(cat "$ROOT/.nvmrc")"
NVM_VERSION="0.40.3"
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
YARN_VERSION="1.22.22"
# protoc v33.4, 2026-10-04, for script/generate. The sha256 of each
# platform's release archive is in ensure_protoc.
PROTOC_VERSION="33.4"
PKGMGR=""
SUDO=""
@@ -74,9 +78,11 @@ verify_sha256() {
fi
}
# nvm is a bash script; run a command in a bash with nvm loaded
# nvm is a bash script; run a command in a bash with nvm loaded.
# --no-use: otherwise loading nvm here switches to the version .nvmrc
# names, and fails silently while that version is not installed yet.
nvm_sh() {
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
bash -c ". \"\$HOME/.nvm/nvm.sh\" --no-use && $*"
}
ensure_nvm() {
@@ -122,6 +128,48 @@ install_js_deps() {
fi
}
# Unpack protoc's release archive for this platform into bin/protoc, after
# checking the archive's sha256, unless bin/protoc already holds the pinned
# version.
ensure_protoc() {
dir="$ROOT/bin/protoc"
if [ "$("$dir/bin/protoc" --version 2>/dev/null)" = \
"libprotoc $PROTOC_VERSION" ]; then
return 0
fi
case "$(uname -s) $(uname -m)" in
"Linux x86_64")
platform="linux-x86_64"
sha256="c0040ea9aef08fdeb2c74ca609b18d5fdbfc44ea0042fcfbfb38860d35f7dd66"
;;
"Linux aarch64" | "Linux arm64")
platform="linux-aarch_64"
sha256="15aa988f4a6090636525ec236a8e4b3aab41eef402751bd5bb2df6afd9b7b5a5"
;;
"Darwin x86_64")
platform="osx-x86_64"
sha256="a49bec10d039e902d3b43e49938c42526f90011467609864fa6386ac4014da58"
;;
"Darwin arm64")
platform="osx-aarch_64"
sha256="726297dcfed58592fd35620a5a6246ae020c39e88f3fd4cb1827df7bcf3dfcf1"
;;
*)
echo "bootstrap: no protoc archive pinned for $(uname -s) $(uname -m)" >&2
exit 1
;;
esac
if missing curl; then pkg_install curl curl curl curl; fi
if missing unzip; then pkg_install unzip unzip unzip unzip; fi
tmp="$(mktemp -d)"
curl -fsSL -o "$tmp/protoc.zip" \
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip"
verify_sha256 "$tmp/protoc.zip" "$sha256"
rm -rf "$dir"
unzip -q "$tmp/protoc.zip" -d "$dir"
rm -rf "$tmp"
}
main() {
cd "$ROOT"
@@ -132,8 +180,10 @@ main() {
# ---- JS / docs repos ----
# This is a Go repo, but node and yarn are required anyway: prettier
# formats the Markdown and JSON, and script/fmt-check verifies it.
# The version is pinned by package.json/yarn.lock, whose integrity
# hashes --frozen-lockfile enforces.
# The version is pinned by package.json/yarn.lock: yarn checks every
# package it fetches against its yarn.lock integrity hash, and
# --frozen-lockfile fails instead of rewriting a yarn.lock that no
# longer matches package.json.
ensure_node
ensure_yarn
install_js_deps
@@ -142,6 +192,10 @@ main() {
if missing go; then pkg_install go golang go go; fi
# No golangci-lint: script/lint runs it in Docker only.
go mod download
# gofumpt and protoc-gen-go: bin/tools/go.mod pins them, and
# script/gofumpt and script/generate build them from there.
(cd "$ROOT/bin/tools" && go mod download)
ensure_protoc
# ---- Python repos ----
# if missing python3; then pkg_install python3 python3 python3 python3; fi
+20 -21
View File
@@ -4,26 +4,17 @@
# regenerates mf.pb.go: it is committed, so building and checking need no
# protoc. A test fails while mf.proto no longer matches the recorded hash.
#
# Needs exactly the protoc and protoc-gen-go versions named in the header of
# the committed mf.pb.go (README.md says how to install them). Another
# version writes a different mf.pb.go, so the script refuses to run.
# Runs the protoc that script/bootstrap unpacks into bin/protoc, and the
# protoc-gen-go that bin/tools/go.mod pins. Another version of either
# writes a different mf.pb.go.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
# The protoc version script/bootstrap installs. protoc 33.4 names itself
# v6.33.4 in the mf.pb.go header.
PROTOC_VERSION="33.4"
PROTOC_GEN_GO_VERSION="v1.36.11"
# require_version <command> <its exact --version output>
require_version() {
actual="$("$1" --version 2>/dev/null || true)"
if [ "$actual" != "$2" ]; then
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
echo " README.md says how to install it." >&2
exit 1
fi
}
PROTOC="$ROOT/bin/protoc/bin/protoc"
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
# where there is no sha256sum.
@@ -39,16 +30,24 @@ sha256() {
}
main() {
# A bin/protoc left from before the pin moved fails here, until
# script/bootstrap replaces it.
actual="$("$PROTOC" --version 2>/dev/null || true)"
if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then
echo "generate: needs protoc $PROTOC_VERSION in bin/protoc," \
"found: ${actual:-none}; run script/bootstrap" >&2
exit 1
fi
# `go tool -n` builds protoc-gen-go from bin/tools and prints where the
# binary is, without running it.
plugin="$(cd "$ROOT/bin/tools" && go tool -n protoc-gen-go)"
cd "$ROOT/mfer"
# `go install` puts protoc-gen-go in $(go env GOPATH)/bin, which is
# often not on PATH.
PATH="$PATH:$(go env GOPATH)/bin"
require_version protoc "libprotoc $PROTOC_VERSION"
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
# Hashed before regenerating, so a missing hash tool stops the script
# before it changes anything. Regenerating leaves mf.proto as it is.
proto_hash="$(sha256 mf.proto)"
go generate .
"$PROTOC" --plugin=protoc-gen-go="$plugin" \
--go_out=paths=source_relative:. ./mf.proto
echo "$proto_hash" >mf.proto.sha256
}
+9 -9
View File
@@ -10,10 +10,10 @@ set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# gofumpt v0.12.0, 2026-10-04. `go run` fetches and builds exactly this
# version, so neither a developer machine nor the lint image needs
# gofumpt installed.
GOFUMPT="mvdan.cc/gofumpt@v0.12.0"
# The gofumpt version is the one bin/tools/go.mod pins. `go tool` run in
# bin/tools builds it from source checked against the hashes in
# bin/tools/go.sum, so neither a developer machine nor the lint image needs
# it installed.
usage() {
echo "usage: script/gofumpt --write|--check" >&2
@@ -22,15 +22,15 @@ usage() {
main() {
[ "$#" -eq 1 ] || usage
cd "$ROOT"
# Every Go file in the repo. gofumpt holds generated files, such as
# mfer/mf.pb.go, to gofmt's rules only.
cd "$ROOT/bin/tools"
# Every Go file in the repo, from $ROOT down. gofumpt holds generated
# files, such as mfer/mf.pb.go, to gofmt's rules only.
case "$1" in
--write) go run "$GOFUMPT" -l -w . ;;
--write) go tool gofumpt -l -w "$ROOT" ;;
--check)
# Own line: a failing command inside `[ -n "$(...)" ]` does
# not trip `set -e`, so a gofumpt that never ran would pass.
unformatted="$(go run "$GOFUMPT" -l .)"
unformatted="$(go tool gofumpt -l "$ROOT")"
if [ -n "$unformatted" ]; then
echo "gofumpt: files need formatting (run make fmt):" >&2
echo "$unformatted" >&2
+24 -24
View File
@@ -18,24 +18,9 @@ usage() {
exit 2
}
# Prefer the version pinned by package.json/yarn.lock so that CI and
# developer machines format identically. Fall back to a prettier on PATH,
# but say so, because a different version formats differently.
find_prettier() {
if [ -x "$ROOT/node_modules/.bin/prettier" ]; then
printf '%s\n' "$ROOT/node_modules/.bin/prettier"
return 0
fi
if command -v prettier >/dev/null 2>&1; then
echo "prettier: node_modules/.bin/prettier is absent; using the" \
"prettier on PATH, which may be a different version than the" \
"one pinned in package.json. Run script/bootstrap to install" \
"the pinned version." >&2
command -v prettier
return 0
fi
return 1
}
# Only the prettier yarn installed from yarn.lock, never one on PATH: a
# different version formats differently.
PRETTIER="$ROOT/node_modules/.bin/prettier"
main() {
[ "$#" -eq 1 ] || usage
@@ -46,10 +31,26 @@ main() {
cd "$ROOT"
if ! prettier_bin="$(find_prettier)"; then
echo "prettier: not found." >&2
echo " Install it with: script/bootstrap" >&2
echo " (installs the version pinned in package.json/yarn.lock)" >&2
# Where there is no node on PATH, script/bootstrap installs the version
# .nvmrc names through nvm, which keeps it in this directory.
if ! command -v node >/dev/null 2>&1; then
PATH="$HOME/.nvm/versions/node/v$(cat .nvmrc)/bin:$PATH"
fi
if ! command -v node >/dev/null 2>&1; then
echo "prettier: node is missing; run script/bootstrap" >&2
exit 1
fi
# node_modules keeps the old prettier after package.json moves to a new
# one, until script/bootstrap runs again, so compare the two.
if ! installed="$("$PRETTIER" --version 2>/dev/null)"; then
echo "prettier: not installed; run script/bootstrap" >&2
exit 1
fi
pinned="$(node -p 'require("./package.json").devDependencies.prettier')"
if [ "$installed" != "$pinned" ]; then
echo "prettier: package.json pins $pinned but $installed is" \
"installed; run script/bootstrap" >&2
exit 1
fi
@@ -62,8 +63,7 @@ main() {
# patterns always match at least one tracked file (README.md,
# package.json), so an empty match means the glob broke, and prettier
# erroring out is exactly what we want rather than a vacuous pass.
"$prettier_bin" "$mode" "**/*.md"
"$prettier_bin" "$mode" "**/*.json"
"$PRETTIER" "$mode" "**/*.md" "**/*.json"
}
main "$@"