diff --git a/Dockerfile b/Dockerfile index 26a9592..4ca4ac3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,6 @@ RUN go mod download COPY . . -# Touch .pb.go so make does not try to regenerate via protoc (file is committed) -RUN touch mfer/mf.pb.go - # Go half of fmt-check only: this image has no node, so no prettier. The # markdown half runs in the mdfmt stage below. RUN make fmt-check-go @@ -46,9 +43,6 @@ RUN go mod download COPY . . -# Touch .pb.go so make does not try to regenerate via protoc (file is committed) -RUN touch mfer/mf.pb.go - RUN make test # A build context sent as a tar archive, as upaas sends it, keeps its files' diff --git a/Makefile b/Makefile index 38d19df..8ba3c71 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ export DOCKER_BUILDKIT := 1 export PROGRESS_NO_TRUNC := 1 GOPATH := $(shell go env GOPATH) export PATH := $(PATH):$(GOPATH)/bin -PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum ARCH := $(shell uname -m) GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown) @@ -13,7 +12,7 @@ GOLDFLAGS += -X main.Version=$(VERSION) GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD) GOFLAGS := -ldflags "$(GOLDFLAGS)" -.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme +.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme generate default: fmt test @@ -35,9 +34,6 @@ test: fuzz: @script/fuzz -$(PROTOC_GEN_GO): - test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 - fixme: @grep -nir fixme . | grep -v Makefile @@ -58,15 +54,14 @@ fmt-check-md: hooks: @script/install-precommit -mfer/mf.pb.go: mfer/mf.proto - cd mfer && go generate . +generate: + @script/generate -bin/mfer: $(SOURCEFILES) mfer/mf.pb.go - protoc --version +bin/mfer: $(SOURCEFILES) cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) . clean: - rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage + rm -rfv bin/mfer cmd/mfer/mfer *.dockerimage fmt: @script/fmt diff --git a/README.md b/README.md index 7234ed5..dc9708e 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,19 @@ provide: `script/bootstrap`, then `script/install-precommit` - `script/projectname` — output the project name (`mfer`); used by other scripts such as `script/docker` -- `script/test` — run the test suite (`go test`), regenerating the protobuf code - first if it is stale +- `script/test` — run the test suite (`go test`); one test fails when + `mfer/mf.proto` no longer matches the hash `script/generate` recorded +- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from + `mfer/mf.proto` and record the hash of that `mfer/mf.proto` in + `mfer/mf.proto.sha256`; the only thing that regenerates the committed + `mfer/mf.pb.go`. It needs the exact versions that wrote the committed file, + and refuses to run with any other: `protoc` 33.4 (unpack + `protoc-33.4-.zip` from + [its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4) + and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11 + (`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which + installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to + `PATH`) - `script/fuzz` — fuzz the manifest parser for one minute; run by hand (`make fuzz`), never by CI, while `script/test` runs its committed seed corpus as ordinary tests diff --git a/mfer/mf.proto.sha256 b/mfer/mf.proto.sha256 new file mode 100644 index 0000000..619c842 --- /dev/null +++ b/mfer/mf.proto.sha256 @@ -0,0 +1 @@ +103901c42b94396aa7ae128fd503ef693a4b7a03b2169481f25fda3d2c254e00 mf.proto diff --git a/mfer/mf_test.go b/mfer/mf_test.go new file mode 100644 index 0000000..f481176 --- /dev/null +++ b/mfer/mf_test.go @@ -0,0 +1,29 @@ +package mfer_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// mf.pb.go is generated from mf.proto and committed. `make generate` +// records the hash of the mf.proto it generated from in mf.proto.sha256. +func TestGeneratedCodeMatchesProto(t *testing.T) { + t.Parallel() + + proto, err := os.ReadFile("mf.proto") + require.NoError(t, err) + + recorded, err := os.ReadFile("mf.proto.sha256") + require.NoError(t, err) + + recordedHash, _, _ := strings.Cut(string(recorded), " ") + sum := sha256.Sum256(proto) + require.Equal(t, recordedHash, hex.EncodeToString(sum[:]), + "mfer/mf.proto has changed since mfer/mf.pb.go was generated "+ + "from it: run `make generate` and commit the result") +} diff --git a/script/fmt b/script/fmt index b9aa31d..fd81efa 100755 --- a/script/fmt +++ b/script/fmt @@ -5,19 +5,8 @@ set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" -# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale -# (mirrors the old Makefile prerequisite; the generated file is -# committed, so this is normally a no-op). -ensure_pb() { - if [ ! -f mfer/mf.pb.go ] || - [ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then - (cd mfer && go generate .) - fi -} - main() { cd "$ROOT" - ensure_pb gofumpt -l -w mfer internal cmd # Markdown and JSON, over the same file set script/fmt-check verifies. "$SCRIPT_DIR/prettier" --write diff --git a/script/fmt-check-go b/script/fmt-check-go index c079f59..e0248d4 100755 --- a/script/fmt-check-go +++ b/script/fmt-check-go @@ -6,19 +6,8 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale -# (mirrors the old Makefile prerequisite; the generated file is -# committed, so this is normally a no-op). -ensure_pb() { - if [ ! -f mfer/mf.pb.go ] || - [ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then - (cd mfer && go generate .) - fi -} - main() { cd "$ROOT" - ensure_pb if [ -n "$(gofmt -l .)" ]; then echo "gofmt: files need formatting:" >&2 gofmt -l . >&2 diff --git a/script/generate b/script/generate new file mode 100755 index 0000000..6d5c649 --- /dev/null +++ b/script/generate @@ -0,0 +1,52 @@ +#!/bin/sh +# script/generate: regenerate mfer/mf.pb.go from mfer/mf.proto, and record +# the hash of that mf.proto in mfer/mf.proto.sha256. Nothing else +# regenerates mf.pb.go: it is committed, so building and checking need no +# protoc. A test fails while mf.proto no longer matches the recorded hash. +# +# Needs exactly the protoc and protoc-gen-go versions named in the header of +# the committed mf.pb.go (README.md says how to install them). Another +# version writes a different mf.pb.go, so the script refuses to run. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# protoc 33.4 names itself v6.33.4 in the mf.pb.go header. +PROTOC_VERSION="33.4" +PROTOC_GEN_GO_VERSION="v1.36.11" + +# require_version +require_version() { + actual="$("$1" --version 2>/dev/null || true)" + if [ "$actual" != "$2" ]; then + echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2 + echo " README.md says how to install it." >&2 + exit 1 + fi +} + +# sha256 : print " ", with sha256sum, or with shasum +# where there is no sha256sum. +sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" + else + echo "generate: needs sha256sum or shasum on PATH" >&2 + exit 1 + fi +} + +main() { + cd "$ROOT/mfer" + require_version protoc "libprotoc $PROTOC_VERSION" + require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION" + # Hashed before regenerating, so a missing hash tool stops the script + # before it changes anything. Regenerating leaves mf.proto as it is. + proto_hash="$(sha256 mf.proto)" + go generate . + echo "$proto_hash" >mf.proto.sha256 +} + +main "$@" diff --git a/script/test b/script/test index a165bc4..2d63e2d 100755 --- a/script/test +++ b/script/test @@ -4,19 +4,8 @@ set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale -# (mirrors the old Makefile prerequisite; the generated file is -# committed, so this is normally a no-op). -ensure_pb() { - if [ ! -f mfer/mf.pb.go ] || - [ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then - (cd mfer && go generate .) - fi -} - main() { cd "$ROOT" - ensure_pb go test -timeout 30s -race -cover ./... || { echo "--- Rerunning with -v for details ---"