1 Commits
Author SHA1 Message Date
sneak df985e44dc fetch: destination directory, skip files already present, save the manifest, require a signer (closes #101)
check / check (push) Failing after 3s
fetch takes --dest (default .) and writes every file there through the
existing symlink and hard-link guards, which now work relative to that
directory. A file already there with the listed size and hash is
skipped; a leftover temp file is still replaced. Once every file
verifies, the manifest is saved as index.mf through the same temp file
and rename, so check runs on the result; a manifest that lists index.mf
or its temp name at the top of the tree is refused, since saving would
replace that file. --require-signature is shared with check and
enforced through verifyRequiredSigner. Both refusals come before any
file is downloaded or anything is written.

Model: opus-5-5
2026-10-04 16:05:22 +00:00
29 changed files with 468 additions and 620 deletions
+3 -5
View File
@@ -59,8 +59,6 @@
**/.vscode
**/*.sublime-*
# This repo's own host-built binary (make build).
/bin/mfer
# The protoc script/bootstrap unpacks for script/generate.
/bin/protoc
# This repo's own host-built archives (Makefile).
*.tmp
*.dockerimage
+5 -1
View File
@@ -1,7 +1,11 @@
/bin/mfer
/bin/protoc/
/tmp
/node_modules/
*.tmp
*.dockerimage
/vendor
vendor.tzst
modcache.tzst
# Generated manifest files
/index.mf
-1
View File
@@ -1 +0,0 @@
22.17.0
+5 -5
View File
@@ -10,15 +10,15 @@ COPY . .
# Go half of fmt-check only: this image has no node, so no prettier. The
# markdown half runs in the mdfmt stage below. The image has no gofumpt
# either; script/gofumpt builds the version bin/tools/go.mod pins.
RUN script/gofumpt --check
# either; script/gofumpt builds the version it pins with `go run`.
RUN make fmt-check-go
# The linter directly, not `make lint`: script/lint builds this stage, and
# there is no docker inside this build.
RUN golangci-lint run --config .golangci.yml ./...
# Markdown/JSON format stage — prettier needs node, which the Go images
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
# 22.17.0 and yarn 1.22.22, the versions .nvmrc and script/bootstrap pin.
# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins.
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
WORKDIR /src
@@ -31,7 +31,7 @@ COPY . .
RUN script/prettier --check
# Build stage — tests and compilation
# golang:1.23.12, 2026-03-14
# golang:1.23 (2026-03-14)
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
@@ -64,7 +64,7 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
exit 1; \
fi; \
cd cmd/mfer && \
CGO_ENABLED=0 go build -ldflags "-X main.Gitrev=$version" -o /mfer .
CGO_ENABLED=0 go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
# Fail unless /mfer is statically linked: scratch has no C library to run it.
RUN ldd /mfer 2>&1 | grep -q 'not a dynamic executable'
+73 -18
View File
@@ -1,8 +1,20 @@
.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build generate fuzz
export DOCKER_BUILDKIT := 1
export PROGRESS_NO_TRUNC := 1
GOPATH := $(shell go env GOPATH)
export PATH := $(PATH):$(GOPATH)/bin
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
ARCH := $(shell uname -m)
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
APPNAME := mfer
VERSION := 0.1.0
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
GOLDFLAGS += -X main.Version=$(VERSION)
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
GOFLAGS := -ldflags "$(GOLDFLAGS)"
# Makefile targets are thin shims; the implementations live in script/
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
# of README.md).
.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme generate
default: fmt test
bootstrap:
@script/bootstrap
@@ -10,32 +22,75 @@ bootstrap:
setup:
@script/setup
run: ./bin/mfer
./$<
./$< gen
ci: test
test:
@script/test
lint:
@script/lint
fuzz:
@script/fuzz
fmt:
@script/fmt
fmt-check:
@script/fmt-check
fixme:
@grep -nir fixme . | grep -v Makefile
check:
@script/check
docker:
@script/docker
fmt-check:
@script/fmt-check
# Halves of fmt-check, for environments that have only one toolchain:
# the Docker lint stage has Go but no node, the markdown stage the reverse.
fmt-check-go:
@script/gofumpt --check
fmt-check-md:
@script/prettier --check
hooks:
@script/install-precommit
build:
@script/build
generate:
@script/generate
fuzz:
@script/fuzz
bin/mfer: $(SOURCEFILES)
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
clean:
rm -rfv bin/mfer cmd/mfer/mfer *.dockerimage
fmt:
@script/fmt
lint:
@script/lint
docker:
@script/docker
sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
docker save sneak/mfer | pv | zstdmt -19 > $@
du -sh $@
godoc:
open http://127.0.0.1:6060
godoc -http=:6060
vendor.tzst: go.mod go.sum
go mod tidy
go mod vendor
cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
rm -rf vendor
mv $@.tmp $@
modcache.tzst: go.mod go.sum
go mod tidy
cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
go mod download -x
cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
mv $@.tmp $@
+21 -30
View File
@@ -29,7 +29,7 @@ is committed, so no `protoc` toolchain is required:
```sh
git clone https://git.eeqj.de/sneak/mfer.git
cd mfer
make build
go build -o bin/mfer ./cmd/mfer
```
Generate a manifest for a directory tree, verify it later, and fetch a published
@@ -67,29 +67,26 @@ standard: normalized scripts in `script/` are the entrypoints for the
development workflow, and the Makefile targets are thin shims that call them. We
provide:
- `script/bootstrap` — install all dependencies, idempotently: Go and the
modules of both `go.mod` and `bin/tools/go.mod`; node (the version `.nvmrc`
names, through nvm when there is no node on `PATH`) and yarn, plus the
prettier version pinned in `package.json`/`yarn.lock`; and `protoc` 33.4,
unpacked into `bin/protoc` from its release archive once the archive matches
the sha256 the script holds for this platform. golangci-lint is not installed,
it runs only in Docker
- `script/bootstrap` — install all dependencies (Go, Go module download, and
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
idempotently; golangci-lint is not installed, it runs only in Docker
- `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit`
- `script/projectname` — output the project name (`mfer`); used by other scripts
such as `script/docker`
- `script/test` — run the test suite (`go test`); one test fails when
`mfer/mf.proto` no longer matches the hash `script/generate` recorded
- `script/build` (`make build`) — build the `mfer` binary into `bin/mfer`,
stamped with the revision `mfer version` prints: the output of
`git describe --tags --always --dirty`, as `script/docker` passes it
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
`mfer/mf.pb.go`. It runs the `protoc` that `script/bootstrap` unpacks into
`bin/protoc`, refusing any version but 33.4, and the `protoc-gen-go` that
`bin/tools/go.mod` pins, which `go tool` builds from source checked against
the hashes in `bin/tools/go.sum`
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file,
and refuses to run with any other: `protoc` 33.4 (unpack
`protoc-33.4-<platform>.zip` from
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to
`PATH`)
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
as ordinary tests
@@ -99,17 +96,14 @@ provide:
- `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and
`script/prettier --write`
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
given mode, `--write` or `--check`, at the version `bin/tools/go.mod` pins
(built on demand by `go tool` from source checked against the hashes in
`bin/tools/go.sum`, so nothing installs it); `script/fmt`, `script/fmt-check`
and the Docker lint stage all go through it, so they cannot disagree about Go
formatting
given mode, `--write` or `--check`, at the one version it pins (built on
demand by `go run`, so nothing installs it); `script/fmt`, `script/fmt-check`
and the Docker lint stage (`make fmt-check-go`) all go through it, so they
cannot disagree about Go formatting
- `script/prettier` — run prettier over the repository's canonical file set
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
`--check`, with the prettier version `yarn.lock` pins, run by the node on
`PATH` or else the one `script/bootstrap` installed through nvm; the single
definition of that file set, so `script/fmt` and `script/fmt-check` cannot
disagree about it
`--check`; the single definition of that file set, so `script/fmt` and
`script/fmt-check` cannot disagree about it
- `script/fmt-check` — check formatting without writing:
`script/gofumpt --check` plus `script/prettier --check`
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
@@ -271,12 +265,9 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
cryptographic integrity of downloaded files. A file already there with the
size and hash the manifest lists is skipped. Once every file is in place,
the manifest is saved there as `index.mf`, so `mfer check` can verify the
tree later. Each file is downloaded to a temp file beside it, such as
`.a.txt.tmp` for `a.txt`, then moved into place. A manifest is refused
before any file is downloaded if it lists a file where fetch writes
another: at the temp file of a listed file, or at `index.mf` or
`.index.mf.tmp` at the top of the tree. Names are compared in any letter
case.
tree later. A manifest that lists `index.mf` (in any letter case) or
`.index.mf.tmp` at the top of the tree is refused before any file is
downloaded, since saving the manifest would replace it.
- `mfer fetch --require-signature <fingerprint> https://example.com/stuff/`
- as above, but first refuses a manifest not signed by the key with that
fingerprint, as `mfer check --require-signature` does, before downloading
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -euo pipefail
main() {
if [[ -n "${GITREV:-}" ]]; then
echo "$GITREV"
else
git describe --tags --always --dirty=-dirty
fi
}
main "$@"
-22
View File
@@ -1,22 +0,0 @@
// The developer tools this repo runs with `go tool`: gofumpt for
// script/gofumpt and protoc-gen-go for script/generate. Kept out of the mfer
// module so they add nothing to what mfer's users download. `go tool` builds
// exactly the source whose hashes go.sum here records.
module sneak.berlin/go/mfer/bin/tools
go 1.26.0
tool (
google.golang.org/protobuf/cmd/protoc-gen-go
mvdan.cc/gofumpt
)
require (
golang.org/x/mod v0.40.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/tools v0.49.0 // indirect
// protoc-gen-go v1.36.11, 2026-10-04
google.golang.org/protobuf v1.36.11 // indirect
// gofumpt v0.12.0, 2026-10-04
mvdan.cc/gofumpt v0.12.0 // indirect
)
-22
View File
@@ -1,22 +0,0 @@
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
mvdan.cc/gofumpt v0.12.0 h1:1Lbudkz2kpM9Cjz2pL4M19u7q+GaEhCTNf7N9mfpcho=
mvdan.cc/gofumpt v0.12.0/go.mod h1:SmBHHrljiZu/uoypeKup3rFzP6eoC9UwCp2iH5E3jZA=
+9 -3
View File
@@ -9,21 +9,27 @@ require (
github.com/klauspost/compress v1.18.2
github.com/multiformats/go-multihash v0.2.3
github.com/spf13/afero v1.8.0
github.com/stretchr/testify v1.12.1
github.com/urfave/cli/v3 v3.14.0
github.com/stretchr/testify v1.8.1
github.com/urfave/cli/v2 v2.27.7
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
google.golang.org/protobuf v1.28.1
)
require (
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/kr/pretty v0.2.0 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/text v0.3.6 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.6 // indirect
)
+22 -6
View File
@@ -46,6 +46,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -132,7 +134,10 @@ github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBF
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
@@ -144,21 +149,30 @@ github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/urfave/cli/v3 v3.14.0 h1:a8414NQlHJs0c/iBsulKLzlES0n/lEAskbL2LKpU4/s=
github.com/urfave/cli/v3 v3.14.0/go.mod h1:vXn6HxPNccJSzQr2QvwVncOKrgYGIHU0HY5h8B2nQj4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -169,8 +183,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -458,9 +470,13 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+17 -26
View File
@@ -16,7 +16,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/spf13/afero"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
@@ -96,10 +96,8 @@ func findManifest(fs afero.Fs, dir string) (string, error) {
// fetchManifestToTemp downloads a manifest URL to a temporary file and
// returns the temp file path. The caller is responsible for removing it.
func (mfa *CLIApp) fetchManifestToTemp(
ctx context.Context, url string,
) (string, error) {
rc, fetchErr := mfa.openManifestReader(ctx, url)
func (mfa *CLIApp) fetchManifestToTemp(url string) (string, error) {
rc, fetchErr := mfa.openManifestReader(url)
if fetchErr != nil {
return "", fetchErr
}
@@ -210,15 +208,13 @@ func countCheckFailures(
// findExtraFiles reports files present on disk but absent from the
// manifest, and anything the search cannot read: each is a failure under
// --no-extra-files, otherwise a warning.
func findExtraFiles(
ctx context.Context, cmd *cli.Command, chk *mfer.Checker, failures *int64,
) error {
func findExtraFiles(ctx *cli.Context, chk *mfer.Checker, failures *int64) error {
extraResults := make(chan mfer.Result, 1)
extraDone := make(chan struct{})
go func() {
for result := range extraResults {
if cmd.Bool("no-extra-files") {
if ctx.Bool("no-extra-files") {
*failures++
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
@@ -230,7 +226,7 @@ func findExtraFiles(
close(extraDone)
}()
err := chk.FindExtraFiles(ctx, extraResults)
err := chk.FindExtraFiles(ctx.Context, extraResults)
if err != nil {
return fmt.Errorf("failed to check for extra files: %w", err)
}
@@ -242,9 +238,7 @@ func findExtraFiles(
// runCheck runs the manifest check with progress and result reporting
// and returns the number of failures.
func runCheck(
ctx context.Context, cmd *cli.Command, chk *mfer.Checker, showProgress bool,
) (int64, error) {
func runCheck(ctx *cli.Context, chk *mfer.Checker, showProgress bool) (int64, error) {
// Set up results channel
results := make(chan mfer.Result, 1)
@@ -270,7 +264,7 @@ func runCheck(
go countCheckFailures(results, &failures, done)
// Run check
err := chk.Check(ctx, results, progress)
err := chk.Check(ctx.Context, results, progress)
progressWg.Wait()
@@ -281,7 +275,7 @@ func runCheck(
// Wait for results processing to complete
<-done
err = findExtraFiles(ctx, cmd, chk, &failures)
err = findExtraFiles(ctx, chk, &failures)
if err != nil {
return 0, err
}
@@ -289,19 +283,17 @@ func runCheck(
return failures, nil
}
func (mfa *CLIApp) checkManifestOperation(
ctx context.Context, cmd *cli.Command,
) error {
func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Debug("checkManifestOperation()")
manifestPath, err := mfa.resolveManifestArg(cmd)
manifestPath, err := mfa.resolveManifestArg(ctx)
if err != nil {
return fmt.Errorf("check: %w", err)
}
// URL manifests need to be downloaded to a temp file for the checker
if isHTTPURL(manifestPath) {
tmpPath, tmpErr := mfa.fetchManifestToTemp(ctx, manifestPath)
tmpPath, tmpErr := mfa.fetchManifestToTemp(manifestPath)
if tmpErr != nil {
return fmt.Errorf("check: %w", tmpErr)
}
@@ -311,13 +303,12 @@ func (mfa *CLIApp) checkManifestOperation(
manifestPath = tmpPath
}
basePath := cmd.String("base")
showProgress := cmd.Bool("progress")
basePath := ctx.String("base")
showProgress := ctx.Bool("progress")
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
// Create checker
//nolint:contextcheck // mfer loads a manifest without a context
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: manifestPath,
BasePath: basePath,
@@ -328,9 +319,9 @@ func (mfa *CLIApp) checkManifestOperation(
}
// Check signature requirement
requiredSigner := cmd.String(flagRequireSignature)
requiredSigner := ctx.String(flagRequireSignature)
if requiredSigner != "" {
err = verifyRequiredSigner(ctx, chk, requiredSigner)
err = verifyRequiredSigner(ctx.Context, chk, requiredSigner)
if err != nil {
return err
}
@@ -339,7 +330,7 @@ func (mfa *CLIApp) checkManifestOperation(
log.Infof("manifest contains %d files, %s", chk.FileCount(),
humanize.IBytes(safeUint64(int64(chk.TotalBytes()))))
failures, err := runCheck(ctx, cmd, chk, showProgress)
failures, err := runCheck(ctx, chk, showProgress)
if err != nil {
return err
}
+25 -21
View File
@@ -18,7 +18,7 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
urfcli "github.com/urfave/cli/v3"
urfcli "github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
@@ -264,9 +264,8 @@ func commandsTakingVerbose() [][]string {
}
// TestVerboseCount asserts that one -v or --verbose gives verbose output and
// two -v, or -v and --verbose, give debug output (issue #125). urfave/cli
// before v2.25.5 counted a flag given by its alias twice, so one -v gave debug
// output.
// two -v give debug output (issue #125). urfave/cli before v2.25.5 counted a
// flag given by its alias twice, so one -v gave debug output.
func TestVerboseCount(t *testing.T) {
t.Parallel()
@@ -277,7 +276,6 @@ func TestVerboseCount(t *testing.T) {
{[]string{"-v"}, log.VerboseLevel},
{[]string{testFlagVerbose}, log.VerboseLevel},
{[]string{"-v", "-v"}, log.DebugLevel},
{[]string{"-v", testFlagVerbose}, log.DebugLevel},
}
for _, command := range commandsTakingVerbose() {
@@ -315,6 +313,28 @@ func TestCombinedShortVerboseRefused(t *testing.T) {
}
}
// TestShortAndLongVerboseRefused asserts that -v and --verbose given together
// are refused (issue #125): urfave/cli v2 refuses a flag given under two of its
// names, and one flag with an alias keeps help and parsing simple.
func TestShortAndLongVerboseRefused(t *testing.T) {
t.Parallel()
for _, command := range commandsTakingVerbose() {
args := slices.Concat(command, []string{"-v", testFlagVerbose})
t.Run(strings.Join(args, " "), func(t *testing.T) {
t.Parallel()
opts := testOpts(args, afero.NewMemMapFs())
exitCode, level := runCLIWithLevel(opts)
assert.Equal(t, 1, exitCode)
assert.Contains(t, testStderr(t, opts), "Cannot use two forms of the same flag")
assert.Equal(t, log.InfoLevel, level)
})
}
}
func TestHelpCommand(t *testing.T) {
t.Parallel()
@@ -1170,22 +1190,6 @@ func TestGenerateValidatesInputPaths(t *testing.T) {
})
}
// TestFlagAfterArgumentIsArgument asserts that flags are read only before a
// command's first argument: after it, -v is a path, not the verbose flag.
func TestFlagAfterArgumentIsArgument(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, fs.MkdirAll(testDir, 0o755))
writeTestFile(t, fs, testFile1, "content")
opts := testOpts([]string{testApp, cmdGenerate, testDir, "-v"}, fs)
exitCode := runCLI(opts)
assert.Equal(t, 1, exitCode)
assert.Contains(t, testStderr(t, opts), "path does not exist: -v")
}
func TestCheckDetectsManifestCorruption(t *testing.T) {
t.Parallel()
+31 -33
View File
@@ -4,6 +4,7 @@ package cli
import (
"bytes"
"context"
"flag"
"net/http"
"net/http/httptest"
"os"
@@ -14,7 +15,7 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
urfcli "github.com/urfave/cli/v3"
urfcli "github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/mfer"
)
@@ -194,17 +195,13 @@ func signedChecker(t *testing.T, manifest []byte) *mfer.Checker {
func TestPathDoesNotExistMessage(t *testing.T) {
t.Parallel()
set := flag.NewFlagSet("gen", flag.ContinueOnError)
require.NoError(t, set.Parse([]string{"nope"}))
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
cmd := &urfcli.Command{
Name: cmdGenerate,
Action: func(_ context.Context, c *urfcli.Command) error {
_, err := mfa.collectInputPaths(c.Args())
ctx := urfcli.NewContext(nil, set, nil)
return err
},
}
err := cmd.Run(context.Background(), []string{cmdGenerate, "nope"})
_, err := mfa.collectInputPaths(ctx.Args())
require.ErrorIs(t, err, errPathNotExist)
assert.EqualError(t, err, "path does not exist: nope")
}
@@ -217,22 +214,18 @@ func TestOutputFileExistsMessage(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/out.mf", []byte("old"), 0o644))
set := flag.NewFlagSet("gen", flag.ContinueOnError)
set.String("output", "", "")
set.Bool("force", false, "")
require.NoError(t, set.Parse([]string{"/d"}))
require.NoError(t, set.Set("output", "/out.mf"))
mfa := &CLIApp{Fs: fs}
cmd := &urfcli.Command{
Name: cmdGenerate,
Flags: []urfcli.Flag{
&urfcli.StringFlag{Name: "output"},
&urfcli.BoolFlag{Name: "force"},
},
Action: mfa.generateManifestOperation,
}
ctx := urfcli.NewContext(nil, set, nil)
// generateManifestOperation writes to the process-global logger during
// enumeration, so serialize with the other CLI runs.
err := runLocked(func() error {
return cmd.Run(context.Background(),
[]string{cmdGenerate, "--output", "/out.mf", "/d"})
})
err := runLocked(func() error { return mfa.generateManifestOperation(ctx) })
require.ErrorIs(t, err, errOutputExists)
assert.EqualError(t, err,
"output file /out.mf already exists (use --force to overwrite)")
@@ -257,7 +250,7 @@ func TestUnknownCommandMessage(t *testing.T) {
err := runLocked(func() error {
mfa.run([]string{testApp})
return mfa.app.Run(context.Background(), []string{testApp, "bogus"})
return mfa.app.Run([]string{testApp, "bogus"})
})
require.ErrorIs(t, err, errUnknownCommand)
assert.EqualError(t, err, `unknown command "bogus"`)
@@ -274,7 +267,7 @@ func TestManifestLoaderHTTPStatusMessage(t *testing.T) {
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
_, err := mfa.openManifestReader(context.Background(), server.URL+"/foo.mf")
_, err := mfa.openManifestReader(server.URL + "/foo.mf")
require.ErrorIs(t, err, errHTTPStatus)
assert.EqualError(t, err,
"failed to fetch "+server.URL+"/foo.mf: HTTP 404")
@@ -291,13 +284,17 @@ func TestFetchManifestHTTPStatusMessage(t *testing.T) {
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
cmd := mfa.fetchCommand()
cmd.Action = mfa.fetchManifestOperation
set := flag.NewFlagSet(cmdFetch, flag.ContinueOnError)
for _, f := range mfa.fetchCommand().Flags {
require.NoError(t, f.Apply(set))
}
require.NoError(t, set.Parse([]string{server.URL}))
ctx := urfcli.NewContext(nil, set, nil)
// fetchManifestOperation logs to the process-global logger.
err := runLocked(func() error {
return cmd.Run(context.Background(), []string{cmdFetch, server.URL})
})
err := runLocked(func() error { return mfa.fetchManifestOperation(ctx) })
require.ErrorIs(t, err, errHTTPStatus)
assert.EqualError(t, err, "failed to fetch manifest: HTTP 404")
}
@@ -323,13 +320,14 @@ func TestFetchFileHTTPStatusMessage(t *testing.T) {
func TestURLRequiredMessage(t *testing.T) {
t.Parallel()
set := flag.NewFlagSet("fetch", flag.ContinueOnError)
require.NoError(t, set.Parse([]string{}))
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
cmd := &urfcli.Command{Name: cmdFetch, Action: mfa.fetchManifestOperation}
ctx := urfcli.NewContext(nil, set, nil)
// fetchManifestOperation logs to the process-global logger.
err := runLocked(func() error {
return cmd.Run(context.Background(), []string{cmdFetch})
})
err := runLocked(func() error { return mfa.fetchManifestOperation(ctx) })
require.ErrorIs(t, err, errURLRequired)
assert.EqualError(t, err, "URL argument required")
}
+4 -8
View File
@@ -1,13 +1,12 @@
package cli
import (
"context"
"encoding/hex"
"encoding/json"
"fmt"
"time"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/mfer"
)
@@ -20,22 +19,19 @@ type ExportEntry struct {
Ctime *string `json:"ctime,omitempty"`
}
func (mfa *CLIApp) exportManifestOperation(
ctx context.Context, cmd *cli.Command,
) error {
pathOrURL, err := mfa.resolveManifestArg(cmd)
func (mfa *CLIApp) exportManifestOperation(ctx *cli.Context) error {
pathOrURL, err := mfa.resolveManifestArg(ctx)
if err != nil {
return fmt.Errorf("export: %w", err)
}
rc, err := mfa.openManifestReader(ctx, pathOrURL)
rc, err := mfa.openManifestReader(pathOrURL)
if err != nil {
return fmt.Errorf("export: %w", err)
}
defer func() { _ = rc.Close() }()
//nolint:contextcheck // mfer loads a manifest without a context
manifest, err := mfer.NewManifestFromReader(rc)
if err != nil {
return fmt.Errorf("export: failed to parse manifest: %w", err)
+30 -52
View File
@@ -20,7 +20,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/multiformats/go-multihash"
"github.com/spf13/afero"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
@@ -91,10 +91,10 @@ var (
// errHashMismatch indicates a downloaded file whose hash matches no
// manifest hash.
errHashMismatch = errors.New("hash mismatch")
// errNameClash indicates a manifest that lists a file where fetch
// writes another file.
errNameClash = errors.New(
"manifest lists a file where fetch writes another file")
// errManifestNameListed indicates a manifest that lists a file where
// fetch saves the manifest.
errManifestNameListed = errors.New(
"manifest lists a file where fetch saves the manifest")
)
// DownloadProgress reports the progress of a single file download.
@@ -328,21 +328,19 @@ func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
return verifyDownloadedHash(h.Sum(nil), entry) == nil
}
func (mfa *CLIApp) fetchManifestOperation(
ctx context.Context, cmd *cli.Command,
) error {
func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
log.Debug("fetchManifestOperation()")
if cmd.Args().Len() == 0 {
if ctx.Args().Len() == 0 {
return errURLRequired
}
timeout := cmd.Duration(flagTimeout)
timeout := ctx.Duration(flagTimeout)
if timeout <= 0 {
return errInvalidTimeout
}
manifestURL, err := resolveManifestURL(cmd.Args().Get(0))
manifestURL, err := resolveManifestURL(ctx.Args().Get(0))
if err != nil {
return fmt.Errorf("invalid URL: %w", err)
}
@@ -352,7 +350,7 @@ func (mfa *CLIApp) fetchManifestOperation(
firstDelay: firstRetryDelay,
}
manifestData, files, err := fetchManifest(ctx, cmd, client, manifestURL)
manifestData, files, err := fetchManifest(ctx, client, manifestURL)
if err != nil {
return err
}
@@ -363,7 +361,7 @@ func (mfa *CLIApp) fetchManifestOperation(
return err
}
dest := cmd.String(flagDest)
dest := ctx.String(flagDest)
err = os.MkdirAll(dest, dirPerms)
if err != nil {
@@ -381,7 +379,7 @@ func (mfa *CLIApp) fetchManifestOperation(
// Download each file
downloaded, downloadedBytes, dlErr := downloadManifestFiles(
ctx, client, baseURL, dest, files, progress)
ctx.Context, client, baseURL, dest, files, progress)
close(progress)
<-done
@@ -414,11 +412,11 @@ func (mfa *CLIApp) fetchManifestOperation(
// fetchManifest downloads the manifest at manifestURL and parses it,
// enforcing --require-signature if it is given and refusing a manifest
// that lists a file where fetch writes another. It returns the manifest as
// that lists a file where it will be saved. It returns the manifest as
// downloaded, to be saved once the files are in place, and the files it
// lists.
func fetchManifest(
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
ctx *cli.Context, client retryingClient, manifestURL string,
) ([]byte, []*mfer.MFFilePath, error) {
log.Infof("fetching manifest from %s", manifestURL)
@@ -427,7 +425,7 @@ func fetchManifest(
// manifest.
var manifestData []byte
err := client.get(ctx, manifestURL, func(resp *http.Response) error {
err := client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
var readErr error
manifestData, readErr = io.ReadAll(resp.Body)
@@ -439,13 +437,12 @@ func fetchManifest(
}
// Parse manifest
//nolint:contextcheck // mfer loads a manifest without a context
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
if err != nil {
return nil, nil, fmt.Errorf("failed to parse manifest: %w", err)
}
requiredSigner := cmd.String(flagRequireSignature)
requiredSigner := ctx.String(flagRequireSignature)
if requiredSigner != "" {
err = verifyFetchedSigner(ctx, manifestData, requiredSigner)
if err != nil {
@@ -455,7 +452,7 @@ func fetchManifest(
files := manifest.Files()
err = checkNoNameClash(files)
err = checkManifestNameUnlisted(files)
if err != nil {
return nil, nil, err
}
@@ -465,37 +462,19 @@ func fetchManifest(
return manifestData, files, nil
}
// checkNoNameClash returns an error if files lists a file, or a directory
// a file is in, under a name where fetch writes another file: the temp
// file it downloads a listed file to, or, at the top of the tree, the
// saved manifest or its temp file. fetch would remove or replace what is
// listed there, or fail partway, leaving a tree check rejects. Names are
// compared ignoring case, since on a case-insensitive filesystem INDEX.MF
// and index.mf are one file.
func checkNoNameClash(files []*mfer.MFFilePath) error {
sep := string(filepath.Separator)
// written maps each name fetch writes, other than the listed files
// themselves, in lower case, to the file it writes there.
written := map[string]string{
defaultManifestName: "the saved manifest",
tempPathFor(defaultManifestName): "the saved manifest's temp file",
}
// checkManifestNameUnlisted returns an error if files lists a file or
// directory at the top of the tree under the name fetch saves the
// manifest as, or under that name's temp file. Saving the manifest would
// replace or remove it, or fail once every file was downloaded, leaving a
// tree check rejects. Names are compared ignoring case, since on a
// case-insensitive filesystem INDEX.MF and index.mf are one file.
func checkManifestNameUnlisted(files []*mfer.MFFilePath) error {
for _, f := range files {
tmpPath := tempPathFor(filepath.Clean(f.GetPath()))
written[strings.ToLower(tmpPath)] = "the temp file for " + f.GetPath()
}
top, _, _ := strings.Cut(filepath.Clean(f.GetPath()), string(filepath.Separator))
for _, f := range files {
// Look up each directory on the file's path, then the file itself.
parts := strings.Split(strings.ToLower(filepath.Clean(f.GetPath())), sep)
for i := range parts {
what, ok := written[strings.Join(parts[:i+1], sep)]
if ok {
return fmt.Errorf("%w: %s (%s)", errNameClash, f.GetPath(), what)
}
if strings.EqualFold(top, defaultManifestName) ||
strings.EqualFold(top, tempPathFor(defaultManifestName)) {
return fmt.Errorf("%w: %s", errManifestNameListed, f.GetPath())
}
}
@@ -507,7 +486,7 @@ func checkNoNameClash(files []*mfer.MFFilePath) error {
// its manifest from a file, so the manifest is handed to it as a file in
// memory.
func verifyFetchedSigner(
ctx context.Context, manifestData []byte, requiredSigner string,
ctx *cli.Context, manifestData []byte, requiredSigner string,
) error {
memFs := afero.NewMemMapFs()
manifestPath := "/" + defaultManifestName
@@ -517,7 +496,6 @@ func verifyFetchedSigner(
return err
}
//nolint:contextcheck // mfer loads a manifest without a context
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
ManifestPath: manifestPath,
BasePath: "/",
@@ -527,7 +505,7 @@ func verifyFetchedSigner(
return fmt.Errorf("failed to load manifest: %w", err)
}
return verifyRequiredSigner(ctx, chk, requiredSigner)
return verifyRequiredSigner(ctx.Context, chk, requiredSigner)
}
// saveManifest writes the fetched manifest into dest under the default
+20 -79
View File
@@ -4,6 +4,7 @@ package cli
import (
"bytes"
"context"
"flag"
"fmt"
"io"
"maps"
@@ -22,6 +23,7 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
urfcli "github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/mfer"
)
@@ -1174,87 +1176,21 @@ func TestFetchRefusesListedManifestName(t *testing.T) {
t.Run(listed, func(t *testing.T) {
t.Parallel()
files := map[string][]byte{listed: []byte("listed")}
assertFetchRefused(t, builtManifest(t, files), files,
"manifest lists a file where fetch writes another file: "+listed)
})
}
}
// TestFetchRefusesListedTempName fetches manifests that list a.txt and
// .a.txt.tmp, the temp file fetch downloads a.txt to, at the top of the
// tree and in a directory. Downloading a.txt would remove .a.txt.tmp, so
// fetch must refuse the manifest before it creates the destination or
// requests any file. README with .README.tmp checks that temp names are
// compared ignoring case. A manifest that lists only one of the two is
// fetched in full.
func TestFetchRefusesListedTempName(t *testing.T) {
t.Parallel()
for _, dir := range []string{"", "sub/"} {
for file, tmp := range map[string]string{
dir + "a.txt": dir + ".a.txt.tmp",
dir + "README": dir + ".README.tmp",
} {
both := map[string][]byte{
file: []byte("a file"),
tmp: []byte("a file at its temp name"),
}
t.Run(file+" and "+tmp, func(t *testing.T) {
t.Parallel()
assertFetchRefused(t, builtManifest(t, both), both,
"manifest lists a file where fetch writes another file: "+
tmp+" (the temp file for "+file+")")
})
for listed, content := range both {
t.Run("only "+listed, func(t *testing.T) {
t.Parallel()
files := map[string][]byte{listed: content}
manifest := builtManifest(t, files)
server := httptest.NewServer(fetchTestHandler(manifest, files))
defer server.Close()
dest := t.TempDir()
opts := testOpts([]string{
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
}, afero.NewOsFs())
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
want := maps.Clone(files)
want[defaultManifestName] = manifest
assert.Equal(t, want, filesUnder(t, dest))
})
}
}
}
}
// builtManifest returns a manifest of files, built directly rather than
// scanned, since a scan lists no hidden files and never a path starting
// with "./".
func builtManifest(t *testing.T, files map[string][]byte) []byte {
t.Helper()
// Built directly rather than scanned, since a scan lists no
// hidden files and never a path starting with "./".
content := []byte("listed")
builder := mfer.NewBuilder()
for p, content := range files {
_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
_, err := builder.AddFile(mfer.RelFilePath(listed), mfer.FileSize(len(content)),
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
require.NoError(t, err)
}
var manifest bytes.Buffer
require.NoError(t, builder.Build(context.Background(), &manifest))
return manifest.Bytes()
assertFetchRefused(t, manifest.Bytes(), map[string][]byte{listed: content},
"manifest lists a file where fetch saves the manifest: "+listed)
})
}
}
// assertFetchRefused serves manifest, a manifest of files, and fetches it
@@ -1320,13 +1256,18 @@ func TestFetchTimeoutFlag(t *testing.T) {
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
cmd := mfa.fetchCommand()
cmd.Action = mfa.fetchManifestOperation
set := flag.NewFlagSet(cmdFetch, flag.ContinueOnError)
for _, f := range mfa.fetchCommand().Flags {
require.NoError(t, f.Apply(set))
}
require.NoError(t, set.Parse([]string{"--" + flagTimeout, "100ms", server.URL}))
cliCtx := urfcli.NewContext(nil, set, nil)
cliCtx.Context = ctx
// fetchManifestOperation logs to the process-global logger.
err := runLocked(func() error {
return cmd.Run(ctx, []string{cmdFetch, "--" + flagTimeout, "100ms", server.URL})
})
err := runLocked(func() error { return mfa.fetchManifestOperation(cliCtx) })
require.Error(t, err)
}
+19 -22
View File
@@ -14,7 +14,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/multiformats/go-multihash"
"github.com/spf13/afero"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
@@ -202,12 +202,12 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
// resolveFreshenManifestPath determines the manifest path from the CLI
// arguments, searching directories for a manifest where needed.
func (mfa *CLIApp) resolveFreshenManifestPath(cmd *cli.Command) (string, error) {
if cmd.Args().Len() == 0 {
func (mfa *CLIApp) resolveFreshenManifestPath(ctx *cli.Context) (string, error) {
if ctx.Args().Len() == 0 {
return findManifest(mfa.Fs, ".")
}
arg := cmd.Args().Get(0)
arg := ctx.Args().Get(0)
info, statErr := mfa.Fs.Stat(arg)
if statErr == nil && info.IsDir() {
@@ -338,14 +338,14 @@ func writeFreshenedManifest(
// newFreshenBuilder constructs the manifest builder configured from CLI
// flags.
func newFreshenBuilder(cmd *cli.Command) *mfer.Builder {
func newFreshenBuilder(ctx *cli.Context) *mfer.Builder {
builder := mfer.NewBuilder()
if cmd.Bool("include-timestamps") {
if ctx.Bool("include-timestamps") {
builder.SetIncludeTimestamps(true)
}
// Set up signing options if sign-key is provided
if signKey := cmd.String("sign-key"); signKey != "" {
if signKey := ctx.String("sign-key"); signKey != "" {
builder.SetSigningOptions(&mfer.SigningOptions{
KeyID: mfer.GPGKeyID(signKey),
})
@@ -358,13 +358,13 @@ func newFreshenBuilder(cmd *cli.Command) *mfer.Builder {
// freshenScan runs the scan phase against the loaded manifest entries
// and returns the populated scanner and the count of removed files.
func (mfa *CLIApp) freshenScan(
cmd *cli.Command, manifestPath, absBase string,
ctx *cli.Context, manifestPath, absBase string,
existingByPath map[string]*mfer.MFFilePath,
) (*freshenScanner, int64, error) {
log.Infof("scanning filesystem...")
startScan := time.Now()
showProgress := cmd.Bool("progress")
showProgress := ctx.Bool("progress")
// Leave out the manifest and a temp file left by an interrupted run,
// as gen does. A path that cannot be stat'd, normally because no file
@@ -382,8 +382,8 @@ func (mfa *CLIApp) freshenScan(
fs: mfa.Fs,
absBase: absBase,
excluded: excluded,
includeDotfiles: cmd.Bool("include-dotfiles"),
followSymlinks: cmd.Bool("follow-symlinks"),
includeDotfiles: ctx.Bool("include-dotfiles"),
followSymlinks: ctx.Bool("follow-symlinks"),
showProgress: showProgress,
existingByPath: existingByPath,
}
@@ -433,7 +433,7 @@ func hashTotals(entries []*freshenEntry) (int64, int64) {
// runFreshenHash processes every entry through the hasher, aborting if
// the context is canceled.
func runFreshenHash(
ctx context.Context, hasher *freshenHasher, entries []*freshenEntry,
ctx *cli.Context, hasher *freshenHasher, entries []*freshenEntry,
) error {
for _, e := range entries {
select {
@@ -479,21 +479,18 @@ func (mfa *CLIApp) loadExistingEntries(
return existingByPath, nil
}
func (mfa *CLIApp) freshenManifestOperation(
ctx context.Context, cmd *cli.Command,
) error {
func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
log.Debug("freshenManifestOperation()")
basePath := cmd.String("base")
showProgress := cmd.Bool("progress")
basePath := ctx.String("base")
showProgress := ctx.Bool("progress")
// Find manifest file
manifestPath, err := mfa.resolveFreshenManifestPath(cmd)
manifestPath, err := mfa.resolveFreshenManifestPath(ctx)
if err != nil {
return fmt.Errorf("freshen: %w", err)
}
//nolint:contextcheck // mfer loads a manifest without a context
existingByPath, err := mfa.loadExistingEntries(manifestPath)
if err != nil {
return err
@@ -505,7 +502,7 @@ func (mfa *CLIApp) freshenManifestOperation(
}
// Phase 1: Scan filesystem
scanner, removed, err := mfa.freshenScan(cmd, manifestPath, absBase,
scanner, removed, err := mfa.freshenScan(ctx, manifestPath, absBase,
existingByPath)
if err != nil {
return err
@@ -527,7 +524,7 @@ func (mfa *CLIApp) freshenManifestOperation(
totalHashBytes: totalHashBytes,
filesToHash: filesToHash,
startHash: time.Now(),
builder: newFreshenBuilder(cmd),
builder: newFreshenBuilder(ctx),
}
err = runFreshenHash(ctx, hasher, scanner.entries)
@@ -551,7 +548,7 @@ func (mfa *CLIApp) freshenManifestOperation(
}
// Write updated manifest atomically (write to temp, then rename)
err = writeFreshenedManifest(ctx, mfa.Fs, hasher.builder, manifestPath)
err = writeFreshenedManifest(ctx.Context, mfa.Fs, hasher.builder, manifestPath)
if err != nil {
return err
}
+18 -21
View File
@@ -1,7 +1,6 @@
package cli
import (
"context"
"errors"
"fmt"
"os"
@@ -13,7 +12,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/spf13/afero"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
@@ -89,12 +88,12 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
}
// buildScannerOptions constructs scanner options from the CLI flags.
func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions {
output := cmd.String("output")
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
output := ctx.String("output")
opts := &mfer.ScannerOptions{
IncludeDotfiles: cmd.Bool("include-dotfiles"),
FollowSymLinks: cmd.Bool("follow-symlinks"),
IncludeTimestamps: cmd.Bool("include-timestamps"),
IncludeDotfiles: ctx.Bool("include-dotfiles"),
FollowSymLinks: ctx.Bool("follow-symlinks"),
IncludeTimestamps: ctx.Bool("include-timestamps"),
Fs: mfa.Fs,
// Neither a manifest being replaced nor a temp file left by an
// interrupted run belongs in the new manifest.
@@ -102,14 +101,14 @@ func (mfa *CLIApp) buildScannerOptions(cmd *cli.Command) *mfer.ScannerOptions {
}
// Set seed for deterministic UUID if provided
if seed := cmd.String("seed"); seed != "" {
if seed := ctx.String("seed"); seed != "" {
opts.Seed = seed
log.Infof("using deterministic seed for manifest UUID")
}
// Set up signing options if sign-key is provided
if signKey := cmd.String("sign-key"); signKey != "" {
if signKey := ctx.String("sign-key"); signKey != "" {
opts.SigningOptions = &mfer.SigningOptions{
KeyID: mfer.GPGKeyID(signKey),
}
@@ -174,14 +173,14 @@ func (mfa *CLIApp) cleanupOnSignal(outFile afero.File, tmpPath string) chan os.S
// runEnumeratePhase enumerates all input paths with optional progress
// reporting and logs the totals.
func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error {
func (mfa *CLIApp) runEnumeratePhase(ctx *cli.Context, s *mfer.Scanner) error {
// Set up enumeration progress reporting
var (
enumProgress chan mfer.EnumerateStatus
enumWg sync.WaitGroup
)
if cmd.Bool("progress") {
if ctx.Bool("progress") {
enumProgress = make(chan mfer.EnumerateStatus, 1)
enumWg.Add(1)
@@ -189,7 +188,7 @@ func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error {
go reportEnumProgress(enumProgress, &enumWg)
}
err := mfa.enumerateInputs(s, cmd.Args(), enumProgress)
err := mfa.enumerateInputs(s, ctx.Args(), enumProgress)
if err != nil {
return err
}
@@ -202,24 +201,22 @@ func (mfa *CLIApp) runEnumeratePhase(cmd *cli.Command, s *mfer.Scanner) error {
return nil
}
func (mfa *CLIApp) generateManifestOperation(
ctx context.Context, cmd *cli.Command,
) error {
func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
log.Debug("generateManifestOperation()")
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(cmd))
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(ctx))
// Phase 1: Enumeration - collect paths and stat files
err := mfa.runEnumeratePhase(cmd, s)
err := mfa.runEnumeratePhase(ctx, s)
if err != nil {
return err
}
showProgress := cmd.Bool("progress")
showProgress := ctx.Bool("progress")
// Check if output file exists
outputPath := cmd.String("output")
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !cmd.Bool("force") {
outputPath := ctx.String("output")
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !ctx.Bool("force") {
return fmt.Errorf("output file %s %w", outputPath, errOutputExists)
}
@@ -262,7 +259,7 @@ func (mfa *CLIApp) generateManifestOperation(
go reportScanProgress(scanProgress, &scanWg)
}
err = s.ToManifest(ctx, outFile, scanProgress)
err = s.ToManifest(ctx.Context, outFile, scanProgress)
scanWg.Wait()
+6 -8
View File
@@ -1,35 +1,33 @@
package cli
import (
"context"
"fmt"
"time"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
func (mfa *CLIApp) listManifestOperation(ctx context.Context, cmd *cli.Command) error {
func (mfa *CLIApp) listManifestOperation(ctx *cli.Context) error {
// Default to ErrorLevel for clean output
log.SetLevel(log.ErrorLevel)
longFormat := cmd.Bool("long")
print0 := cmd.Bool("print0")
longFormat := ctx.Bool("long")
print0 := ctx.Bool("print0")
pathOrURL, err := mfa.resolveManifestArg(cmd)
pathOrURL, err := mfa.resolveManifestArg(ctx)
if err != nil {
return fmt.Errorf("list: %w", err)
}
rc, err := mfa.openManifestReader(ctx, pathOrURL)
rc, err := mfa.openManifestReader(pathOrURL)
if err != nil {
return fmt.Errorf("list: %w", err)
}
defer func() { _ = rc.Close() }()
//nolint:contextcheck // mfer loads a manifest without a context
manifest, err := mfer.NewManifestFromReader(rc)
if err != nil {
return fmt.Errorf("list: failed to parse manifest: %w", err)
+8 -8
View File
@@ -9,7 +9,7 @@ import (
"strings"
"time"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
)
// manifestFetchTimeout bounds HTTP requests made to fetch a manifest.
@@ -30,13 +30,13 @@ func isHTTPURL(s string) bool {
// openManifestReader opens a manifest from a path or URL and returns a ReadCloser.
// The caller must close the returned reader.
func (mfa *CLIApp) openManifestReader(
ctx context.Context, pathOrURL string,
) (io.ReadCloser, error) {
func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
if isHTTPURL(pathOrURL) {
client := &http.Client{Timeout: manifestFetchTimeout}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pathOrURL, nil)
req, err := http.NewRequestWithContext(
context.Background(), http.MethodGet, pathOrURL, nil,
)
if err != nil {
return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
}
@@ -67,9 +67,9 @@ func (mfa *CLIApp) openManifestReader(
// resolveManifestArg resolves the manifest path from CLI arguments.
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
// If no argument is given, the current directory is searched.
func (mfa *CLIApp) resolveManifestArg(cmd *cli.Command) (string, error) {
if cmd.Args().Len() > 0 {
arg := cmd.Args().Get(0)
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
if ctx.Args().Len() > 0 {
arg := ctx.Args().Get(0)
if isHTTPURL(arg) {
return arg, nil
}
+47 -66
View File
@@ -1,7 +1,6 @@
package cli
import (
"context"
"errors"
"fmt"
"io"
@@ -9,7 +8,7 @@ import (
"time"
"github.com/spf13/afero"
"github.com/urfave/cli/v3"
"github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
)
@@ -55,7 +54,7 @@ type CLIApp struct {
gitrev string
startupTime time.Time
exitCode int
app *cli.Command
app *cli.App
Stdin io.Reader // Standard input stream
Stdout io.Writer // Standard output stream for normal output
@@ -106,15 +105,15 @@ func (mfa *CLIApp) printBanner() {
// urfave/cli reads a flag from the nearest command that defines it, so each
// command in the lineage is asked. The highest -v count wins rather than the
// sum, because a subcommand without its own copies reads the root's.
func (mfa *CLIApp) setVerbosity(cmd *cli.Command) {
func (mfa *CLIApp) setVerbosity(c *cli.Context) {
_, present := os.LookupEnv("MFER_DEBUG")
verbosity := 0
quiet := false
for _, c := range cmd.Lineage() {
verbosity = max(verbosity, c.Count("verbose"))
quiet = quiet || c.Bool("quiet")
for _, ctx := range c.Lineage() {
verbosity = max(verbosity, ctx.Count("verbose"))
quiet = quiet || ctx.Bool("quiet")
}
switch {
@@ -128,34 +127,23 @@ func (mfa *CLIApp) setVerbosity(cmd *cli.Command) {
}
// commonFlags returns the -v and -q flags taken by the root and by the
// generate, check, freshen and fetch subcommands. They are local, so the
// root's copies are not inherited by the subcommands that do not take them.
// generate, check, freshen and fetch subcommands.
func commonFlags() []cli.Flag {
return []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
Aliases: []string{"v"},
Usage: "Increase verbosity (-v for verbose, -v -v for debug)",
Local: true,
Count: new(int),
},
&cli.BoolFlag{
Name: "quiet",
Aliases: []string{"q"},
Usage: "Suppress output except errors",
Local: true,
},
}
}
// stopOnFirstArg returns the StopOnNthArg setting every command uses: flags
// are read only before the command's first argument, and everything after it
// is an argument, as with urfave/cli v2. So `mfer gen d -v` names a path "-v".
func stopOnFirstArg() *int {
n := 1
return &n
}
// requireSignatureFlag returns the --require-signature flag taken by the
// check and fetch subcommands.
func requireSignatureFlag() *cli.StringFlag {
@@ -163,7 +151,7 @@ func requireSignatureFlag() *cli.StringFlag {
Name: flagRequireSignature,
Aliases: []string{"S"},
Usage: "Require manifest to be signed by the specified GPG key ID",
Sources: cli.EnvVars("MFER_REQUIRE_SIGNATURE"),
EnvVars: []string{"MFER_REQUIRE_SIGNATURE"},
}
}
@@ -173,12 +161,11 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
Aliases: []string{"gen"},
Usage: "Generate manifest file",
ArgsUsage: "[path ...]",
StopOnNthArg: stopOnFirstArg(),
Action: func(ctx context.Context, cmd *cli.Command) error {
mfa.setVerbosity(cmd)
Action: func(c *cli.Context) error {
mfa.setVerbosity(c)
mfa.printBanner()
return mfa.generateManifestOperation(ctx, cmd)
return mfa.generateManifestOperation(c)
},
Flags: append(commonFlags(),
&cli.BoolFlag{
@@ -212,12 +199,12 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
Name: "sign-key",
Aliases: []string{"s"},
Usage: "GPG key ID to sign the manifest with",
Sources: cli.EnvVars("MFER_SIGN_KEY"),
EnvVars: []string{"MFER_SIGN_KEY"},
},
&cli.StringFlag{
Name: "seed",
Usage: "Seed value for deterministic manifest UUID",
Sources: cli.EnvVars("MFER_SEED"),
EnvVars: []string{"MFER_SEED"},
},
&cli.BoolFlag{
Name: "include-timestamps",
@@ -233,12 +220,11 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
Name: cmdCheck,
Usage: "Validate files using manifest file",
ArgsUsage: manifestArgsUsage,
StopOnNthArg: stopOnFirstArg(),
Action: func(ctx context.Context, cmd *cli.Command) error {
mfa.setVerbosity(cmd)
Action: func(c *cli.Context) error {
mfa.setVerbosity(c)
mfa.printBanner()
return mfa.checkManifestOperation(ctx, cmd)
return mfa.checkManifestOperation(c)
},
Flags: append(commonFlags(),
&cli.StringFlag{
@@ -266,12 +252,11 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
Name: cmdFreshen,
Usage: "Update manifest with changed, new, and removed files",
ArgsUsage: manifestArgsUsage,
StopOnNthArg: stopOnFirstArg(),
Action: func(ctx context.Context, cmd *cli.Command) error {
mfa.setVerbosity(cmd)
Action: func(c *cli.Context) error {
mfa.setVerbosity(c)
mfa.printBanner()
return mfa.freshenManifestOperation(ctx, cmd)
return mfa.freshenManifestOperation(c)
},
Flags: append(commonFlags(),
&cli.StringFlag{
@@ -300,7 +285,7 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
Name: "sign-key",
Aliases: []string{"s"},
Usage: "GPG key ID to sign the manifest with",
Sources: cli.EnvVars("MFER_SIGN_KEY"),
EnvVars: []string{"MFER_SIGN_KEY"},
},
&cli.BoolFlag{
Name: "include-timestamps",
@@ -316,11 +301,10 @@ func (mfa *CLIApp) exportCommand() *cli.Command {
Name: cmdExport,
Usage: "Export manifest contents as JSON",
ArgsUsage: "[manifest file or URL]",
StopOnNthArg: stopOnFirstArg(),
Action: func(ctx context.Context, cmd *cli.Command) error {
mfa.setVerbosity(cmd)
Action: func(c *cli.Context) error {
mfa.setVerbosity(c)
return mfa.exportManifestOperation(ctx, cmd)
return mfa.exportManifestOperation(c)
},
}
}
@@ -329,8 +313,7 @@ func (mfa *CLIApp) versionCommand() *cli.Command {
return &cli.Command{
Name: cmdVersion,
Usage: "Show version",
StopOnNthArg: stopOnFirstArg(),
Action: func(context.Context, *cli.Command) error {
Action: func(_ *cli.Context) error {
mfa.printVersion()
return nil
@@ -344,8 +327,9 @@ func (mfa *CLIApp) listCommand() *cli.Command {
Aliases: []string{"ls"},
Usage: "List files in manifest",
ArgsUsage: manifestArgsUsage,
StopOnNthArg: stopOnFirstArg(),
Action: mfa.listManifestOperation,
Action: func(c *cli.Context) error {
return mfa.listManifestOperation(c)
},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "long",
@@ -365,12 +349,11 @@ func (mfa *CLIApp) fetchCommand() *cli.Command {
Name: cmdFetch,
Usage: "fetch manifest and referenced files",
ArgsUsage: "URL",
StopOnNthArg: stopOnFirstArg(),
Action: func(ctx context.Context, cmd *cli.Command) error {
mfa.setVerbosity(cmd)
Action: func(c *cli.Context) error {
mfa.setVerbosity(c)
mfa.printBanner()
return mfa.fetchManifestOperation(ctx, cmd)
return mfa.fetchManifestOperation(c)
},
Flags: append(commonFlags(),
&cli.DurationFlag{
@@ -402,40 +385,38 @@ func (mfa *CLIApp) run(args []string) {
log.SetOutput(mfa.Stdout, mfa.Stderr)
log.Init()
// -v means verbose, not version, at the root as on the generate, check,
// freshen and fetch subcommands: verbose is the more common meaning of -v
// in tools that offer both. urfave/cli's built-in version flag claims -v
// by default, so the version flag takes the capital -V instead.
// -v means verbose, not version. urfave/cli's built-in version flag
// claims -v by default, which made "mfer -v --version" fail to parse and
// gave -v a different meaning at the root than on the generate, check,
// freshen and fetch subcommands, where it means verbose. Verbose is the
// more common meaning of -v in tools that offer both, so -v means verbose
// at the root too and the version flag takes the capital -V.
// VersionFlag and VersionPrinter are urfave/cli package globals; run() is
// serialized in tests, so assigning them here is safe.
cli.VersionFlag = &cli.BoolFlag{
Name: cmdVersion,
Aliases: []string{"V"},
Usage: "print the version",
Local: true,
}
cli.VersionPrinter = func(*cli.Command) { mfa.printVersion() }
cli.VersionPrinter = func(_ *cli.Context) { mfa.printVersion() }
mfa.app = &cli.Command{
mfa.app = &cli.App{
Name: mfa.appname,
Usage: "Manifest generator",
Version: mfa.VersionString(),
EnableShellCompletion: true,
EnableBashCompletion: true,
Writer: mfa.Stdout,
// v3 writes its "Incorrect Usage" line to ErrWriter; v2 wrote it to
// stdout, before the help, so it stays on stdout.
ErrWriter: mfa.Stdout,
ErrWriter: mfa.Stderr,
Flags: commonFlags(),
StopOnNthArg: stopOnFirstArg(),
Action: func(_ context.Context, cmd *cli.Command) error {
if cmd.Args().Len() > 0 {
return fmt.Errorf("%w %q", errUnknownCommand, cmd.Args().First())
Action: func(c *cli.Context) error {
if c.Args().Len() > 0 {
return fmt.Errorf("%w %q", errUnknownCommand, c.Args().First())
}
mfa.setVerbosity(cmd)
mfa.setVerbosity(c)
mfa.printBanner()
return cli.ShowRootCommandHelp(cmd)
return cli.ShowAppHelp(c)
},
Commands: []*cli.Command{
mfa.generateCommand(),
@@ -450,7 +431,7 @@ func (mfa *CLIApp) run(args []string) {
mfa.app.HideVersion = false
err := mfa.app.Run(context.Background(), args)
err := mfa.app.Run(args)
if err != nil {
mfa.exitCode = 1
+3
View File
@@ -0,0 +1,3 @@
package mfer
//go:generate protoc ./mf.proto --go_out=paths=source_relative:.
+1
View File
@@ -1,5 +1,6 @@
{
"name": "mfer",
"version": "0.1.0",
"private": true,
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
"license": "WTFPL",
+5 -59
View File
@@ -13,15 +13,11 @@ set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
# The node version is in .nvmrc, where script/prettier reads it too.
NODE_VERSION="$(cat "$ROOT/.nvmrc")"
NODE_VERSION="22.17.0"
NVM_VERSION="0.40.3"
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
YARN_VERSION="1.22.22"
# protoc v33.4, 2026-10-04, for script/generate. The sha256 of each
# platform's release archive is in ensure_protoc.
PROTOC_VERSION="33.4"
PKGMGR=""
SUDO=""
@@ -78,11 +74,9 @@ verify_sha256() {
fi
}
# nvm is a bash script; run a command in a bash with nvm loaded.
# --no-use: otherwise loading nvm here switches to the version .nvmrc
# names, and fails silently while that version is not installed yet.
# nvm is a bash script; run a command in a bash with nvm loaded
nvm_sh() {
bash -c ". \"\$HOME/.nvm/nvm.sh\" --no-use && $*"
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
}
ensure_nvm() {
@@ -128,48 +122,6 @@ install_js_deps() {
fi
}
# Unpack protoc's release archive for this platform into bin/protoc, after
# checking the archive's sha256, unless bin/protoc already holds the pinned
# version.
ensure_protoc() {
dir="$ROOT/bin/protoc"
if [ "$("$dir/bin/protoc" --version 2>/dev/null)" = \
"libprotoc $PROTOC_VERSION" ]; then
return 0
fi
case "$(uname -s) $(uname -m)" in
"Linux x86_64")
platform="linux-x86_64"
sha256="c0040ea9aef08fdeb2c74ca609b18d5fdbfc44ea0042fcfbfb38860d35f7dd66"
;;
"Linux aarch64" | "Linux arm64")
platform="linux-aarch_64"
sha256="15aa988f4a6090636525ec236a8e4b3aab41eef402751bd5bb2df6afd9b7b5a5"
;;
"Darwin x86_64")
platform="osx-x86_64"
sha256="a49bec10d039e902d3b43e49938c42526f90011467609864fa6386ac4014da58"
;;
"Darwin arm64")
platform="osx-aarch_64"
sha256="726297dcfed58592fd35620a5a6246ae020c39e88f3fd4cb1827df7bcf3dfcf1"
;;
*)
echo "bootstrap: no protoc archive pinned for $(uname -s) $(uname -m)" >&2
exit 1
;;
esac
if missing curl; then pkg_install curl curl curl curl; fi
if missing unzip; then pkg_install unzip unzip unzip unzip; fi
tmp="$(mktemp -d)"
curl -fsSL -o "$tmp/protoc.zip" \
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip"
verify_sha256 "$tmp/protoc.zip" "$sha256"
rm -rf "$dir"
unzip -q "$tmp/protoc.zip" -d "$dir"
rm -rf "$tmp"
}
main() {
cd "$ROOT"
@@ -180,10 +132,8 @@ main() {
# ---- JS / docs repos ----
# This is a Go repo, but node and yarn are required anyway: prettier
# formats the Markdown and JSON, and script/fmt-check verifies it.
# The version is pinned by package.json/yarn.lock: yarn checks every
# package it fetches against its yarn.lock integrity hash, and
# --frozen-lockfile fails instead of rewriting a yarn.lock that no
# longer matches package.json.
# The version is pinned by package.json/yarn.lock, whose integrity
# hashes --frozen-lockfile enforces.
ensure_node
ensure_yarn
install_js_deps
@@ -192,10 +142,6 @@ main() {
if missing go; then pkg_install go golang go go; fi
# No golangci-lint: script/lint runs it in Docker only.
go mod download
# gofumpt and protoc-gen-go: bin/tools/go.mod pins them, and
# script/gofumpt and script/generate build them from there.
(cd "$ROOT/bin/tools" && go mod download)
ensure_protoc
# ---- Python repos ----
# if missing python3; then pkg_install python3 python3 python3 python3; fi
-18
View File
@@ -1,18 +0,0 @@
#!/bin/sh
# script/build: build the mfer binary into bin/mfer, stamped with the
# revision `mfer version` prints, derived as script/docker derives it.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
# Own line: a failing command substitution inside an argument does
# not trip `set -e`, so the inline form degrades silently to an
# empty constant.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
go build -ldflags "-X main.Gitrev=$version" -o bin/mfer ./cmd/mfer
}
main "$@"
+18 -20
View File
@@ -4,17 +4,26 @@
# regenerates mf.pb.go: it is committed, so building and checking need no
# protoc. A test fails while mf.proto no longer matches the recorded hash.
#
# Runs the protoc that script/bootstrap unpacks into bin/protoc, and the
# protoc-gen-go that bin/tools/go.mod pins. Another version of either
# writes a different mf.pb.go.
# Needs exactly the protoc and protoc-gen-go versions named in the header of
# the committed mf.pb.go (README.md says how to install them). Another
# version writes a different mf.pb.go, so the script refuses to run.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# The protoc version script/bootstrap installs. protoc 33.4 names itself
# v6.33.4 in the mf.pb.go header.
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
PROTOC_VERSION="33.4"
PROTOC="$ROOT/bin/protoc/bin/protoc"
PROTOC_GEN_GO_VERSION="v1.36.11"
# require_version <command> <its exact --version output>
require_version() {
actual="$("$1" --version 2>/dev/null || true)"
if [ "$actual" != "$2" ]; then
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
echo " README.md says how to install it." >&2
exit 1
fi
}
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
# where there is no sha256sum.
@@ -30,24 +39,13 @@ sha256() {
}
main() {
# A bin/protoc left from before the pin moved fails here, until
# script/bootstrap replaces it.
actual="$("$PROTOC" --version 2>/dev/null || true)"
if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then
echo "generate: needs protoc $PROTOC_VERSION in bin/protoc," \
"found: ${actual:-none}; run script/bootstrap" >&2
exit 1
fi
# `go tool -n` builds protoc-gen-go from bin/tools and prints where the
# binary is, without running it.
plugin="$(cd "$ROOT/bin/tools" && go tool -n protoc-gen-go)"
cd "$ROOT/mfer"
require_version protoc "libprotoc $PROTOC_VERSION"
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
# Hashed before regenerating, so a missing hash tool stops the script
# before it changes anything. Regenerating leaves mf.proto as it is.
proto_hash="$(sha256 mf.proto)"
"$PROTOC" --plugin=protoc-gen-go="$plugin" \
--go_out=paths=source_relative:. ./mf.proto
go generate .
echo "$proto_hash" >mf.proto.sha256
}
+11 -11
View File
@@ -4,16 +4,16 @@
# Takes exactly one mode argument, --write or --check, and runs the same
# gofumpt version over the same files in both modes. script/fmt and
# script/fmt-check both go through here, and so does the Docker lint
# stage, so what gets formatted and what gets verified cannot drift
# apart.
# stage (make fmt-check-go), so what gets formatted and what gets
# verified cannot drift apart.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# The gofumpt version is the one bin/tools/go.mod pins. `go tool` run in
# bin/tools builds it from source checked against the hashes in
# bin/tools/go.sum, so neither a developer machine nor the lint image needs
# it installed.
# gofumpt v0.12.0, 2026-10-04. `go run` fetches and builds exactly this
# version, so neither a developer machine nor the lint image needs
# gofumpt installed.
GOFUMPT="mvdan.cc/gofumpt@v0.12.0"
usage() {
echo "usage: script/gofumpt --write|--check" >&2
@@ -22,15 +22,15 @@ usage() {
main() {
[ "$#" -eq 1 ] || usage
cd "$ROOT/bin/tools"
# Every Go file in the repo, from $ROOT down. gofumpt holds generated
# files, such as mfer/mf.pb.go, to gofmt's rules only.
cd "$ROOT"
# Every Go file in the repo. gofumpt holds generated files, such as
# mfer/mf.pb.go, to gofmt's rules only.
case "$1" in
--write) go tool gofumpt -l -w "$ROOT" ;;
--write) go run "$GOFUMPT" -l -w . ;;
--check)
# Own line: a failing command inside `[ -n "$(...)" ]` does
# not trip `set -e`, so a gofumpt that never ran would pass.
unformatted="$(go tool gofumpt -l "$ROOT")"
unformatted="$(go run "$GOFUMPT" -l .)"
if [ -n "$unformatted" ]; then
echo "gofumpt: files need formatting (run make fmt):" >&2
echo "$unformatted" >&2
+24 -24
View File
@@ -18,9 +18,24 @@ usage() {
exit 2
}
# Only the prettier yarn installed from yarn.lock, never one on PATH: a
# different version formats differently.
PRETTIER="$ROOT/node_modules/.bin/prettier"
# Prefer the version pinned by package.json/yarn.lock so that CI and
# developer machines format identically. Fall back to a prettier on PATH,
# but say so, because a different version formats differently.
find_prettier() {
if [ -x "$ROOT/node_modules/.bin/prettier" ]; then
printf '%s\n' "$ROOT/node_modules/.bin/prettier"
return 0
fi
if command -v prettier >/dev/null 2>&1; then
echo "prettier: node_modules/.bin/prettier is absent; using the" \
"prettier on PATH, which may be a different version than the" \
"one pinned in package.json. Run script/bootstrap to install" \
"the pinned version." >&2
command -v prettier
return 0
fi
return 1
}
main() {
[ "$#" -eq 1 ] || usage
@@ -31,26 +46,10 @@ main() {
cd "$ROOT"
# Where there is no node on PATH, script/bootstrap installs the version
# .nvmrc names through nvm, which keeps it in this directory.
if ! command -v node >/dev/null 2>&1; then
PATH="$HOME/.nvm/versions/node/v$(cat .nvmrc)/bin:$PATH"
fi
if ! command -v node >/dev/null 2>&1; then
echo "prettier: node is missing; run script/bootstrap" >&2
exit 1
fi
# node_modules keeps the old prettier after package.json moves to a new
# one, until script/bootstrap runs again, so compare the two.
if ! installed="$("$PRETTIER" --version 2>/dev/null)"; then
echo "prettier: not installed; run script/bootstrap" >&2
exit 1
fi
pinned="$(node -p 'require("./package.json").devDependencies.prettier')"
if [ "$installed" != "$pinned" ]; then
echo "prettier: package.json pins $pinned but $installed is" \
"installed; run script/bootstrap" >&2
if ! prettier_bin="$(find_prettier)"; then
echo "prettier: not found." >&2
echo " Install it with: script/bootstrap" >&2
echo " (installs the version pinned in package.json/yarn.lock)" >&2
exit 1
fi
@@ -63,7 +62,8 @@ main() {
# patterns always match at least one tracked file (README.md,
# package.json), so an empty match means the glob broke, and prettier
# erroring out is exactly what we want rather than a vacuous pass.
"$PRETTIER" "$mode" "**/*.md" "**/*.json"
"$prettier_bin" "$mode" "**/*.md"
"$prettier_bin" "$mode" "**/*.json"
}
main "$@"