Compare commits
6
Commits
5cee2b28d0
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab72692439 | ||
|
|
9bb0ab3a03 | ||
|
|
d3394bd2a2 | ||
|
|
a3e37d1ab8 | ||
|
|
acff23d92b | ||
|
|
8fe0244291 |
+5
-3
@@ -59,6 +59,8 @@
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# This repo's own host-built archives (Makefile).
|
||||
*.tmp
|
||||
*.dockerimage
|
||||
# This repo's own host-built binary (make build).
|
||||
/bin/mfer
|
||||
|
||||
# The protoc script/bootstrap unpacks for script/generate.
|
||||
/bin/protoc
|
||||
|
||||
+1
-5
@@ -1,11 +1,7 @@
|
||||
/bin/mfer
|
||||
/bin/protoc/
|
||||
/tmp
|
||||
/node_modules/
|
||||
*.tmp
|
||||
*.dockerimage
|
||||
/vendor
|
||||
vendor.tzst
|
||||
modcache.tzst
|
||||
|
||||
# Generated manifest files
|
||||
/index.mf
|
||||
|
||||
+5
-4
@@ -9,15 +9,16 @@ RUN go mod download
|
||||
COPY . .
|
||||
|
||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||
# markdown half runs in the mdfmt stage below.
|
||||
RUN make fmt-check-go
|
||||
# markdown half runs in the mdfmt stage below. The image has no gofumpt
|
||||
# either; script/gofumpt builds the version bin/tools/go.mod pins.
|
||||
RUN script/gofumpt --check
|
||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
||||
# there is no docker inside this build.
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||
# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins.
|
||||
# 22.17.0 and yarn 1.22.22, the versions .nvmrc and script/bootstrap pin.
|
||||
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
|
||||
|
||||
WORKDIR /src
|
||||
@@ -30,7 +31,7 @@ COPY . .
|
||||
RUN script/prettier --check
|
||||
|
||||
# Build stage — tests and compilation
|
||||
# golang:1.23 (2026-03-14)
|
||||
# golang:1.23.12, 2026-03-14
|
||||
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
||||
|
||||
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
|
||||
|
||||
@@ -1,20 +1,8 @@
|
||||
export DOCKER_BUILDKIT := 1
|
||||
export PROGRESS_NO_TRUNC := 1
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
export PATH := $(PATH):$(GOPATH)/bin
|
||||
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
||||
ARCH := $(shell uname -m)
|
||||
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
|
||||
APPNAME := mfer
|
||||
VERSION := 0.1.0
|
||||
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
|
||||
GOLDFLAGS += -X main.Version=$(VERSION)
|
||||
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||
.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build generate fuzz
|
||||
|
||||
.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme generate
|
||||
|
||||
default: fmt test
|
||||
# Makefile targets are thin shims; the implementations live in script/
|
||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||
# of README.md).
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
@@ -22,75 +10,32 @@ bootstrap:
|
||||
setup:
|
||||
@script/setup
|
||||
|
||||
run: ./bin/mfer
|
||||
./$<
|
||||
./$< gen
|
||||
|
||||
ci: test
|
||||
|
||||
test:
|
||||
@script/test
|
||||
|
||||
fuzz:
|
||||
@script/fuzz
|
||||
|
||||
fixme:
|
||||
@grep -nir fixme . | grep -v Makefile
|
||||
|
||||
check:
|
||||
@script/check
|
||||
|
||||
fmt-check:
|
||||
@script/fmt-check
|
||||
|
||||
# Halves of fmt-check, for environments that have only one toolchain:
|
||||
# the Docker lint stage has Go but no node, the markdown stage the reverse.
|
||||
fmt-check-go:
|
||||
@script/fmt-check-go
|
||||
|
||||
fmt-check-md:
|
||||
@script/prettier --check
|
||||
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
generate:
|
||||
@script/generate
|
||||
|
||||
bin/mfer: $(SOURCEFILES)
|
||||
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
|
||||
|
||||
clean:
|
||||
rm -rfv bin/mfer cmd/mfer/mfer *.dockerimage
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
fmt-check:
|
||||
@script/fmt-check
|
||||
|
||||
check:
|
||||
@script/check
|
||||
|
||||
docker:
|
||||
@script/docker
|
||||
|
||||
sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
|
||||
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
|
||||
docker save sneak/mfer | pv | zstdmt -19 > $@
|
||||
du -sh $@
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
godoc:
|
||||
open http://127.0.0.1:6060
|
||||
godoc -http=:6060
|
||||
build:
|
||||
@script/build
|
||||
|
||||
vendor.tzst: go.mod go.sum
|
||||
go mod tidy
|
||||
go mod vendor
|
||||
cd vendor && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
|
||||
rm -rf vendor
|
||||
mv $@.tmp $@
|
||||
generate:
|
||||
@script/generate
|
||||
|
||||
modcache.tzst: go.mod go.sum
|
||||
go mod tidy
|
||||
cd $(HOME)/go/pkg && chmod -R u+rw . && rm -rf mod sumdb
|
||||
go mod download -x
|
||||
cd $(shell go env GOMODCACHE) && tar -c . | pv | zstdmt -19 > $(PWD)/$@.tmp
|
||||
mv $@.tmp $@
|
||||
fuzz:
|
||||
@script/fuzz
|
||||
|
||||
@@ -29,7 +29,7 @@ is committed, so no `protoc` toolchain is required:
|
||||
```sh
|
||||
git clone https://git.eeqj.de/sneak/mfer.git
|
||||
cd mfer
|
||||
go build -o bin/mfer ./cmd/mfer
|
||||
make build
|
||||
```
|
||||
|
||||
Generate a manifest for a directory tree, verify it later, and fetch a published
|
||||
@@ -40,12 +40,14 @@ tree by URL:
|
||||
bin/mfer gen .
|
||||
|
||||
# Verify the files on disk against the manifest. Exits nonzero if any file
|
||||
# is missing or corrupted.
|
||||
# it lists is missing or corrupted; warns about files it does not list.
|
||||
bin/mfer check index.mf
|
||||
|
||||
# Download and cryptographically verify a tree published over HTTP: mfer
|
||||
# fetches <url>/index.mf, then downloads every file it lists.
|
||||
bin/mfer fetch https://example.com/tree/
|
||||
# Download and cryptographically verify a tree published over HTTP into
|
||||
# ./mirror: mfer fetches <url>/index.mf, downloads every file it lists,
|
||||
# skipping any already there with the right hash, then saves the manifest as
|
||||
# mirror/index.mf.
|
||||
bin/mfer fetch --dest mirror https://example.com/tree/
|
||||
```
|
||||
|
||||
Run `bin/mfer help` for the full command list, or `bin/mfer <command> --help`
|
||||
@@ -65,42 +67,51 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
||||
development workflow, and the Makefile targets are thin shims that call them. We
|
||||
provide:
|
||||
|
||||
- `script/bootstrap` — install all dependencies (Go, Go module download, and
|
||||
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
|
||||
idempotently; golangci-lint is not installed, it runs only in Docker
|
||||
- `script/bootstrap` — install all dependencies, idempotently: Go and the
|
||||
modules of both `go.mod` and `bin/tools/go.mod`; node (the version `.nvmrc`
|
||||
names, through nvm when there is no node on `PATH`) and yarn, plus the
|
||||
prettier version pinned in `package.json`/`yarn.lock`; and `protoc` 33.4,
|
||||
unpacked into `bin/protoc` from its release archive once the archive matches
|
||||
the sha256 the script holds for this platform. golangci-lint is not installed,
|
||||
it runs only in Docker
|
||||
- `script/setup` — make a fresh clone ready for development: runs
|
||||
`script/bootstrap`, then `script/install-precommit`
|
||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||
such as `script/docker`
|
||||
- `script/test` — run the test suite (`go test`); one test fails when
|
||||
`mfer/mf.proto` no longer matches the hash `script/generate` recorded
|
||||
- `script/build` (`make build`) — build the `mfer` binary into `bin/mfer`,
|
||||
stamped with the revision `mfer version` prints: the output of
|
||||
`git describe --tags --always --dirty`, as `script/docker` passes it
|
||||
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
|
||||
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
||||
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
||||
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file,
|
||||
and refuses to run with any other: `protoc` 33.4 (unpack
|
||||
`protoc-33.4-<platform>.zip` from
|
||||
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
|
||||
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
|
||||
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
|
||||
installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to
|
||||
`PATH`)
|
||||
`mfer/mf.pb.go`. It runs the `protoc` that `script/bootstrap` unpacks into
|
||||
`bin/protoc`, refusing any version but 33.4, and the `protoc-gen-go` that
|
||||
`bin/tools/go.mod` pins, which `go tool` builds from source checked against
|
||||
the hashes in `bin/tools/go.sum`
|
||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
||||
as ordinary tests
|
||||
- `script/lint` — run `golangci-lint` in Docker: builds only the `lint` stage of
|
||||
the `Dockerfile` (the Go format check, then the linter), uncached so it runs
|
||||
every time, then removes the image
|
||||
- `script/fmt` — format all code and docs (writes): `gofumpt` and
|
||||
- `script/fmt` — format all code and docs (writes): `script/gofumpt --write` and
|
||||
`script/prettier --write`
|
||||
- `script/gofumpt` — run `gofumpt` over every Go file in the repository in the
|
||||
given mode, `--write` or `--check`, at the version `bin/tools/go.mod` pins
|
||||
(built on demand by `go tool` from source checked against the hashes in
|
||||
`bin/tools/go.sum`, so nothing installs it); `script/fmt`, `script/fmt-check`
|
||||
and the Docker lint stage all go through it, so they cannot disagree about Go
|
||||
formatting
|
||||
- `script/prettier` — run prettier over the repository's canonical file set
|
||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||
`--check`; the single definition of that file set, so `script/fmt` and
|
||||
`script/fmt-check` cannot disagree about it
|
||||
- `script/fmt-check` — check formatting without writing: `script/fmt-check-go`
|
||||
plus `script/prettier --check`
|
||||
- `script/fmt-check-go` — the Go half of `script/fmt-check`, on its own, for the
|
||||
Docker lint stage, whose image has no node
|
||||
`--check`, with the prettier version `yarn.lock` pins, run by the node on
|
||||
`PATH` or else the one `script/bootstrap` installed through nvm; the single
|
||||
definition of that file set, so `script/fmt` and `script/fmt-check` cannot
|
||||
disagree about it
|
||||
- `script/fmt-check` — check formatting without writing:
|
||||
`script/gofumpt --check` plus `script/prettier --check`
|
||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
||||
- `script/docker` — build the Docker image tagged with the project name
|
||||
- `script/cibuild` — CI entrypoint: builds the image with the same command as
|
||||
@@ -251,10 +262,25 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
- `mfer check` / `mfer check .`
|
||||
- verifies checksums of all files in manifest, displaying error and exiting
|
||||
nonzero if any files are missing or corrupted
|
||||
- warns about each file under the base directory that the manifest does not
|
||||
list, hidden files included; with `--no-extra-files` each one is a failure
|
||||
instead
|
||||
- `mfer fetch https://example.com/stuff/`
|
||||
- fetches `/stuff/index.mf` and downloads all files listed in manifest,
|
||||
optionally resuming any that already exist locally, and assures
|
||||
cryptographic integrity of downloaded files.
|
||||
- fetches `/stuff/index.mf` and downloads all files listed in manifest into
|
||||
the current directory, or the one given with `--dest`, and assures
|
||||
cryptographic integrity of downloaded files. A file already there with the
|
||||
size and hash the manifest lists is skipped. Once every file is in place,
|
||||
the manifest is saved there as `index.mf`, so `mfer check` can verify the
|
||||
tree later. Each file is downloaded to a temp file beside it, such as
|
||||
`.a.txt.tmp` for `a.txt`, then moved into place. A manifest is refused
|
||||
before any file is downloaded if it lists a file where fetch writes
|
||||
another: at the temp file of a listed file, or at `index.mf` or
|
||||
`.index.mf.tmp` at the top of the tree. Names are compared in any letter
|
||||
case.
|
||||
- `mfer fetch --require-signature <fingerprint> https://example.com/stuff/`
|
||||
- as above, but first refuses a manifest not signed by the key with that
|
||||
fingerprint, as `mfer check --require-signature` does, before downloading
|
||||
any file.
|
||||
|
||||
# Implementation Plan
|
||||
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
main() {
|
||||
if [[ -n "${GITREV:-}" ]]; then
|
||||
echo "$GITREV"
|
||||
else
|
||||
git describe --tags --always --dirty=-dirty
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,22 @@
|
||||
// The developer tools this repo runs with `go tool`: gofumpt for
|
||||
// script/gofumpt and protoc-gen-go for script/generate. Kept out of the mfer
|
||||
// module so they add nothing to what mfer's users download. `go tool` builds
|
||||
// exactly the source whose hashes go.sum here records.
|
||||
module sneak.berlin/go/mfer/bin/tools
|
||||
|
||||
go 1.26.0
|
||||
|
||||
tool (
|
||||
google.golang.org/protobuf/cmd/protoc-gen-go
|
||||
mvdan.cc/gofumpt
|
||||
)
|
||||
|
||||
require (
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/tools v0.49.0 // indirect
|
||||
// protoc-gen-go v1.36.11, 2026-10-04
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
// gofumpt v0.12.0, 2026-10-04
|
||||
mvdan.cc/gofumpt v0.12.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
|
||||
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
|
||||
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
mvdan.cc/gofumpt v0.12.0 h1:1Lbudkz2kpM9Cjz2pL4M19u7q+GaEhCTNf7N9mfpcho=
|
||||
mvdan.cc/gofumpt v0.12.0/go.mod h1:SmBHHrljiZu/uoypeKup3rFzP6eoC9UwCp2iH5E3jZA=
|
||||
+12
-10
@@ -206,16 +206,21 @@ func countCheckFailures(
|
||||
}
|
||||
|
||||
// findExtraFiles reports files present on disk but absent from the
|
||||
// manifest, counting each as a failure.
|
||||
// manifest, and anything the search cannot read: each is a failure under
|
||||
// --no-extra-files, otherwise a warning.
|
||||
func findExtraFiles(ctx *cli.Context, chk *mfer.Checker, failures *int64) error {
|
||||
extraResults := make(chan mfer.Result, 1)
|
||||
extraDone := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
for result := range extraResults {
|
||||
*failures++
|
||||
if ctx.Bool("no-extra-files") {
|
||||
*failures++
|
||||
|
||||
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
|
||||
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
|
||||
} else {
|
||||
log.Warnf("%s: %s (%s)", result.Status, result.Path, result.Message)
|
||||
}
|
||||
}
|
||||
|
||||
close(extraDone)
|
||||
@@ -270,12 +275,9 @@ func runCheck(ctx *cli.Context, chk *mfer.Checker, showProgress bool) (int64, er
|
||||
// Wait for results processing to complete
|
||||
<-done
|
||||
|
||||
// Check for extra files if requested
|
||||
if ctx.Bool("no-extra-files") {
|
||||
err = findExtraFiles(ctx, chk, &failures)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
err = findExtraFiles(ctx, chk, &failures)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return failures, nil
|
||||
@@ -317,7 +319,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
||||
}
|
||||
|
||||
// Check signature requirement
|
||||
requiredSigner := ctx.String("require-signature")
|
||||
requiredSigner := ctx.String(flagRequireSignature)
|
||||
if requiredSigner != "" {
|
||||
err = verifyRequiredSigner(ctx.Context, chk, requiredSigner)
|
||||
if err != nil {
|
||||
|
||||
+141
-16
@@ -705,30 +705,155 @@ func TestNoExtraFilesWithSubdirectory(t *testing.T) {
|
||||
"check should fail when extra files exist in subdirectory")
|
||||
}
|
||||
|
||||
func TestCheckWithoutNoExtraFilesIgnoresExtra(t *testing.T) {
|
||||
// TestCheckWarnsAboutUnlistedFiles adds one file the manifest does not list
|
||||
// to a tree that had none: it gets one warning and the check passes, unless
|
||||
// --no-extra-files makes it a failure. --quiet hides the warning, not the
|
||||
// failure.
|
||||
func TestCheckWarnsAboutUnlistedFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
for name, unlisted := range map[string]string{
|
||||
"regular file": "extra.txt",
|
||||
"dotfile": ".hidden",
|
||||
"file in hidden directory": ".git/config",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Create test file
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello")
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello")
|
||||
|
||||
// Generate manifest
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testManifest, testDir}, fs)
|
||||
exitCode := runCLI(opts)
|
||||
require.Equal(t, 0, exitCode)
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
// Add extra file
|
||||
writeTestFile(t, fs, "/testdir/extra.txt", "extra")
|
||||
check := func(flags ...string) (int, string) {
|
||||
args := append([]string{testApp, cmdCheck, testFlagBase, testDir}, flags...)
|
||||
opts := testOpts(append(args, testManifest), fs)
|
||||
|
||||
return runCLI(opts), testStderr(t, opts)
|
||||
}
|
||||
|
||||
exitCode, stderr := check()
|
||||
assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
|
||||
assert.NotContains(t, stderr, "not in manifest")
|
||||
|
||||
writeTestFile(t, fs, filepath.Join(testDir, unlisted), "unlisted")
|
||||
|
||||
exitCode, stderr = check()
|
||||
assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
|
||||
assert.Equal(t, 1, strings.Count(stderr, "not in manifest"), stderr)
|
||||
assert.Contains(t, stderr, unlisted)
|
||||
|
||||
exitCode, stderr = check("-q")
|
||||
assert.Equal(t, 0, exitCode, "stderr: %s", stderr)
|
||||
assert.NotContains(t, stderr, "not in manifest")
|
||||
|
||||
exitCode, stderr = check(testFlagNoExtra)
|
||||
assert.Equal(t, 1, exitCode, "stderr: %s", stderr)
|
||||
assert.Contains(t, stderr, unlisted)
|
||||
|
||||
exitCode, _ = check("-q", testFlagNoExtra)
|
||||
assert.Equal(t, 1, exitCode)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckNeverReportsManifest keeps the manifest inside the checked tree,
|
||||
// under several names and path spellings, and names the tree both directly
|
||||
// and through a symlink: the manifest is never reported, even under
|
||||
// --no-extra-files. The manifest is recognized by file identity, which needs
|
||||
// the real filesystem.
|
||||
func TestCheckNeverReportsManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Manifest paths are relative to the checked tree.
|
||||
for name, manifest := range map[string]string{
|
||||
"default name": defaultManifestName,
|
||||
"hidden name": ".index.mf",
|
||||
"other name in a subdirectory": "sub/listing.mf",
|
||||
"path spelled through ..": "sub/../index.mf",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A temp dir holding data/tree and link, a symlink to data.
|
||||
root := t.TempDir()
|
||||
tree := filepath.Join(root, "data", "tree")
|
||||
// Not filepath.Join, which would clean away a "..".
|
||||
manifestPath := tree + "/" + manifest
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
|
||||
require.NoError(t,
|
||||
os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
|
||||
writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", "-o", manifestPath, tree,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
for _, base := range []string{tree, filepath.Join(root, "link", "tree")} {
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, testFlagNoExtra, testFlagBase, base, manifestPath,
|
||||
}, fs)
|
||||
assert.Equal(t, 0, runCLI(opts),
|
||||
"base %s, stderr: %s", base, testStderr(t, opts))
|
||||
assert.NotContains(t, testStderr(t, opts), "not in manifest")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// unlistableDirFs is a filesystem on which one directory cannot be listed.
|
||||
type unlistableDirFs struct {
|
||||
afero.Fs
|
||||
|
||||
dir string
|
||||
}
|
||||
|
||||
//nolint:ireturn // Open must return afero.File to satisfy afero.Fs.
|
||||
func (f unlistableDirFs) Open(name string) (afero.File, error) {
|
||||
if name == f.dir {
|
||||
return nil, os.ErrPermission
|
||||
}
|
||||
|
||||
return f.Fs.Open(name)
|
||||
}
|
||||
|
||||
// TestCheckWithUnlistableDirectory has a directory under the base that
|
||||
// cannot be listed, followed by an unlisted file: both are warned about and
|
||||
// the check still passes, unless --no-extra-files is given.
|
||||
func TestCheckWithUnlistableDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mem := afero.NewMemMapFs()
|
||||
require.NoError(t, mem.MkdirAll("/testdir/locked", 0o755))
|
||||
writeTestFile(t, mem, testFile1, "hello")
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", "-o", testManifest, testDir,
|
||||
}, mem)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
// Directories are searched in name order, so this comes after "locked".
|
||||
writeTestFile(t, mem, "/testdir/unlisted.txt", "unlisted")
|
||||
|
||||
fs := unlistableDirFs{Fs: mem, dir: "/testdir/locked"}
|
||||
|
||||
opts = testOpts([]string{testApp, cmdCheck, testFlagBase, testDir, testManifest}, fs)
|
||||
assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
assert.Contains(t, testStderr(t, opts), os.ErrPermission.Error())
|
||||
assert.Contains(t, testStderr(t, opts), "unlisted.txt")
|
||||
|
||||
// Check WITHOUT --no-extra-files (should pass - extra files ignored)
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q", testFlagBase, testDir, testManifest,
|
||||
testApp, cmdCheck, testFlagNoExtra, testFlagBase, testDir, testManifest,
|
||||
}, fs)
|
||||
exitCode = runCLI(opts)
|
||||
assert.Equal(t, 0, exitCode,
|
||||
"check without --no-extra-files should ignore extra files")
|
||||
assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
assert.Contains(t, testStderr(t, opts), "unlisted.txt")
|
||||
}
|
||||
|
||||
func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
||||
|
||||
+22
-12
@@ -111,9 +111,10 @@ func TestVerifyRequiredSignerMessages(t *testing.T) {
|
||||
// string; the required signer is a fixed value that cannot match it. Requires
|
||||
// gpg and is skipped where it is absent, as the other signing tests are.
|
||||
//
|
||||
//nolint:paralleltest // signedChecker calls t.Setenv, which bars t.Parallel
|
||||
//nolint:paralleltest // signedManifest calls t.Setenv, which bars t.Parallel
|
||||
func TestSignerMismatchMessage(t *testing.T) {
|
||||
chk := signedChecker(t)
|
||||
chk := signedChecker(t,
|
||||
signedManifest(t, map[string][]byte{"f.txt": []byte("signed file")}))
|
||||
|
||||
embeddedFP, err := chk.ExtractEmbeddedSigningKeyFP(context.Background())
|
||||
require.NoError(t, err)
|
||||
@@ -125,9 +126,10 @@ func TestSignerMismatchMessage(t *testing.T) {
|
||||
" does not match required "+msgFpB)
|
||||
}
|
||||
|
||||
// signedChecker builds a Checker over a manifest signed by a throwaway GPG
|
||||
// key generated in a temporary GNUPGHOME.
|
||||
func signedChecker(t *testing.T) *mfer.Checker {
|
||||
// signedManifest returns a manifest of files signed by a throwaway GPG key
|
||||
// generated in a temporary GNUPGHOME, which it leaves set for the rest of
|
||||
// the test.
|
||||
func signedManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
_, err := exec.LookPath("gpg")
|
||||
@@ -159,17 +161,25 @@ func signedChecker(t *testing.T) *mfer.Checker {
|
||||
b := mfer.NewBuilder()
|
||||
b.SetSigningOptions(&mfer.SigningOptions{KeyID: mfer.GPGKeyID("test@mfer.test")})
|
||||
|
||||
content := []byte("signed file")
|
||||
_, err = b.AddFile("f.txt", mfer.FileSize(len(content)), mfer.ModTime{},
|
||||
bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
for path, content := range files {
|
||||
_, err = b.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)),
|
||||
mfer.ModTime{}, bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
require.NoError(t, b.Build(context.Background(), &buf))
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// signedChecker builds a Checker over manifest, a signed manifest.
|
||||
func signedChecker(t *testing.T, manifest []byte) *mfer.Checker {
|
||||
t.Helper()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644))
|
||||
require.NoError(t, afero.WriteFile(fs, "/index.mf", manifest, 0o644))
|
||||
|
||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||
ManifestPath: "/index.mf",
|
||||
@@ -300,7 +310,7 @@ func TestFetchFileHTTPStatusMessage(t *testing.T) {
|
||||
|
||||
// downloadFile logs each retry of the 500 to the process-global logger.
|
||||
err := runLocked(func() error {
|
||||
return downloadFile(context.Background(), testClient(), server.URL+"/x", "x",
|
||||
return downloadFile(context.Background(), testClient(), server.URL+"/x", ".", "x",
|
||||
&mfer.MFFilePath{}, nil)
|
||||
})
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
@@ -355,7 +365,7 @@ func TestSizeMismatchMessage(t *testing.T) {
|
||||
|
||||
// finishDownload returns the size-mismatch error before it touches the
|
||||
// paths, digest, or entry, so those can be zero here.
|
||||
err := finishDownload("", "", 9, 10, nil, nil, nil, nil)
|
||||
err := finishDownload("", "", "", 9, 10, nil, nil, nil, nil)
|
||||
require.ErrorIs(t, err, errSizeMismatch)
|
||||
assert.EqualError(t, err, "size mismatch: expected 10 bytes, got 9")
|
||||
}
|
||||
|
||||
+304
-93
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/urfave/cli/v2"
|
||||
"sneak.berlin/go/mfer/internal/log"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
@@ -90,6 +91,10 @@ var (
|
||||
// errHashMismatch indicates a downloaded file whose hash matches no
|
||||
// manifest hash.
|
||||
errHashMismatch = errors.New("hash mismatch")
|
||||
// errNameClash indicates a manifest that lists a file where fetch
|
||||
// writes another file.
|
||||
errNameClash = errors.New(
|
||||
"manifest lists a file where fetch writes another file")
|
||||
)
|
||||
|
||||
// DownloadProgress reports the progress of a single file download.
|
||||
@@ -239,20 +244,34 @@ func manifestBaseURL(manifestURL string) (*url.URL, error) {
|
||||
return parsed.JoinPath(".."), nil
|
||||
}
|
||||
|
||||
// downloadManifestFiles downloads every file in the manifest, reporting
|
||||
// progress on the progress channel.
|
||||
// downloadManifestFiles downloads every file in the manifest into dest,
|
||||
// reporting progress on the progress channel. A file already present in
|
||||
// dest is skipped. It returns how many files it downloaded and their
|
||||
// total size.
|
||||
func downloadManifestFiles(
|
||||
ctx context.Context,
|
||||
client retryingClient,
|
||||
baseURL *url.URL,
|
||||
dest string,
|
||||
files []*mfer.MFFilePath,
|
||||
progress chan<- DownloadProgress,
|
||||
) error {
|
||||
) (int, int64, error) {
|
||||
var (
|
||||
downloaded int
|
||||
downloadedBytes int64
|
||||
)
|
||||
|
||||
for _, f := range files {
|
||||
// Sanitize the path to prevent path traversal attacks
|
||||
localPath, err := sanitizePath(f.GetPath())
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid path in manifest: %w", err)
|
||||
return 0, 0, fmt.Errorf("invalid path in manifest: %w", err)
|
||||
}
|
||||
|
||||
if alreadyPresent(dest, localPath, f) {
|
||||
log.Infof("skipping %s: already present", f.GetPath())
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// JoinPath takes escaped path text, so a name such as "100%.txt"
|
||||
@@ -260,13 +279,53 @@ func downloadManifestFiles(
|
||||
fileURL := baseURL.JoinPath(encodeFilePath(f.GetPath())).String()
|
||||
log.Infof("fetching %s", f.GetPath())
|
||||
|
||||
err = downloadFile(ctx, client, fileURL, localPath, f, progress)
|
||||
err = downloadFile(ctx, client, fileURL, dest, localPath, f, progress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
|
||||
return 0, 0, fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
|
||||
}
|
||||
|
||||
downloaded++
|
||||
downloadedBytes += f.GetSize()
|
||||
}
|
||||
|
||||
return nil
|
||||
return downloaded, downloadedBytes, nil
|
||||
}
|
||||
|
||||
// alreadyPresent reports whether localPath under dest is a regular file
|
||||
// with the size and one of the hashes the manifest lists for entry. It
|
||||
// hashes the whole file, since a matching size alone would accept a
|
||||
// corrupted or partly written one. A file it cannot read, or reaches only
|
||||
// through a symlink, is not present: fetch downloads it, and the download
|
||||
// reports the problem.
|
||||
func alreadyPresent(dest, localPath string, entry *mfer.MFFilePath) bool {
|
||||
if checkNoSymlinks(dest, localPath) != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
path := filepath.Join(dest, localPath)
|
||||
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() != entry.GetSize() {
|
||||
return false
|
||||
}
|
||||
|
||||
// G304: localPath is a relative path that sanitizePath keeps inside
|
||||
// dest as text, and checkNoSymlinks just found no symlink in it.
|
||||
f, err := os.Open(path) //nolint:gosec // G304: see comment above
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
h := sha256.New()
|
||||
|
||||
_, err = io.Copy(h, f)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return verifyDownloadedHash(h.Sum(nil), entry) == nil
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
||||
@@ -291,43 +350,22 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
||||
firstDelay: firstRetryDelay,
|
||||
}
|
||||
|
||||
log.Infof("fetching manifest from %s", manifestURL)
|
||||
|
||||
// Read the whole manifest before parsing it, so that a connection
|
||||
// lost partway through is retried rather than reported as a bad
|
||||
// manifest.
|
||||
var manifestData []byte
|
||||
|
||||
err = client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
|
||||
var readErr error
|
||||
|
||||
manifestData, readErr = io.ReadAll(resp.Body)
|
||||
|
||||
return readErr
|
||||
})
|
||||
manifestData, files, err := fetchManifest(ctx, client, manifestURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch manifest: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Parse manifest
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse manifest: %w", err)
|
||||
}
|
||||
|
||||
files := manifest.Files()
|
||||
log.Infof("manifest contains %d files", len(files))
|
||||
|
||||
// Compute base URL (directory containing manifest)
|
||||
baseURL, err := manifestBaseURL(manifestURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Calculate total bytes to download
|
||||
var totalBytes int64
|
||||
for _, f := range files {
|
||||
totalBytes += f.GetSize()
|
||||
dest := ctx.String(flagDest)
|
||||
|
||||
err = os.MkdirAll(dest, dirPerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create destination directory %s: %w", dest, err)
|
||||
}
|
||||
|
||||
// Create progress channel and start progress reporter goroutine
|
||||
@@ -340,7 +378,8 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
||||
startTime := time.Now()
|
||||
|
||||
// Download each file
|
||||
dlErr := downloadManifestFiles(ctx.Context, client, baseURL, files, progress)
|
||||
downloaded, downloadedBytes, dlErr := downloadManifestFiles(
|
||||
ctx.Context, client, baseURL, dest, files, progress)
|
||||
|
||||
close(progress)
|
||||
<-done
|
||||
@@ -349,15 +388,168 @@ func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
||||
return dlErr
|
||||
}
|
||||
|
||||
// Saved only now that every file is in place and verified, so that
|
||||
// "mfer check" can verify the tree later.
|
||||
err = saveManifest(dest, manifestData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save manifest: %w", err)
|
||||
}
|
||||
|
||||
// Print summary
|
||||
elapsed := time.Since(startTime)
|
||||
avgBytesPerSec := float64(totalBytes) / elapsed.Seconds()
|
||||
avgBytesPerSec := float64(downloadedBytes) / elapsed.Seconds()
|
||||
avgRate := formatBitrate(avgBytesPerSec * bitsPerByte)
|
||||
log.Infof("downloaded %d files (%s) in %.1fs (%s avg)",
|
||||
len(files),
|
||||
humanize.IBytes(safeUint64(totalBytes)),
|
||||
log.Infof("downloaded %d files (%s) in %.1fs (%s avg), skipped %d already present",
|
||||
downloaded,
|
||||
humanize.IBytes(safeUint64(downloadedBytes)),
|
||||
elapsed.Seconds(),
|
||||
avgRate)
|
||||
avgRate,
|
||||
len(files)-downloaded)
|
||||
log.Infof("saved manifest to %s", filepath.Join(dest, defaultManifestName))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchManifest downloads the manifest at manifestURL and parses it,
|
||||
// enforcing --require-signature if it is given and refusing a manifest
|
||||
// that lists a file where fetch writes another. It returns the manifest as
|
||||
// downloaded, to be saved once the files are in place, and the files it
|
||||
// lists.
|
||||
func fetchManifest(
|
||||
ctx *cli.Context, client retryingClient, manifestURL string,
|
||||
) ([]byte, []*mfer.MFFilePath, error) {
|
||||
log.Infof("fetching manifest from %s", manifestURL)
|
||||
|
||||
// Read the whole manifest before parsing it, so that a connection
|
||||
// lost partway through is retried rather than reported as a bad
|
||||
// manifest.
|
||||
var manifestData []byte
|
||||
|
||||
err := client.get(ctx.Context, manifestURL, func(resp *http.Response) error {
|
||||
var readErr error
|
||||
|
||||
manifestData, readErr = io.ReadAll(resp.Body)
|
||||
|
||||
return readErr
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err)
|
||||
}
|
||||
|
||||
// Parse manifest
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse manifest: %w", err)
|
||||
}
|
||||
|
||||
requiredSigner := ctx.String(flagRequireSignature)
|
||||
if requiredSigner != "" {
|
||||
err = verifyFetchedSigner(ctx, manifestData, requiredSigner)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
files := manifest.Files()
|
||||
|
||||
err = checkNoNameClash(files)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
log.Infof("manifest contains %d files", len(files))
|
||||
|
||||
return manifestData, files, nil
|
||||
}
|
||||
|
||||
// checkNoNameClash returns an error if files lists a file, or a directory
|
||||
// a file is in, under a name where fetch writes another file: the temp
|
||||
// file it downloads a listed file to, or, at the top of the tree, the
|
||||
// saved manifest or its temp file. fetch would remove or replace what is
|
||||
// listed there, or fail partway, leaving a tree check rejects. Names are
|
||||
// compared ignoring case, since on a case-insensitive filesystem INDEX.MF
|
||||
// and index.mf are one file.
|
||||
func checkNoNameClash(files []*mfer.MFFilePath) error {
|
||||
sep := string(filepath.Separator)
|
||||
|
||||
// written maps each name fetch writes, other than the listed files
|
||||
// themselves, in lower case, to the file it writes there.
|
||||
written := map[string]string{
|
||||
defaultManifestName: "the saved manifest",
|
||||
tempPathFor(defaultManifestName): "the saved manifest's temp file",
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
tmpPath := tempPathFor(filepath.Clean(f.GetPath()))
|
||||
written[strings.ToLower(tmpPath)] = "the temp file for " + f.GetPath()
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
// Look up each directory on the file's path, then the file itself.
|
||||
parts := strings.Split(strings.ToLower(filepath.Clean(f.GetPath())), sep)
|
||||
|
||||
for i := range parts {
|
||||
what, ok := written[strings.Join(parts[:i+1], sep)]
|
||||
if ok {
|
||||
return fmt.Errorf("%w: %s (%s)", errNameClash, f.GetPath(), what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyFetchedSigner enforces --require-signature on the fetched manifest
|
||||
// exactly as check does. verifyRequiredSigner takes a Checker, which loads
|
||||
// its manifest from a file, so the manifest is handed to it as a file in
|
||||
// memory.
|
||||
func verifyFetchedSigner(
|
||||
ctx *cli.Context, manifestData []byte, requiredSigner string,
|
||||
) error {
|
||||
memFs := afero.NewMemMapFs()
|
||||
manifestPath := "/" + defaultManifestName
|
||||
|
||||
err := afero.WriteFile(memFs, manifestPath, manifestData, filePerms)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||
ManifestPath: manifestPath,
|
||||
BasePath: "/",
|
||||
Fs: memFs,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load manifest: %w", err)
|
||||
}
|
||||
|
||||
return verifyRequiredSigner(ctx.Context, chk, requiredSigner)
|
||||
}
|
||||
|
||||
// saveManifest writes the fetched manifest into dest under the default
|
||||
// manifest name, the way fetch writes every file: to a new temp file that
|
||||
// is then renamed into place.
|
||||
func saveManifest(dest string, manifestData []byte) error {
|
||||
tmpPath := tempPathFor(defaultManifestName)
|
||||
|
||||
out, err := createTempFile(dest, tmpPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, writeErr := out.Write(manifestData)
|
||||
closeErr := out.Close()
|
||||
|
||||
err = errors.Join(writeErr, closeErr)
|
||||
if err == nil {
|
||||
err = moveIntoPlace(dest, tmpPath, defaultManifestName)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
_ = os.Remove(filepath.Join(dest, tmpPath))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -401,14 +593,15 @@ func sanitizePath(p string) (string, error) {
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
// checkNoSymlinks returns an error if any part of the relative path p
|
||||
// already exists as a symlink. sanitizePath checks p only as text, so
|
||||
// without this a symlink inside the target directory could send a write
|
||||
// to p outside of it. Parts that do not exist yet are fine: fetch creates
|
||||
// them as plain directories and files. Call it immediately before each
|
||||
// write: a symlink created after it returns is not caught.
|
||||
func checkNoSymlinks(p string) error {
|
||||
current := ""
|
||||
// checkNoSymlinks returns an error if any part of p, a path relative to
|
||||
// dest, already exists under dest as a symlink. dest itself is the user's
|
||||
// choice and may be one. sanitizePath checks p only as text, so without
|
||||
// this a symlink inside dest could send a write to p outside of it. Parts
|
||||
// that do not exist yet are fine: fetch creates them as plain directories
|
||||
// and files. Call it immediately before each write: a symlink created
|
||||
// after it returns is not caught.
|
||||
func checkNoSymlinks(dest, p string) error {
|
||||
current := dest
|
||||
|
||||
for _, part := range strings.Split(p, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, part)
|
||||
@@ -548,13 +741,14 @@ func verifyDownloadedHash(digest []byte, entry *mfer.MFFilePath) error {
|
||||
return errHashMismatch
|
||||
}
|
||||
|
||||
// downloadFile downloads a URL to a local file path with hash verification.
|
||||
// It downloads to a temporary file, verifies the hash, then renames to the final path.
|
||||
// Progress is reported via the progress channel.
|
||||
// downloadFile downloads a URL to localPath, a path relative to dest, with
|
||||
// hash verification. It downloads to a temporary file, verifies the hash,
|
||||
// then renames to the final path. Progress is reported via the progress
|
||||
// channel.
|
||||
func downloadFile(
|
||||
ctx context.Context,
|
||||
client retryingClient,
|
||||
fileURL, localPath string,
|
||||
fileURL, dest, localPath string,
|
||||
entry *mfer.MFFilePath,
|
||||
progress chan<- DownloadProgress,
|
||||
) error {
|
||||
@@ -568,11 +762,13 @@ func downloadFile(
|
||||
// Create parent directories if needed
|
||||
dir := filepath.Dir(localPath)
|
||||
if dir != "" && dir != "." {
|
||||
err = checkNoSymlinks(dir)
|
||||
err = checkNoSymlinks(dest, dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir = filepath.Join(dest, dir)
|
||||
|
||||
err = os.MkdirAll(dir, dirPerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %w", dir, err)
|
||||
@@ -582,17 +778,61 @@ func downloadFile(
|
||||
tmpPath := tempPathFor(localPath)
|
||||
|
||||
return client.get(ctx, fileURL, func(resp *http.Response) error {
|
||||
return saveResponse(resp, tmpPath, localPath, entry, progress)
|
||||
return saveResponse(resp, dest, tmpPath, localPath, entry, progress)
|
||||
})
|
||||
}
|
||||
|
||||
// createTempFile creates tmpPath, a path relative to dest, as a new empty
|
||||
// file.
|
||||
func createTempFile(dest, tmpPath string) (*os.File, error) {
|
||||
err := checkNoSymlinks(dest, tmpPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
path := filepath.Join(dest, tmpPath)
|
||||
|
||||
// Remove whatever is at tmpPath, such as a leftover from an
|
||||
// interrupted run, rather than write into it: it may be a hard link
|
||||
// to a file outside dest, and removing a hard link removes only this
|
||||
// name. If the removal fails, O_EXCL below makes the create fail.
|
||||
_ = os.Remove(path)
|
||||
|
||||
// Create the temp file only if nothing is at tmpPath (O_EXCL).
|
||||
//
|
||||
// G304: tmpPath is a relative path that sanitizePath keeps inside dest
|
||||
// as text, and checkNoSymlinks just found no symlink in it.
|
||||
out, err := os.OpenFile( //nolint:gosec // G304: see comment above
|
||||
path, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// moveIntoPlace renames tmpPath to localPath, both relative to dest.
|
||||
func moveIntoPlace(dest, tmpPath, localPath string) error {
|
||||
err := checkNoSymlinks(dest, localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Rename(filepath.Join(dest, tmpPath), filepath.Join(dest, localPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveResponse writes resp's body to tmpPath, verifies it against entry,
|
||||
// and renames it to localPath. It starts a new temp file each time and
|
||||
// removes it on failure, so a retry after a failed try never appends to
|
||||
// or keeps a partial file.
|
||||
// and renames it to localPath, both paths relative to dest. It starts a
|
||||
// new temp file each time and removes it on failure, so a retry after a
|
||||
// failed try never appends to or keeps a partial file.
|
||||
func saveResponse(
|
||||
resp *http.Response,
|
||||
tmpPath, localPath string,
|
||||
dest, tmpPath, localPath string,
|
||||
entry *mfer.MFFilePath,
|
||||
progress chan<- DownloadProgress,
|
||||
) error {
|
||||
@@ -604,29 +844,11 @@ func saveResponse(
|
||||
totalBytes = expectedSize
|
||||
}
|
||||
|
||||
err := checkNoSymlinks(tmpPath)
|
||||
out, err := createTempFile(dest, tmpPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove whatever is at tmpPath, such as a leftover from an
|
||||
// interrupted run, rather than write into it: it may be a hard link
|
||||
// to a file outside the target directory, and removing a hard link
|
||||
// removes only this name. If the removal fails, O_EXCL below makes
|
||||
// the create fail.
|
||||
_ = os.Remove(tmpPath)
|
||||
|
||||
// Create the temp file only if nothing is at tmpPath (O_EXCL).
|
||||
//
|
||||
// G304: tmpPath is a relative path that sanitizePath keeps inside the
|
||||
// target directory as text, and checkNoSymlinks just found no symlink
|
||||
// in it.
|
||||
out, err := os.OpenFile( //nolint:gosec // G304: see comment above
|
||||
tmpPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
|
||||
// Set up hash computation
|
||||
h := sha256.New()
|
||||
|
||||
@@ -646,10 +868,10 @@ func saveResponse(
|
||||
closeErr := out.Close()
|
||||
|
||||
err = finishDownload(
|
||||
tmpPath, localPath, written, expectedSize, h.Sum(nil), entry,
|
||||
dest, tmpPath, localPath, written, expectedSize, h.Sum(nil), entry,
|
||||
copyErr, closeErr)
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
_ = os.Remove(filepath.Join(dest, tmpPath))
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -660,7 +882,7 @@ func saveResponse(
|
||||
// finishDownload validates the copy result, verifies size and hash, and
|
||||
// moves the temp file into place. On error the caller removes tmpPath.
|
||||
func finishDownload(
|
||||
tmpPath, localPath string,
|
||||
dest, tmpPath, localPath string,
|
||||
written, expectedSize int64,
|
||||
digest []byte,
|
||||
entry *mfer.MFFilePath,
|
||||
@@ -686,16 +908,5 @@ func finishDownload(
|
||||
return err
|
||||
}
|
||||
|
||||
err = checkNoSymlinks(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Rename temp file to final path
|
||||
err = os.Rename(tmpPath, localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return moveIntoPlace(dest, tmpPath, localPath)
|
||||
}
|
||||
|
||||
+358
-33
@@ -13,6 +13,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -338,7 +339,7 @@ func TestFetchFromHTTP(t *testing.T) {
|
||||
|
||||
fileURL := baseURL + f.GetPath()
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
fileURL, localPath, f, progress)
|
||||
fileURL, ".", localPath, f, progress)
|
||||
require.NoError(t, err, "failed to download %s", f.GetPath())
|
||||
}
|
||||
|
||||
@@ -386,7 +387,7 @@ func TestFetchHashMismatch(t *testing.T) {
|
||||
|
||||
// Try to download - should fail with hash mismatch
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/file.txt", testFileTxt, files[0], nil)
|
||||
server.URL+"/file.txt", ".", testFileTxt, files[0], nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "mismatch")
|
||||
|
||||
@@ -432,7 +433,7 @@ func TestFetchSizeMismatch(t *testing.T) {
|
||||
|
||||
// Try to download - should fail with size mismatch
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/file.txt", testFileTxt, files[0], nil)
|
||||
server.URL+"/file.txt", ".", testFileTxt, files[0], nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "size mismatch")
|
||||
|
||||
@@ -490,7 +491,7 @@ func TestFetchProgress(t *testing.T) {
|
||||
|
||||
// Download
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/large.txt", "large.txt", files[0], progress)
|
||||
server.URL+"/large.txt", ".", "large.txt", files[0], progress)
|
||||
close(progress)
|
||||
<-done
|
||||
|
||||
@@ -513,24 +514,51 @@ func TestFetchProgress(t *testing.T) {
|
||||
assert.Equal(t, content, downloaded)
|
||||
}
|
||||
|
||||
// TestFetchRefusesSymlinks runs fetch into a destination directory that
|
||||
// holds a symlink pointing outside it, in each of the three places fetch
|
||||
// writes: a parent directory, the temp file, and the file itself, which
|
||||
// the temp file is renamed onto; and once as a directory inside a plain
|
||||
// directory. The fetch must fail and nothing outside may change.
|
||||
// TestFetchRefusesSymlinks runs fetch with --dest naming a directory other
|
||||
// than the current one, which holds a symlink pointing outside it, in each
|
||||
// place fetch writes: a parent directory, the temp file, the file itself,
|
||||
// which the temp file is renamed onto, and the saved manifest's temp file
|
||||
// and final name; and once as a directory inside a plain directory. The
|
||||
// fetch must fail, and neither the outside directory nor the current one
|
||||
// may change.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestFetchRefusesSymlinks(t *testing.T) {
|
||||
// What a link standing for a file points to: a file outside that does
|
||||
// not exist yet.
|
||||
const newFile = "new.txt"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
entry string // the manifest's only file
|
||||
link string // symlink placed in the destination directory
|
||||
target string // what link points to, relative to the outside directory
|
||||
name string
|
||||
entry string // the manifest's only file
|
||||
link string // symlink placed in the destination directory
|
||||
target string // what link points to, relative to the outside directory
|
||||
failure string // what fetch reports it was doing when it found link
|
||||
}{
|
||||
{"parent directory", "sub/deeper/file.txt", "sub", "."},
|
||||
{"directory inside a plain directory", "docs/data/passwd", "docs/data", "."},
|
||||
{"temp file", testFileTxt, ".file.txt.tmp", "new.txt"},
|
||||
{"file", testFileTxt, testFileTxt, "new.txt"},
|
||||
{
|
||||
"parent directory", "sub/deeper/file.txt", "sub", ".",
|
||||
"failed to download sub/deeper/file.txt",
|
||||
},
|
||||
{
|
||||
"directory inside a plain directory", "docs/data/passwd", "docs/data", ".",
|
||||
"failed to download docs/data/passwd",
|
||||
},
|
||||
{
|
||||
"temp file", testFileTxt, ".file.txt.tmp", newFile,
|
||||
"failed to download " + testFileTxt,
|
||||
},
|
||||
{
|
||||
"file", testFileTxt, testFileTxt, newFile,
|
||||
"failed to download " + testFileTxt,
|
||||
},
|
||||
{
|
||||
"manifest temp file", testFileTxt, tempPathFor(defaultManifestName), newFile,
|
||||
"failed to save manifest",
|
||||
},
|
||||
{
|
||||
"manifest", testFileTxt, defaultManifestName, newFile,
|
||||
"failed to save manifest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -545,23 +573,61 @@ func TestFetchRefusesSymlinks(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
outside := t.TempDir()
|
||||
cwd := chdirTemp(t)
|
||||
dest := t.TempDir()
|
||||
link := filepath.Join(dest, tt.link)
|
||||
|
||||
chdirTemp(t)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(tt.link), 0o750))
|
||||
require.NoError(t, os.Symlink(filepath.Join(outside, tt.target), tt.link))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(link), 0o750))
|
||||
require.NoError(t, os.Symlink(filepath.Join(outside, tt.target), link))
|
||||
|
||||
opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
|
||||
}, afero.NewOsFs())
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), "failed to download "+tt.entry+
|
||||
": symlink in path not allowed: "+tt.link)
|
||||
assert.Contains(t, testStderr(t, opts),
|
||||
tt.failure+": symlink in path not allowed: "+link)
|
||||
|
||||
written, err := os.ReadDir(outside)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, written, "fetch wrote outside the destination")
|
||||
|
||||
written, err = os.ReadDir(cwd)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, written, "fetch wrote to the current directory")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchDoesNotSkipThroughSymlink runs fetch with --dest holding a
|
||||
// symlink to a directory outside it, where the file the manifest lists
|
||||
// through that symlink already sits with the listed content. fetch must
|
||||
// not take that file as already present: it must fail on the symlink, as
|
||||
// the download would, and leave the outside file alone.
|
||||
func TestFetchDoesNotSkipThroughSymlink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := []byte("fetched")
|
||||
files := map[string][]byte{"sub/" + testFileTxt: content}
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
|
||||
defer server.Close()
|
||||
|
||||
outside := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(outside, testFileTxt), content, 0o600))
|
||||
|
||||
dest := t.TempDir()
|
||||
link := filepath.Join(dest, "sub")
|
||||
require.NoError(t, os.Symlink(outside, link))
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
|
||||
}, afero.NewOsFs())
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts),
|
||||
"failed to download sub/"+testFileTxt+": symlink in path not allowed: "+link)
|
||||
assert.Equal(t, map[string][]byte{testFileTxt: content}, filesUnder(t, outside))
|
||||
}
|
||||
|
||||
// TestFetchReplacesHardLinkAtTempName runs fetch into a destination
|
||||
// directory that holds, at the temp file's name, a hard link to a file
|
||||
// outside it. To fetch that is an ordinary leftover from an interrupted
|
||||
@@ -787,7 +853,7 @@ func TestDownloadFileRetriesToSuccess(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/"+testFileTxt, testFileTxt, manifest.Files()[0], nil)
|
||||
server.URL+"/"+testFileTxt, ".", testFileTxt, manifest.Files()[0], nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(3), requests.Load())
|
||||
|
||||
@@ -878,7 +944,7 @@ func TestFetchEscapedPaths(t *testing.T) {
|
||||
|
||||
// TestFetchTree runs fetch on a tree with nested directories. Every file
|
||||
// the manifest lists must land under its own path with its own content,
|
||||
// and nothing else may be left in the destination.
|
||||
// beside the manifest, and nothing else may be left in the destination.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestFetchTree(t *testing.T) {
|
||||
@@ -889,7 +955,9 @@ func TestFetchTree(t *testing.T) {
|
||||
"other/deep/est.txt": []byte("in a second directory"),
|
||||
}
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
|
||||
manifest := manifestOf(t, files)
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifest, files))
|
||||
defer server.Close()
|
||||
|
||||
dest := chdirTemp(t)
|
||||
@@ -897,7 +965,9 @@ func TestFetchTree(t *testing.T) {
|
||||
opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
assert.Equal(t, files, filesUnder(t, dest))
|
||||
want := maps.Clone(files)
|
||||
want[defaultManifestName] = manifest
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
}
|
||||
|
||||
// TestFetchFailsOnHashMismatch runs fetch against a server that serves a
|
||||
@@ -922,9 +992,10 @@ func TestFetchFailsOnHashMismatch(t *testing.T) {
|
||||
// TestFetchIntoPartlyFilledDestination runs fetch where an interrupted
|
||||
// fetch of an older version of the tree left one file current, one file
|
||||
// out of date and one half written to its temp file, beside a file the
|
||||
// manifest does not list. fetch downloads every file the manifest lists,
|
||||
// those already present included, and replaces what is there; the file
|
||||
// the manifest does not list is left alone.
|
||||
// manifest does not list. fetch skips the current file and downloads the
|
||||
// other two. The out-of-date file has the same size as the new version,
|
||||
// so only its hash shows that it must be replaced. The file the manifest
|
||||
// does not list is left alone, and the manifest is saved beside the files.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestFetchIntoPartlyFilledDestination(t *testing.T) {
|
||||
@@ -935,7 +1006,8 @@ func TestFetchIntoPartlyFilledDestination(t *testing.T) {
|
||||
}
|
||||
unlisted := []byte("not in the manifest")
|
||||
|
||||
tree := fetchTestHandler(manifestOf(t, files), files)
|
||||
manifest := manifestOf(t, files)
|
||||
tree := fetchTestHandler(manifest, files)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
@@ -963,21 +1035,274 @@ func TestFetchIntoPartlyFilledDestination(t *testing.T) {
|
||||
os.WriteFile(tempPathFor("sub/partial.txt"), []byte("cut off"), 0o600))
|
||||
require.NoError(t, os.WriteFile("unlisted.txt", unlisted, 0o600))
|
||||
|
||||
opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
|
||||
opts := testOpts([]string{testApp, cmdFetch, server.URL}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
assert.Contains(t, testStderr(t, opts), "skipping current.txt: already present")
|
||||
|
||||
want := maps.Clone(files)
|
||||
want["unlisted.txt"] = unlisted
|
||||
want[defaultManifestName] = manifest
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
assert.ElementsMatch(t, []string{
|
||||
"/" + defaultManifestName, "/current.txt", "/sub/changed.txt", "/sub/partial.txt",
|
||||
"/" + defaultManifestName, "/sub/changed.txt", "/sub/partial.txt",
|
||||
}, requested)
|
||||
}
|
||||
|
||||
// TestFetchIntoDest fetches a tree with --dest into a directory that does
|
||||
// not exist yet. The files and the manifest must land there and nowhere
|
||||
// else, and check must pass on the result with no extra files. A second
|
||||
// fetch into the same directory must download nothing but the manifest.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestFetchIntoDest(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
"top.txt": []byte("at the top"),
|
||||
"sub/one.txt": []byte("one level down"),
|
||||
}
|
||||
|
||||
manifest := manifestOf(t, files)
|
||||
tree := fetchTestHandler(manifest, files)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
requested []string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
|
||||
requested = append(requested, r.URL.Path)
|
||||
|
||||
mu.Unlock()
|
||||
|
||||
tree.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cwd := chdirTemp(t)
|
||||
dest := filepath.Join(t.TempDir(), "mirror")
|
||||
fetch := []string{testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL}
|
||||
|
||||
opts := testOpts(fetch, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
want := maps.Clone(files)
|
||||
want[defaultManifestName] = manifest
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
assert.Empty(t, filesUnder(t, cwd), "fetch wrote outside --dest")
|
||||
|
||||
check := testOpts([]string{
|
||||
testApp, cmdCheck, "-q", testFlagBase, dest, testFlagNoExtra,
|
||||
filepath.Join(dest, defaultManifestName),
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(check), testStderr(t, check))
|
||||
|
||||
mu.Lock()
|
||||
requested = nil
|
||||
mu.Unlock()
|
||||
|
||||
opts = testOpts(fetch, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
assert.Equal(t, []string{"/" + defaultManifestName}, requested)
|
||||
}
|
||||
|
||||
// TestFetchRequireSignature runs fetch with --require-signature. A
|
||||
// manifest that is unsigned, or signed by another key, must stop fetch
|
||||
// with check's message before it downloads or writes anything; the
|
||||
// required key lets it through. The signed cases need gpg and are skipped
|
||||
// without it, as the other signing tests are.
|
||||
//
|
||||
//nolint:paralleltest // signedManifest calls t.Setenv, which bars t.Parallel
|
||||
func TestFetchRequireSignature(t *testing.T) {
|
||||
files := map[string][]byte{testFileTxt: []byte("signed file")}
|
||||
|
||||
t.Run("unsigned", func(t *testing.T) {
|
||||
assertFetchRefused(t, manifestOf(t, files), files,
|
||||
"manifest is not signed, but signature from "+msgFpA+" is required",
|
||||
"--"+flagRequireSignature, msgFpA)
|
||||
})
|
||||
|
||||
t.Run("signed", func(t *testing.T) {
|
||||
manifest := signedManifest(t, files)
|
||||
|
||||
signer, err := signedChecker(t, manifest).
|
||||
ExtractEmbeddedSigningKeyFP(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
assertFetchRefused(t, manifest, files,
|
||||
"embedded signing key fingerprint "+signer+" does not match required "+msgFpB,
|
||||
"--"+flagRequireSignature, msgFpB)
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifest, files))
|
||||
defer server.Close()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFetch, "-q", "--" + flagDest, dest,
|
||||
"--" + flagRequireSignature, signer, server.URL,
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
assert.Equal(t, files[testFileTxt], filesUnder(t, dest)[testFileTxt])
|
||||
})
|
||||
}
|
||||
|
||||
// TestFetchRefusesListedManifestName fetches manifests that list, at the
|
||||
// top of the tree, the name fetch saves the manifest under or that name's
|
||||
// temp file: as a file, as a directory, in capitals, and with a leading
|
||||
// "./". Saving the manifest would replace or remove what is listed there,
|
||||
// so fetch must refuse the manifest before it creates the destination or
|
||||
// requests any file.
|
||||
func TestFetchRefusesListedManifestName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, listed := range []string{
|
||||
defaultManifestName,
|
||||
tempPathFor(defaultManifestName),
|
||||
defaultManifestName + "/" + testFileTxt,
|
||||
"INDEX.MF",
|
||||
"./" + defaultManifestName,
|
||||
} {
|
||||
t.Run(listed, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{listed: []byte("listed")}
|
||||
|
||||
assertFetchRefused(t, builtManifest(t, files), files,
|
||||
"manifest lists a file where fetch writes another file: "+listed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchRefusesListedTempName fetches manifests that list a.txt and
|
||||
// .a.txt.tmp, the temp file fetch downloads a.txt to, at the top of the
|
||||
// tree and in a directory. Downloading a.txt would remove .a.txt.tmp, so
|
||||
// fetch must refuse the manifest before it creates the destination or
|
||||
// requests any file. README with .README.tmp checks that temp names are
|
||||
// compared ignoring case. A manifest that lists only one of the two is
|
||||
// fetched in full.
|
||||
func TestFetchRefusesListedTempName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, dir := range []string{"", "sub/"} {
|
||||
for file, tmp := range map[string]string{
|
||||
dir + "a.txt": dir + ".a.txt.tmp",
|
||||
dir + "README": dir + ".README.tmp",
|
||||
} {
|
||||
both := map[string][]byte{
|
||||
file: []byte("a file"),
|
||||
tmp: []byte("a file at its temp name"),
|
||||
}
|
||||
|
||||
t.Run(file+" and "+tmp, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assertFetchRefused(t, builtManifest(t, both), both,
|
||||
"manifest lists a file where fetch writes another file: "+
|
||||
tmp+" (the temp file for "+file+")")
|
||||
})
|
||||
|
||||
for listed, content := range both {
|
||||
t.Run("only "+listed, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
files := map[string][]byte{listed: content}
|
||||
manifest := builtManifest(t, files)
|
||||
|
||||
server := httptest.NewServer(fetchTestHandler(manifest, files))
|
||||
defer server.Close()
|
||||
|
||||
dest := t.TempDir()
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL,
|
||||
}, afero.NewOsFs())
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
want := maps.Clone(files)
|
||||
want[defaultManifestName] = manifest
|
||||
assert.Equal(t, want, filesUnder(t, dest))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// builtManifest returns a manifest of files, built directly rather than
|
||||
// scanned, since a scan lists no hidden files and never a path starting
|
||||
// with "./".
|
||||
func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
builder := mfer.NewBuilder()
|
||||
|
||||
for p, content := range files {
|
||||
_, err := builder.AddFile(mfer.RelFilePath(p), mfer.FileSize(len(content)),
|
||||
mfer.ModTime(time.Now()), bytes.NewReader(content), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var manifest bytes.Buffer
|
||||
|
||||
require.NoError(t, builder.Build(context.Background(), &manifest))
|
||||
|
||||
return manifest.Bytes()
|
||||
}
|
||||
|
||||
// assertFetchRefused serves manifest, a manifest of files, and fetches it
|
||||
// with flags into a directory that does not exist yet. fetch must fail
|
||||
// with message after requesting only the manifest, and must not create
|
||||
// the directory.
|
||||
func assertFetchRefused(
|
||||
t *testing.T, manifest []byte, files map[string][]byte,
|
||||
message string, flags ...string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
tree := fetchTestHandler(manifest, files)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
requested []string
|
||||
)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
|
||||
requested = append(requested, r.URL.Path)
|
||||
|
||||
mu.Unlock()
|
||||
|
||||
tree.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "mirror")
|
||||
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdFetch, "-q", "--" + flagDest, dest}, flags, []string{server.URL},
|
||||
), afero.NewOsFs())
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), message)
|
||||
assert.NoDirExists(t, dest, "fetch created the destination before refusing")
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
assert.Equal(t, []string{"/" + defaultManifestName}, requested)
|
||||
}
|
||||
|
||||
// TestFetchTimeoutFlag runs fetch with --timeout against a server that
|
||||
// never answers. Without the flag's limit the request would wait forever;
|
||||
// once fetch gives up on it, the server cancels fetch's context so that
|
||||
|
||||
+24
-9
@@ -22,8 +22,10 @@ const (
|
||||
cmdFetch = "fetch"
|
||||
cmdVersion = "version"
|
||||
|
||||
flagProgress = "progress"
|
||||
flagTimeout = "timeout"
|
||||
flagProgress = "progress"
|
||||
flagTimeout = "timeout"
|
||||
flagDest = "dest"
|
||||
flagRequireSignature = "require-signature"
|
||||
|
||||
manifestArgsUsage = "[manifest file]"
|
||||
|
||||
@@ -142,6 +144,17 @@ func commonFlags() []cli.Flag {
|
||||
}
|
||||
}
|
||||
|
||||
// requireSignatureFlag returns the --require-signature flag taken by the
|
||||
// check and fetch subcommands.
|
||||
func requireSignatureFlag() *cli.StringFlag {
|
||||
return &cli.StringFlag{
|
||||
Name: flagRequireSignature,
|
||||
Aliases: []string{"S"},
|
||||
Usage: "Require manifest to be signed by the specified GPG key ID",
|
||||
EnvVars: []string{"MFER_REQUIRE_SIGNATURE"},
|
||||
}
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdGenerate,
|
||||
@@ -227,14 +240,9 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "no-extra-files",
|
||||
Usage: "Fail if files exist in base directory that are not in manifest",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "require-signature",
|
||||
Aliases: []string{"S"},
|
||||
Usage: "Require manifest to be signed by the specified GPG key ID",
|
||||
EnvVars: []string{"MFER_REQUIRE_SIGNATURE"},
|
||||
Usage: "Fail, instead of warning, if files in base directory are not in manifest",
|
||||
},
|
||||
requireSignatureFlag(),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -354,6 +362,13 @@ func (mfa *CLIApp) fetchCommand() *cli.Command {
|
||||
Usage: "Time limit for each HTTP request, including the download " +
|
||||
"of its body",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: flagDest,
|
||||
Aliases: []string{"d"},
|
||||
Value: ".",
|
||||
Usage: "Directory to download the files and the manifest into",
|
||||
},
|
||||
requireSignatureFlag(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
+50
-35
@@ -77,9 +77,9 @@ type Checker struct {
|
||||
fs afero.Fs
|
||||
// manifestPaths is a set of paths in the manifest for quick lookup
|
||||
manifestPaths map[RelFilePath]struct{}
|
||||
// manifestRelPath is the relative path of the manifest file from
|
||||
// basePath (for exclusion)
|
||||
manifestRelPath RelFilePath
|
||||
// manifestInfo is the manifest file, which FindExtraFiles leaves out,
|
||||
// matched with os.SameFile.
|
||||
manifestInfo os.FileInfo
|
||||
// signature info from the manifest
|
||||
signature []byte
|
||||
signer []byte
|
||||
@@ -133,26 +133,20 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
|
||||
manifestPaths[RelFilePath(f.GetPath())] = struct{}{}
|
||||
}
|
||||
|
||||
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
|
||||
absManifest, err := filepath.Abs(opts.ManifestPath)
|
||||
manifestInfo, err := fs.Stat(opts.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
manifestRel, err := filepath.Rel(abs, absManifest)
|
||||
if err != nil {
|
||||
manifestRel = ""
|
||||
}
|
||||
|
||||
return &Checker{
|
||||
basePath: AbsFilePath(abs),
|
||||
files: files,
|
||||
fs: fs,
|
||||
manifestPaths: manifestPaths,
|
||||
manifestRelPath: RelFilePath(manifestRel),
|
||||
signature: m.pbOuter.GetSignature(),
|
||||
signer: m.pbOuter.GetSigner(),
|
||||
signingPubKey: m.pbOuter.GetSigningPubKey(),
|
||||
basePath: AbsFilePath(abs),
|
||||
files: files,
|
||||
fs: fs,
|
||||
manifestPaths: manifestPaths,
|
||||
manifestInfo: manifestInfo,
|
||||
signature: m.pbOuter.GetSignature(),
|
||||
signer: m.pbOuter.GetSigner(),
|
||||
signingPubKey: m.pbOuter.GetSigningPubKey(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -273,20 +267,27 @@ func (c *Checker) Check(
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindExtraFiles walks the filesystem and reports files not in the manifest.
|
||||
// Results are sent to the results channel. The channel is closed when done.
|
||||
// Hidden files/directories (starting with .) are skipped, as they are excluded
|
||||
// from manifests by default. The manifest file itself is also skipped.
|
||||
// FindExtraFiles walks the filesystem and reports files not in the manifest,
|
||||
// hidden files and directories included. The manifest file itself is not
|
||||
// reported. Anything the search cannot read, such as a directory that cannot
|
||||
// be listed, is reported with StatusError and the search goes on. Results are
|
||||
// sent to the results channel. The channel is closed when done.
|
||||
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
|
||||
if results != nil {
|
||||
defer close(results)
|
||||
}
|
||||
|
||||
walkFn := func(walkPath string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The search does not follow symlinks, so a base directory named
|
||||
// through one is resolved first. If that fails, the base is searched as
|
||||
// named and the search reports the problem.
|
||||
root := string(c.basePath)
|
||||
|
||||
resolved, err := filepath.EvalSymlinks(root)
|
||||
if err == nil {
|
||||
root = resolved
|
||||
}
|
||||
|
||||
walkFn := func(walkPath string, info os.FileInfo, walkErr error) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
@@ -294,15 +295,22 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
||||
}
|
||||
|
||||
// Get relative path
|
||||
rel, err := filepath.Rel(string(c.basePath), walkPath)
|
||||
rel, err := filepath.Rel(root, walkPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip hidden files and directories (dotfiles)
|
||||
if IsHiddenPath(filepath.ToSlash(rel)) {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
relPath := RelFilePath(rel)
|
||||
|
||||
// Report what cannot be read, such as a directory that cannot be
|
||||
// listed, and go on with the rest.
|
||||
if walkErr != nil {
|
||||
if results != nil {
|
||||
results <- Result{
|
||||
Path: relPath,
|
||||
Status: StatusError,
|
||||
Message: walkErr.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -313,10 +321,17 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath := RelFilePath(rel)
|
||||
// A symlink is compared by what it points to, so a manifest reached
|
||||
// through one is not reported either.
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
target, statErr := c.fs.Stat(walkPath)
|
||||
if statErr == nil {
|
||||
info = target
|
||||
}
|
||||
}
|
||||
|
||||
// Skip the manifest file itself
|
||||
if relPath == c.manifestRelPath {
|
||||
// Skip the manifest file itself, however its path is spelled
|
||||
if os.SameFile(info, c.manifestInfo) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -334,7 +349,7 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
||||
return nil
|
||||
}
|
||||
|
||||
return afero.Walk(c.fs, string(c.basePath), walkFn)
|
||||
return afero.Walk(c.fs, root, walkFn)
|
||||
}
|
||||
|
||||
func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
|
||||
|
||||
+94
-97
@@ -21,8 +21,6 @@ const (
|
||||
testExistsFile = "exists.txt"
|
||||
testManifestPath = "/manifest.mf"
|
||||
testDataDir = "/data"
|
||||
// testDataManifestPath is a manifest kept inside the checked tree.
|
||||
testDataManifestPath = testDataDir + "/index.mf"
|
||||
)
|
||||
|
||||
func TestStatusString(t *testing.T) {
|
||||
@@ -459,50 +457,120 @@ func TestFindExtraFiles(t *testing.T) {
|
||||
assert.Equal(t, "not in manifest", extras[0].Message)
|
||||
}
|
||||
|
||||
func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
||||
// TestFindExtraFilesReportsHiddenFilesButNotManifest keeps the manifest
|
||||
// inside the checked tree: hidden files and directories are reported, the
|
||||
// manifest is not. The manifest is recognized by file identity, which needs
|
||||
// the real filesystem.
|
||||
func TestFindExtraFilesReportsHiddenFilesButNotManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
manifestFiles := map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
}
|
||||
createTestManifest(t, fs, testDataManifestPath, manifestFiles)
|
||||
createFilesOnDisk(t, fs, map[string][]byte{
|
||||
dir := t.TempDir()
|
||||
manifestPath := filepath.Join(dir, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
createTestManifest(t, fs, manifestPath, map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
})
|
||||
// Create dotfile and manifest that should be skipped
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
|
||||
// Create a real extra file
|
||||
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
||||
|
||||
unlisted := []RelFilePath{"extra.txt", ".hidden", ".git/config"}
|
||||
for _, p := range append([]RelFilePath{testFile1}, unlisted...) {
|
||||
path := filepath.Join(dir, string(p))
|
||||
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o750))
|
||||
require.NoError(t, afero.WriteFile(fs, path, []byte("x"), 0o600))
|
||||
}
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testDataManifestPath,
|
||||
BasePath: testDataDir,
|
||||
ManifestPath: manifestPath,
|
||||
BasePath: dir,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
err = chk.FindExtraFiles(context.Background(), results)
|
||||
require.NoError(t, chk.FindExtraFiles(context.Background(), results))
|
||||
|
||||
var extras []RelFilePath
|
||||
for r := range results {
|
||||
extras = append(extras, r.Path)
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, unlisted, extras)
|
||||
}
|
||||
|
||||
// TestFindExtraFilesSkipsManifestReachedThroughSymlink checks a tree whose
|
||||
// index.mf is a symlink to the manifest kept outside the tree: the symlink is
|
||||
// not reported.
|
||||
func TestFindExtraFilesSkipsManifestReachedThroughSymlink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
tree := filepath.Join(dir, "tree")
|
||||
manifestPath := filepath.Join(dir, "real.mf")
|
||||
linkPath := filepath.Join(tree, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
createTestManifest(t, fs, manifestPath, map[string][]byte{testFile1: []byte("x")})
|
||||
require.NoError(t, fs.MkdirAll(tree, 0o750))
|
||||
require.NoError(t,
|
||||
afero.WriteFile(fs, filepath.Join(tree, testFile1), []byte("x"), 0o600))
|
||||
require.NoError(t, os.Symlink(manifestPath, linkPath))
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: linkPath,
|
||||
BasePath: tree,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
var extras []Result
|
||||
results := make(chan Result, 10)
|
||||
require.NoError(t, chk.FindExtraFiles(context.Background(), results))
|
||||
|
||||
var extras []RelFilePath
|
||||
for r := range results {
|
||||
extras = append(extras, r)
|
||||
extras = append(extras, r.Path)
|
||||
}
|
||||
|
||||
// Should only report extra.txt, not .hidden, .config/settings, or index.mf
|
||||
for _, e := range extras {
|
||||
t.Logf("extra: %s", e.Path)
|
||||
assert.Empty(t, extras)
|
||||
}
|
||||
|
||||
// TestFindExtraFilesSearchesBaseNamedThroughSymlink names the checked tree
|
||||
// through a symlink to it: the files in the tree are searched, and the
|
||||
// symlink itself is not reported.
|
||||
func TestFindExtraFilesSearchesBaseNamedThroughSymlink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
tree := filepath.Join(dir, "tree")
|
||||
link := filepath.Join(dir, "link")
|
||||
manifestPath := filepath.Join(dir, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
createTestManifest(t, fs, manifestPath, map[string][]byte{testFile1: []byte("x")})
|
||||
require.NoError(t, fs.MkdirAll(tree, 0o750))
|
||||
|
||||
for _, name := range []string{testFile1, testFile2} {
|
||||
require.NoError(t,
|
||||
afero.WriteFile(fs, filepath.Join(tree, name), []byte("x"), 0o600))
|
||||
}
|
||||
|
||||
assert.Len(t, extras, 1)
|
||||
require.NoError(t, os.Symlink(tree, link))
|
||||
|
||||
if len(extras) > 0 {
|
||||
assert.Equal(t, RelFilePath("extra.txt"), extras[0].Path)
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: manifestPath,
|
||||
BasePath: link,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
require.NoError(t, chk.FindExtraFiles(context.Background(), results))
|
||||
|
||||
var extras []RelFilePath
|
||||
for r := range results {
|
||||
extras = append(extras, r.Path)
|
||||
}
|
||||
|
||||
assert.Equal(t, []RelFilePath{testFile2}, extras)
|
||||
}
|
||||
|
||||
func TestFindExtraFilesContextCancellation(t *testing.T) {
|
||||
@@ -649,77 +717,6 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
|
||||
assert.Equal(t, 0, statusCounts[StatusError], "no files should be ERROR")
|
||||
}
|
||||
|
||||
func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Regression test for #16: FindExtraFiles should not report dotfiles
|
||||
// or the manifest file itself as extra files.
|
||||
fs := afero.NewMemMapFs()
|
||||
files := map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
}
|
||||
createTestManifest(t, fs, testDataManifestPath, files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
// Add dotfiles and manifest file on disk
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("dotfile"), 0o644))
|
||||
require.NoError(t, fs.MkdirAll("/data/.git", 0o755))
|
||||
require.NoError(t,
|
||||
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testDataManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
err = chk.FindExtraFiles(context.Background(), results)
|
||||
require.NoError(t, err)
|
||||
|
||||
var extras []Result
|
||||
for r := range results {
|
||||
extras = append(extras, r)
|
||||
}
|
||||
|
||||
// Should report NO extra files — dotfiles and manifest should be skipped
|
||||
assert.Empty(t, extras,
|
||||
"FindExtraFiles should not report dotfiles or manifest file as extra; got: %v",
|
||||
extras)
|
||||
}
|
||||
|
||||
func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The manifest file itself should never be reported as extra
|
||||
fs := afero.NewMemMapFs()
|
||||
files := map[string][]byte{
|
||||
testFile1: []byte("content"),
|
||||
}
|
||||
createTestManifest(t, fs, testDataManifestPath, files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testDataManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
err = chk.FindExtraFiles(context.Background(), results)
|
||||
require.NoError(t, err)
|
||||
|
||||
var extras []Result
|
||||
for r := range results {
|
||||
extras = append(extras, r)
|
||||
}
|
||||
|
||||
assert.Empty(t, extras,
|
||||
"manifest file should not be reported as extra; got: %v", extras)
|
||||
}
|
||||
|
||||
func TestCheckEmptyManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
package mfer
|
||||
|
||||
//go:generate protoc ./mf.proto --go_out=paths=source_relative:.
|
||||
@@ -1,6 +1,5 @@
|
||||
{
|
||||
"name": "mfer",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
|
||||
"license": "WTFPL",
|
||||
|
||||
+59
-5
@@ -13,11 +13,15 @@ set -eu
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
|
||||
NODE_VERSION="22.17.0"
|
||||
# The node version is in .nvmrc, where script/prettier reads it too.
|
||||
NODE_VERSION="$(cat "$ROOT/.nvmrc")"
|
||||
NVM_VERSION="0.40.3"
|
||||
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
||||
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
||||
YARN_VERSION="1.22.22"
|
||||
# protoc v33.4, 2026-10-04, for script/generate. The sha256 of each
|
||||
# platform's release archive is in ensure_protoc.
|
||||
PROTOC_VERSION="33.4"
|
||||
|
||||
PKGMGR=""
|
||||
SUDO=""
|
||||
@@ -74,9 +78,11 @@ verify_sha256() {
|
||||
fi
|
||||
}
|
||||
|
||||
# nvm is a bash script; run a command in a bash with nvm loaded
|
||||
# nvm is a bash script; run a command in a bash with nvm loaded.
|
||||
# --no-use: otherwise loading nvm here switches to the version .nvmrc
|
||||
# names, and fails silently while that version is not installed yet.
|
||||
nvm_sh() {
|
||||
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
|
||||
bash -c ". \"\$HOME/.nvm/nvm.sh\" --no-use && $*"
|
||||
}
|
||||
|
||||
ensure_nvm() {
|
||||
@@ -122,6 +128,48 @@ install_js_deps() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Unpack protoc's release archive for this platform into bin/protoc, after
|
||||
# checking the archive's sha256, unless bin/protoc already holds the pinned
|
||||
# version.
|
||||
ensure_protoc() {
|
||||
dir="$ROOT/bin/protoc"
|
||||
if [ "$("$dir/bin/protoc" --version 2>/dev/null)" = \
|
||||
"libprotoc $PROTOC_VERSION" ]; then
|
||||
return 0
|
||||
fi
|
||||
case "$(uname -s) $(uname -m)" in
|
||||
"Linux x86_64")
|
||||
platform="linux-x86_64"
|
||||
sha256="c0040ea9aef08fdeb2c74ca609b18d5fdbfc44ea0042fcfbfb38860d35f7dd66"
|
||||
;;
|
||||
"Linux aarch64" | "Linux arm64")
|
||||
platform="linux-aarch_64"
|
||||
sha256="15aa988f4a6090636525ec236a8e4b3aab41eef402751bd5bb2df6afd9b7b5a5"
|
||||
;;
|
||||
"Darwin x86_64")
|
||||
platform="osx-x86_64"
|
||||
sha256="a49bec10d039e902d3b43e49938c42526f90011467609864fa6386ac4014da58"
|
||||
;;
|
||||
"Darwin arm64")
|
||||
platform="osx-aarch_64"
|
||||
sha256="726297dcfed58592fd35620a5a6246ae020c39e88f3fd4cb1827df7bcf3dfcf1"
|
||||
;;
|
||||
*)
|
||||
echo "bootstrap: no protoc archive pinned for $(uname -s) $(uname -m)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if missing curl; then pkg_install curl curl curl curl; fi
|
||||
if missing unzip; then pkg_install unzip unzip unzip unzip; fi
|
||||
tmp="$(mktemp -d)"
|
||||
curl -fsSL -o "$tmp/protoc.zip" \
|
||||
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip"
|
||||
verify_sha256 "$tmp/protoc.zip" "$sha256"
|
||||
rm -rf "$dir"
|
||||
unzip -q "$tmp/protoc.zip" -d "$dir"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
@@ -132,8 +180,10 @@ main() {
|
||||
# ---- JS / docs repos ----
|
||||
# This is a Go repo, but node and yarn are required anyway: prettier
|
||||
# formats the Markdown and JSON, and script/fmt-check verifies it.
|
||||
# The version is pinned by package.json/yarn.lock, whose integrity
|
||||
# hashes --frozen-lockfile enforces.
|
||||
# The version is pinned by package.json/yarn.lock: yarn checks every
|
||||
# package it fetches against its yarn.lock integrity hash, and
|
||||
# --frozen-lockfile fails instead of rewriting a yarn.lock that no
|
||||
# longer matches package.json.
|
||||
ensure_node
|
||||
ensure_yarn
|
||||
install_js_deps
|
||||
@@ -142,6 +192,10 @@ main() {
|
||||
if missing go; then pkg_install go golang go go; fi
|
||||
# No golangci-lint: script/lint runs it in Docker only.
|
||||
go mod download
|
||||
# gofumpt and protoc-gen-go: bin/tools/go.mod pins them, and
|
||||
# script/gofumpt and script/generate build them from there.
|
||||
(cd "$ROOT/bin/tools" && go mod download)
|
||||
ensure_protoc
|
||||
|
||||
# ---- Python repos ----
|
||||
# if missing python3; then pkg_install python3 python3 python3 python3; fi
|
||||
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
# script/build: build the mfer binary into bin/mfer, stamped with the
|
||||
# revision `mfer version` prints, derived as script/docker derives it.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
go build -tags urfave_cli_no_docs \
|
||||
-ldflags "-X main.Gitrev=$version" \
|
||||
-o bin/mfer ./cmd/mfer
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+3
-2
@@ -7,8 +7,9 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
gofumpt -l -w mfer internal cmd
|
||||
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
||||
# Go, then Markdown and JSON, through the same scripts script/fmt-check
|
||||
# uses, so both see the same files and the same tool versions.
|
||||
"$SCRIPT_DIR/gofumpt" --write
|
||||
"$SCRIPT_DIR/prettier" --write
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
#!/bin/sh
|
||||
# script/fmt-check: check formatting (read-only). Same scope as
|
||||
# script/fmt, but fails instead of writing: Go via script/fmt-check-go,
|
||||
# script/fmt, but fails instead of writing: Go via script/gofumpt,
|
||||
# Markdown and JSON via script/prettier.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
main() {
|
||||
"$SCRIPT_DIR/fmt-check-go"
|
||||
"$SCRIPT_DIR/gofumpt" --check
|
||||
"$SCRIPT_DIR/prettier" --check
|
||||
}
|
||||
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/fmt-check-go: check Go formatting (read-only). Split out from
|
||||
# script/fmt-check so the Docker lint stage, whose image has no node and
|
||||
# therefore no prettier, can run the Go half on its own.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
if [ -n "$(gofmt -l .)" ]; then
|
||||
echo "gofmt: files need formatting:" >&2
|
||||
gofmt -l . >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+20
-18
@@ -4,26 +4,17 @@
|
||||
# regenerates mf.pb.go: it is committed, so building and checking need no
|
||||
# protoc. A test fails while mf.proto no longer matches the recorded hash.
|
||||
#
|
||||
# Needs exactly the protoc and protoc-gen-go versions named in the header of
|
||||
# the committed mf.pb.go (README.md says how to install them). Another
|
||||
# version writes a different mf.pb.go, so the script refuses to run.
|
||||
# Runs the protoc that script/bootstrap unpacks into bin/protoc, and the
|
||||
# protoc-gen-go that bin/tools/go.mod pins. Another version of either
|
||||
# writes a different mf.pb.go.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
|
||||
# The protoc version script/bootstrap installs. protoc 33.4 names itself
|
||||
# v6.33.4 in the mf.pb.go header.
|
||||
PROTOC_VERSION="33.4"
|
||||
PROTOC_GEN_GO_VERSION="v1.36.11"
|
||||
|
||||
# require_version <command> <its exact --version output>
|
||||
require_version() {
|
||||
actual="$("$1" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "$2" ]; then
|
||||
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
|
||||
echo " README.md says how to install it." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
PROTOC="$ROOT/bin/protoc/bin/protoc"
|
||||
|
||||
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
|
||||
# where there is no sha256sum.
|
||||
@@ -39,13 +30,24 @@ sha256() {
|
||||
}
|
||||
|
||||
main() {
|
||||
# A bin/protoc left from before the pin moved fails here, until
|
||||
# script/bootstrap replaces it.
|
||||
actual="$("$PROTOC" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "libprotoc $PROTOC_VERSION" ]; then
|
||||
echo "generate: needs protoc $PROTOC_VERSION in bin/protoc," \
|
||||
"found: ${actual:-none}; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
# `go tool -n` builds protoc-gen-go from bin/tools and prints where the
|
||||
# binary is, without running it.
|
||||
plugin="$(cd "$ROOT/bin/tools" && go tool -n protoc-gen-go)"
|
||||
|
||||
cd "$ROOT/mfer"
|
||||
require_version protoc "libprotoc $PROTOC_VERSION"
|
||||
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
|
||||
# Hashed before regenerating, so a missing hash tool stops the script
|
||||
# before it changes anything. Regenerating leaves mf.proto as it is.
|
||||
proto_hash="$(sha256 mf.proto)"
|
||||
go generate .
|
||||
"$PROTOC" --plugin=protoc-gen-go="$plugin" \
|
||||
--go_out=paths=source_relative:. ./mf.proto
|
||||
echo "$proto_hash" >mf.proto.sha256
|
||||
}
|
||||
|
||||
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
# script/gofumpt: run gofumpt over this repo's Go files.
|
||||
#
|
||||
# Takes exactly one mode argument, --write or --check, and runs the same
|
||||
# gofumpt version over the same files in both modes. script/fmt and
|
||||
# script/fmt-check both go through here, and so does the Docker lint
|
||||
# stage, so what gets formatted and what gets verified cannot drift
|
||||
# apart.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# The gofumpt version is the one bin/tools/go.mod pins. `go tool` run in
|
||||
# bin/tools builds it from source checked against the hashes in
|
||||
# bin/tools/go.sum, so neither a developer machine nor the lint image needs
|
||||
# it installed.
|
||||
|
||||
usage() {
|
||||
echo "usage: script/gofumpt --write|--check" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
main() {
|
||||
[ "$#" -eq 1 ] || usage
|
||||
cd "$ROOT/bin/tools"
|
||||
# Every Go file in the repo, from $ROOT down. gofumpt holds generated
|
||||
# files, such as mfer/mf.pb.go, to gofmt's rules only.
|
||||
case "$1" in
|
||||
--write) go tool gofumpt -l -w "$ROOT" ;;
|
||||
--check)
|
||||
# Own line: a failing command inside `[ -n "$(...)" ]` does
|
||||
# not trip `set -e`, so a gofumpt that never ran would pass.
|
||||
unformatted="$(go tool gofumpt -l "$ROOT")"
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "gofumpt: files need formatting (run make fmt):" >&2
|
||||
echo "$unformatted" >&2
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+24
-24
@@ -18,24 +18,9 @@ usage() {
|
||||
exit 2
|
||||
}
|
||||
|
||||
# Prefer the version pinned by package.json/yarn.lock so that CI and
|
||||
# developer machines format identically. Fall back to a prettier on PATH,
|
||||
# but say so, because a different version formats differently.
|
||||
find_prettier() {
|
||||
if [ -x "$ROOT/node_modules/.bin/prettier" ]; then
|
||||
printf '%s\n' "$ROOT/node_modules/.bin/prettier"
|
||||
return 0
|
||||
fi
|
||||
if command -v prettier >/dev/null 2>&1; then
|
||||
echo "prettier: node_modules/.bin/prettier is absent; using the" \
|
||||
"prettier on PATH, which may be a different version than the" \
|
||||
"one pinned in package.json. Run script/bootstrap to install" \
|
||||
"the pinned version." >&2
|
||||
command -v prettier
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
# Only the prettier yarn installed from yarn.lock, never one on PATH: a
|
||||
# different version formats differently.
|
||||
PRETTIER="$ROOT/node_modules/.bin/prettier"
|
||||
|
||||
main() {
|
||||
[ "$#" -eq 1 ] || usage
|
||||
@@ -46,10 +31,26 @@ main() {
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
if ! prettier_bin="$(find_prettier)"; then
|
||||
echo "prettier: not found." >&2
|
||||
echo " Install it with: script/bootstrap" >&2
|
||||
echo " (installs the version pinned in package.json/yarn.lock)" >&2
|
||||
# Where there is no node on PATH, script/bootstrap installs the version
|
||||
# .nvmrc names through nvm, which keeps it in this directory.
|
||||
if ! command -v node >/dev/null 2>&1; then
|
||||
PATH="$HOME/.nvm/versions/node/v$(cat .nvmrc)/bin:$PATH"
|
||||
fi
|
||||
if ! command -v node >/dev/null 2>&1; then
|
||||
echo "prettier: node is missing; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# node_modules keeps the old prettier after package.json moves to a new
|
||||
# one, until script/bootstrap runs again, so compare the two.
|
||||
if ! installed="$("$PRETTIER" --version 2>/dev/null)"; then
|
||||
echo "prettier: not installed; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
pinned="$(node -p 'require("./package.json").devDependencies.prettier')"
|
||||
if [ "$installed" != "$pinned" ]; then
|
||||
echo "prettier: package.json pins $pinned but $installed is" \
|
||||
"installed; run script/bootstrap" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -62,8 +63,7 @@ main() {
|
||||
# patterns always match at least one tracked file (README.md,
|
||||
# package.json), so an empty match means the glob broke, and prettier
|
||||
# erroring out is exactly what we want rather than a vacuous pass.
|
||||
"$prettier_bin" "$mode" "**/*.md"
|
||||
"$prettier_bin" "$mode" "**/*.json"
|
||||
"$PRETTIER" "$mode" "**/*.md" "**/*.json"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user