Compare commits
1
Commits
next
..
b9ef06f23d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9ef06f23d |
+1
-1
@@ -8,7 +8,7 @@ vendor.tzst
|
|||||||
modcache.tzst
|
modcache.tzst
|
||||||
|
|
||||||
# Generated manifest files
|
# Generated manifest files
|
||||||
/index.mf
|
.index.mf
|
||||||
|
|
||||||
# Secrets
|
# Secrets
|
||||||
.env
|
.env
|
||||||
|
|||||||
@@ -27,8 +27,5 @@ source for coding standards, formatting, linting, and workflow rules.
|
|||||||
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
||||||
committed (required for `go get` compatibility).
|
committed (required for `go get` compatibility).
|
||||||
- The format specification is in `FORMAT.md`.
|
- The format specification is in `FORMAT.md`.
|
||||||
- Open work, open design questions included, is tracked only in the repo's
|
- See the TODO section in `README.md` for the 1.0 implementation plan and open
|
||||||
issues: https://git.eeqj.de/sneak/mfer/issues. There is no `TODO.md` and no
|
design questions.
|
||||||
TODO list in `README.md`; do not add either. For this repo this overrides the
|
|
||||||
`REPO_POLICIES.md` rule to put the todo list in the README, per sneak's
|
|
||||||
ruling: https://git.eeqj.de/sneak/mfer/issues/76#issuecomment-118130.
|
|
||||||
|
|||||||
+8
-9
@@ -8,12 +8,13 @@ RUN go mod download
|
|||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
||||||
|
RUN touch mfer/mf.pb.go
|
||||||
|
|
||||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||||
# markdown half runs in the mdfmt stage below.
|
# markdown half runs in the mdfmt stage below.
|
||||||
RUN make fmt-check-go
|
RUN make fmt-check-go
|
||||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
RUN make lint
|
||||||
# there is no docker inside this build.
|
|
||||||
RUN golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
|
||||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||||
@@ -43,6 +44,9 @@ RUN go mod download
|
|||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
||||||
|
RUN touch mfer/mf.pb.go
|
||||||
|
|
||||||
RUN make test
|
RUN make test
|
||||||
|
|
||||||
# A build context sent as a tar archive, as upaas sends it, keeps its files'
|
# A build context sent as a tar archive, as upaas sends it, keeps its files'
|
||||||
@@ -63,13 +67,8 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
|||||||
exit 1; \
|
exit 1; \
|
||||||
fi; \
|
fi; \
|
||||||
cd cmd/mfer && \
|
cd cmd/mfer && \
|
||||||
CGO_ENABLED=0 go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
|
go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
|
||||||
|
|
||||||
# Fail unless /mfer is statically linked: scratch has no C library to run it.
|
|
||||||
RUN ldd /mfer 2>&1 | grep -q 'not a dynamic executable'
|
|
||||||
|
|
||||||
FROM scratch
|
FROM scratch
|
||||||
# scratch has no CA certificates; fetch needs them to verify HTTPS servers.
|
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
|
||||||
COPY --from=builder /mfer /mfer
|
COPY --from=builder /mfer /mfer
|
||||||
ENTRYPOINT ["/mfer"]
|
ENTRYPOINT ["/mfer"]
|
||||||
|
|||||||
@@ -47,11 +47,7 @@ allows verifying data integrity before decompression.
|
|||||||
The `innerMessage` field is compressed with
|
The `innerMessage` field is compressed with
|
||||||
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
|
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
|
||||||
enforce a decompression size limit to prevent decompression bombs. The reference
|
enforce a decompression size limit to prevent decompression bombs. The reference
|
||||||
implementation limits decompressed size to 256 MB. It writes zstd frames with a
|
implementation limits decompressed size to 256 MB.
|
||||||
window of at most 8 MiB, the largest window the zstd format recommends decoders
|
|
||||||
support, and refuses frames that ask for a larger one. It also refuses an inner
|
|
||||||
message whose file entries, hashes, timestamps and MIME types, counted at 160,
|
|
||||||
112, 64 and 16 bytes each, add up to more than 8 times its size.
|
|
||||||
|
|
||||||
## Inner Message (`MFFile`)
|
## Inner Message (`MFFile`)
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ export DOCKER_BUILDKIT := 1
|
|||||||
export PROGRESS_NO_TRUNC := 1
|
export PROGRESS_NO_TRUNC := 1
|
||||||
GOPATH := $(shell go env GOPATH)
|
GOPATH := $(shell go env GOPATH)
|
||||||
export PATH := $(PATH):$(GOPATH)/bin
|
export PATH := $(PATH):$(GOPATH)/bin
|
||||||
|
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
|
||||||
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
||||||
ARCH := $(shell uname -m)
|
ARCH := $(shell uname -m)
|
||||||
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
|
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
|
||||||
@@ -12,7 +13,7 @@ GOLDFLAGS += -X main.Version=$(VERSION)
|
|||||||
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
||||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||||
|
|
||||||
.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme generate
|
.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme
|
||||||
|
|
||||||
default: fmt test
|
default: fmt test
|
||||||
|
|
||||||
@@ -31,8 +32,8 @@ ci: test
|
|||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
fuzz:
|
$(PROTOC_GEN_GO):
|
||||||
@script/fuzz
|
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
|
||||||
|
|
||||||
fixme:
|
fixme:
|
||||||
@grep -nir fixme . | grep -v Makefile
|
@grep -nir fixme . | grep -v Makefile
|
||||||
@@ -54,14 +55,18 @@ fmt-check-md:
|
|||||||
hooks:
|
hooks:
|
||||||
@script/install-precommit
|
@script/install-precommit
|
||||||
|
|
||||||
generate:
|
devprereqs:
|
||||||
@script/generate
|
which golangci-lint || go install -v github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||||
|
|
||||||
bin/mfer: $(SOURCEFILES)
|
mfer/mf.pb.go: mfer/mf.proto
|
||||||
|
cd mfer && go generate .
|
||||||
|
|
||||||
|
bin/mfer: $(SOURCEFILES) mfer/mf.pb.go
|
||||||
|
protoc --version
|
||||||
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
|
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rfv bin/mfer cmd/mfer/mfer *.dockerimage
|
rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage
|
||||||
|
|
||||||
fmt:
|
fmt:
|
||||||
@script/fmt
|
@script/fmt
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
# mfer
|
# mfer
|
||||||
|
|
||||||
[mfer](https://git.eeqj.de/sneak/mfer) is a [WTFPL](https://wtfpl.net)-licensed
|
[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library and
|
||||||
(public domain) [Go](https://golang.org) library and command-line tool by
|
thin wrapper command-line utility written in [Go](https://golang.org) and first
|
||||||
[@sneak](https://sneak.berlin) that specifies and generates `.mf` manifest files
|
published in 2022 under the [WTFPL](https://wtfpl.net) (public domain) license.
|
||||||
over a directory tree to encapsulate metadata about the files — such as
|
It specifies and generates `.mf` manifest files over a directory tree of files
|
||||||
cryptographic checksums and signatures over same — to aid in archiving,
|
to encapsulate metadata about them (such as cryptographic checksums or
|
||||||
downloading, streaming, and mirroring. It was first published in 2022. The
|
signatures over same) to aid in archiving, downloading, and streaming, or
|
||||||
manifest files' data is serialized with Google's
|
mirroring. The manifest files' data is serialized with Google's
|
||||||
[protobuf serialization format](https://developers.google.com/protocol-buffers).
|
[protobuf serialization format](https://developers.google.com/protocol-buffers).
|
||||||
The structure of these files can be found
|
The structure of these files can be found
|
||||||
[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
||||||
@@ -21,41 +21,10 @@ This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
|||||||
as a de-facto standard and be incorporated into other software. A compatible
|
as a de-facto standard and be incorporated into other software. A compatible
|
||||||
javascript library is planned.
|
javascript library is planned.
|
||||||
|
|
||||||
# Getting Started
|
|
||||||
|
|
||||||
`mfer` builds from source with a Go 1.23+ toolchain. The generated protobuf code
|
|
||||||
is committed, so no `protoc` toolchain is required:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git clone https://git.eeqj.de/sneak/mfer.git
|
|
||||||
cd mfer
|
|
||||||
go build -o bin/mfer ./cmd/mfer
|
|
||||||
```
|
|
||||||
|
|
||||||
Generate a manifest for a directory tree, verify it later, and fetch a published
|
|
||||||
tree by URL:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# Write index.mf, a manifest of the files under the current directory.
|
|
||||||
bin/mfer gen .
|
|
||||||
|
|
||||||
# Verify the files on disk against the manifest. Exits nonzero if any file
|
|
||||||
# is missing or corrupted.
|
|
||||||
bin/mfer check index.mf
|
|
||||||
|
|
||||||
# Download and cryptographically verify a tree published over HTTP: mfer
|
|
||||||
# fetches <url>/index.mf, then downloads every file it lists.
|
|
||||||
bin/mfer fetch https://example.com/tree/
|
|
||||||
```
|
|
||||||
|
|
||||||
Run `bin/mfer help` for the full command list, or `bin/mfer <command> --help`
|
|
||||||
for a single command's options.
|
|
||||||
|
|
||||||
# Build Status
|
# Build Status
|
||||||
|
|
||||||
CI runs `script/cibuild`, which builds the Docker image with `--no-cache`, so
|
CI runs via `script/cibuild` (`docker build .`), which executes `make check`
|
||||||
the formatting, lint and test steps in the `Dockerfile` run on every build. The
|
(formatting, linting, tests). The `main` branch must always be green.
|
||||||
`main` branch must always be green.
|
|
||||||
|
|
||||||
# Entrypoints
|
# Entrypoints
|
||||||
|
|
||||||
@@ -65,34 +34,18 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
|||||||
development workflow, and the Makefile targets are thin shims that call them. We
|
development workflow, and the Makefile targets are thin shims that call them. We
|
||||||
provide:
|
provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (Go, Go module download, and
|
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go module
|
||||||
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
|
download, and node/yarn plus the prettier version pinned in
|
||||||
idempotently; golangci-lint is not installed, it runs only in Docker
|
`package.json`/`yarn.lock`), idempotently
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
- `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
`script/bootstrap`, then `script/install-precommit`
|
||||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||||
such as `script/docker`
|
such as `script/docker`
|
||||||
- `script/test` — run the test suite (`go test`); one test fails when
|
- `script/test` — run the test suite (`go test`), regenerating the protobuf code
|
||||||
`mfer/mf.proto` no longer matches the hash `script/generate` recorded
|
first if it is stale
|
||||||
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
|
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
|
||||||
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
- `script/fmt` — format all code and docs (writes): `gofumpt`,
|
||||||
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
`golangci-lint run --fix`, and `script/prettier --write`
|
||||||
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file,
|
|
||||||
and refuses to run with any other: `protoc` 33.4 (unpack
|
|
||||||
`protoc-33.4-<platform>.zip` from
|
|
||||||
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
|
|
||||||
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
|
|
||||||
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
|
|
||||||
installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to
|
|
||||||
`PATH`)
|
|
||||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
|
||||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
|
||||||
as ordinary tests
|
|
||||||
- `script/lint` — run `golangci-lint` in Docker: builds only the `lint` stage of
|
|
||||||
the `Dockerfile` (the Go format check, then the linter), uncached so it runs
|
|
||||||
every time, then removes the image
|
|
||||||
- `script/fmt` — format all code and docs (writes): `gofumpt` and
|
|
||||||
`script/prettier --write`
|
|
||||||
- `script/prettier` — run prettier over the repository's canonical file set
|
- `script/prettier` — run prettier over the repository's canonical file set
|
||||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||||
`--check`; the single definition of that file set, so `script/fmt` and
|
`--check`; the single definition of that file set, so `script/fmt` and
|
||||||
@@ -103,8 +56,8 @@ provide:
|
|||||||
Docker lint stage, whose image has no node
|
Docker lint stage, whose image has no node
|
||||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
||||||
- `script/docker` — build the Docker image tagged with the project name
|
- `script/docker` — build the Docker image tagged with the project name
|
||||||
- `script/cibuild` — CI entrypoint: builds the image with the same command as
|
- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile runs the
|
||||||
`script/docker`, uncached, so the checks in the Dockerfile run every time
|
checks)
|
||||||
- `script/precommit` — pre-commit checks: `go mod tidy` verification, then
|
- `script/precommit` — pre-commit checks: `go mod tidy` verification, then
|
||||||
`script/check`
|
`script/check`
|
||||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||||
@@ -118,18 +71,17 @@ yet. Primary development happens on a privately-run Gitea instance at
|
|||||||
[tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
[tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
||||||
|
|
||||||
Changes must always be formatted with a standard `go fmt`, syntactically valid,
|
Changes must always be formatted with a standard `go fmt`, syntactically valid,
|
||||||
and must pass the linting defined in the repository's `.golangci.yml`, which
|
and must pass the linting defined in the repository (presently only the
|
||||||
`make lint` runs in Docker. The `main` branch is protected and all changes must
|
`golangci-lint` defaults), which can be run with a `make lint`. The `main`
|
||||||
be made via [pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to
|
branch is protected and all changes must be made via
|
||||||
be merged. Any changes submitted to this project must also be
|
[pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
||||||
|
Any changes submitted to this project must also be
|
||||||
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
||||||
|
|
||||||
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
|
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
|
||||||
tooling requirements, and workflow conventions.
|
tooling requirements, and workflow conventions.
|
||||||
|
|
||||||
# Rationale
|
# Problem Statement
|
||||||
|
|
||||||
## The problem
|
|
||||||
|
|
||||||
Given a plain URL, there is no standard way to safely and programmatically
|
Given a plain URL, there is no standard way to safely and programmatically
|
||||||
download everything "under" that URL path. `wget -r` can traverse directory
|
download everything "under" that URL path. `wget -r` can traverse directory
|
||||||
@@ -153,7 +105,7 @@ Real issues I face:
|
|||||||
- when I download a large file via HTTP, I have no way of knowing if the file
|
- when I download a large file via HTTP, I have no way of knowing if the file
|
||||||
content is what it's supposed to be
|
content is what it's supposed to be
|
||||||
|
|
||||||
## The solution
|
# Proposed Solution
|
||||||
|
|
||||||
A standard, a manifest file format, and a tool for generating same.
|
A standard, a manifest file format, and a tool for generating same.
|
||||||
|
|
||||||
@@ -185,28 +137,6 @@ The manifest file would do several important things:
|
|||||||
- maybe a bittorrent chunklist for torrent client compatibility? perhaps a
|
- maybe a bittorrent chunklist for torrent client compatibility? perhaps a
|
||||||
top-level infohash for the whole manifest?
|
top-level infohash for the whole manifest?
|
||||||
|
|
||||||
# Design
|
|
||||||
|
|
||||||
The repository is split into a reusable library and a thin command-line wrapper
|
|
||||||
around it.
|
|
||||||
|
|
||||||
- `mfer/` is the reusable library and the heart of the project: it defines the
|
|
||||||
manifest format and implements building, scanning, checking, serialization,
|
|
||||||
and signing. The protobuf schema is `mfer/mf.proto`, and the generated code it
|
|
||||||
produces (`mfer/mf.pb.go`) is committed alongside it so the library builds
|
|
||||||
with `go get` and needs no `protoc` toolchain.
|
|
||||||
- `internal/cli/` holds the command implementations — `generate` (alias `gen`),
|
|
||||||
`check`, `freshen`, `export`, `list` (alias `ls`), `fetch`, and `version` —
|
|
||||||
that wire the library to the command-line interface.
|
|
||||||
- `internal/log/` provides the logging used by the commands and the library.
|
|
||||||
- `internal/bork/` provides the error the library returns when a manifest's
|
|
||||||
decompressed contents are not the size the manifest records.
|
|
||||||
- `cmd/mfer/` is the entrypoint: its `main` package runs `internal/cli` and
|
|
||||||
exits with the status it returns.
|
|
||||||
|
|
||||||
Everything under `internal/` is private to this repository; only the `mfer/`
|
|
||||||
package is intended for import by other software.
|
|
||||||
|
|
||||||
# Design Goals
|
# Design Goals
|
||||||
|
|
||||||
- Replace SHASUMS/SHASUMS.asc files
|
- Replace SHASUMS/SHASUMS.asc files
|
||||||
@@ -226,23 +156,18 @@ package is intended for import by other software.
|
|||||||
- metadata size should not be used as an excuse to sacrifice utility (such
|
- metadata size should not be used as an excuse to sacrifice utility (such
|
||||||
as providing checksums over each chunk of a large file)
|
as providing checksums over each chunk of a large file)
|
||||||
|
|
||||||
# Original Design Questions
|
# Open Questions
|
||||||
|
|
||||||
These were the open questions when the project started; open design questions
|
|
||||||
are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
|
||||||
|
|
||||||
- Should the manifest file include checksums of individual file chunks, or just
|
- Should the manifest file include checksums of individual file chunks, or just
|
||||||
for the whole assembled file? If so, should the chunk size be fixed or
|
for the whole assembled file?
|
||||||
dynamic? Still open, on
|
|
||||||
[issue 81](https://git.eeqj.de/sneak/mfer/issues/81#issuecomment-118698).
|
- If so, should the chunksize be fixed or dynamic?
|
||||||
|
|
||||||
- Should the manifest signature format be GnuPG signatures, or those from
|
- Should the manifest signature format be GnuPG signatures, or those from
|
||||||
OpenBSD's signify (of which there is a good
|
OpenBSD's signify (of which there is a good
|
||||||
[golang implementation](https://github.com/frankbraun/gosignify))? Still open,
|
[golang implementation](https://github.com/frankbraun/gosignify)?
|
||||||
as question 10 on [issue 82](https://git.eeqj.de/sneak/mfer/issues/82).
|
|
||||||
|
|
||||||
- Should the on-disk serialization format be proto3 or json? Settled: it is
|
- Should the on-disk serialization format be proto3 or json?
|
||||||
proto3, see `FORMAT.md` and `mfer/mf.proto`.
|
|
||||||
|
|
||||||
# Tool Examples
|
# Tool Examples
|
||||||
|
|
||||||
@@ -320,10 +245,207 @@ regardless of filesystem format.
|
|||||||
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your desired
|
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your desired
|
||||||
username for an account on this Gitea instance.
|
username for an account on this Gitea instance.
|
||||||
|
|
||||||
# TODO
|
# TODO: Remaining Work for 1.0
|
||||||
|
|
||||||
Open work, open design questions included, is tracked in this repo's issues:
|
## Design Questions (Owner Decision Required)
|
||||||
[https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues).
|
|
||||||
|
These require @sneak's input before implementation. Answers should be added
|
||||||
|
inline below each question.
|
||||||
|
|
||||||
|
### Format Design
|
||||||
|
|
||||||
|
**1. Should `MFFileChecksum` be simplified?** Currently it's a separate message
|
||||||
|
wrapping a single `bytes multiHash` field. Since multihash already
|
||||||
|
self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath`
|
||||||
|
would be simpler and reduce per-file protobuf overhead. Is the extra message
|
||||||
|
layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**2. Should file permissions/mode be stored?** The format stores mtime/ctime but
|
||||||
|
not Unix file permissions. For archival use this may not matter, but for
|
||||||
|
software distribution or filesystem restoration it's a gap. Should we reserve a
|
||||||
|
field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**3. Should `atime` be removed from the schema?** Access time is volatile,
|
||||||
|
non-deterministic, and often disabled (`noatime`). Including it means two
|
||||||
|
manifests of the same directory at different times will differ, which conflicts
|
||||||
|
with the determinism goal. Remove it, or document it as "never set by default"?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**4. What are the path normalization rules?** The proto has `string path` with
|
||||||
|
no specification about: always forward-slash? Must be relative? No `..`
|
||||||
|
components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path
|
||||||
|
length? This is a security issue (path traversal) and a cross-platform
|
||||||
|
compatibility issue. What rules should the spec mandate?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**5. Should we add a version byte after the magic?** Currently `ZNAVSRFG` is
|
||||||
|
followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would
|
||||||
|
allow future framing changes without requiring protobuf parsing to detect the
|
||||||
|
version. `MFFileOuter.Version` serves this purpose but requires successful
|
||||||
|
deserialization to read. Worth the extra byte?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**6. Should we add a length-prefix after the magic?** Protobuf is not
|
||||||
|
self-delimiting. If we ever want to concatenate manifests or append data after
|
||||||
|
the protobuf, the current framing is insufficient. Add a varint or fixed-width
|
||||||
|
length-prefix?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
### Signature Design
|
||||||
|
|
||||||
|
**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
|
||||||
|
The code currently hashes compressed data (good for verifying before
|
||||||
|
decompression), but this should be explicitly documented. Which is the intended
|
||||||
|
behavior?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**8. Should `signatureString()` sign raw bytes instead of a hex-encoded
|
||||||
|
string?** Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex
|
||||||
|
encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or
|
||||||
|
compressed `innerMessage` directly) would be simpler. Keep the string format or
|
||||||
|
switch to raw bytes?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**9. Should we support detached signature files (`.mf.sig`)?** Embedded
|
||||||
|
signatures are better for single-file distribution. Detached `.mf.sig` files
|
||||||
|
follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP
|
||||||
|
serving. Support both modes?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**10. GPG vs pure-Go crypto for signatures?** Shelling out to `gpg` is fragile
|
||||||
|
(may not be installed, version-dependent output).
|
||||||
|
`github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could use
|
||||||
|
Ed25519/signify (simpler, no key management). Which direction?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
### Implementation Design
|
||||||
|
|
||||||
|
**11. Should manifests be deterministic by default?** This means: sort file
|
||||||
|
entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`.
|
||||||
|
Should determinism be the default, with a `--include-timestamps` flag to opt in?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**12. Should we consolidate or keep both scanner/checker implementations?**
|
||||||
|
There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go`
|
||||||
|
(typed with `FileSize`, `RelFilePath`) and `internal/scanner/` +
|
||||||
|
`internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior.
|
||||||
|
Delete the `internal/` versions?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**13. Should the `manifest` type be exported?** Currently unexported with
|
||||||
|
exported constructors (`NewManifestFromReader`, `NewManifestFromFile`).
|
||||||
|
Consumers can't declare `var m *mfer.manifest`. Export the type, or define an
|
||||||
|
interface?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
**14. What should the Go module path be for 1.0?** Currently
|
||||||
|
`sneak.berlin/go/mfer` in `go.mod` but `git.eeqj.de/sneak/mfer/mfer` in the
|
||||||
|
proto `go_package` option. Which is canonical?
|
||||||
|
|
||||||
|
> _answer:_
|
||||||
|
|
||||||
|
## Implementation Tasks
|
||||||
|
|
||||||
|
### Repo Infrastructure
|
||||||
|
|
||||||
|
- [ ] Add `.golangci.yml` (fetch from
|
||||||
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`)
|
||||||
|
- [ ] Add `.editorconfig`
|
||||||
|
- [ ] Add `.gitea/workflows/check.yml` that runs `docker build .`
|
||||||
|
|
||||||
|
### Format & Correctness
|
||||||
|
|
||||||
|
- [ ] Resolve proto `go_package` path inconsistency
|
||||||
|
(`git.eeqj.de/sneak/mfer/mfer` vs `sneak.berlin/go/mfer`)
|
||||||
|
- [ ] Specify path invariants — add proto comments requiring UTF-8,
|
||||||
|
forward-slash, relative paths, no `..`, no leading `/`; validate in
|
||||||
|
`Builder.AddFile` and `Builder.AddFileWithHash` (pending design question
|
||||||
|
answer)
|
||||||
|
- [ ] Remove or deprecate `atime` from proto (pending design question answer)
|
||||||
|
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file
|
||||||
|
permissions (pending design question answer)
|
||||||
|
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
|
||||||
|
(pending design question answer)
|
||||||
|
- [ ] Write format specification document — separate from README: magic, outer
|
||||||
|
structure, compression, inner structure, path invariants, signature
|
||||||
|
scheme, canonical serialization
|
||||||
|
|
||||||
|
### Library
|
||||||
|
|
||||||
|
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on
|
||||||
|
`mfer/` package versions; update CLI code (pending design question answer)
|
||||||
|
- [ ] Add deterministic file ordering — sort entries by path (lexicographic,
|
||||||
|
byte-order) in `Builder.Build()`; add test asserting byte-identical output
|
||||||
|
from two runs
|
||||||
|
- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()`
|
||||||
|
with `m.pbOuter.Size` as bound
|
||||||
|
- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)`
|
||||||
|
or `errors.Is(err, fs.ErrNotExist)`
|
||||||
|
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading,
|
||||||
|
return error on mismatch
|
||||||
|
- [ ] Export the `manifest` type or define a public interface (pending design
|
||||||
|
question answer) — currently consumers cannot hold a reference to a loaded
|
||||||
|
manifest in their own type declarations
|
||||||
|
- [ ] Replace GPG subprocess calls with pure-Go crypto (pending design question
|
||||||
|
answer) — current implementation shells out to `gpg` which may not be
|
||||||
|
installed
|
||||||
|
- [ ] Add timeout to any remaining subprocess calls
|
||||||
|
|
||||||
|
### CLI
|
||||||
|
|
||||||
|
- [ ] Fix flag naming — all CLI flags should use kebab-case as primary
|
||||||
|
(`--include-dotfiles`, `--follow-symlinks`)
|
||||||
|
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or
|
||||||
|
`url.JoinPath()` instead of string concatenation
|
||||||
|
- [ ] Add progress rate-limiting to Checker — throttle to once per second,
|
||||||
|
matching Scanner
|
||||||
|
- [ ] Add `--deterministic` flag or make it default — omit `createdAt`, sort
|
||||||
|
files (pending design question answer)
|
||||||
|
- [ ] Wire `--version` flag properly (currently only a `version` subcommand
|
||||||
|
exists; top-level `--version` shows urfave/cli generic output)
|
||||||
|
- [ ] Add retry logic to `fetch` — currently no retries on transient HTTP
|
||||||
|
errors; needs exponential backoff
|
||||||
|
- [ ] `fetch` command uses bare `http.Get` with no timeout — needs `http.Client`
|
||||||
|
with configurable timeout
|
||||||
|
|
||||||
|
### Testing & Robustness
|
||||||
|
|
||||||
|
- [ ] Add fuzzing tests for `NewManifestFromReader` — protobuf deserialization
|
||||||
|
of untrusted input needs fuzz coverage
|
||||||
|
- [ ] Add integration test for `freshen` CLI command — current tests only verify
|
||||||
|
setup, not the actual freshen operation end-to-end
|
||||||
|
- [ ] Add test for `fetch` CLI command end-to-end (currently only `downloadFile`
|
||||||
|
is tested)
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
|
||||||
|
- [ ] Promote `FORMAT.md` as primary spec reference; README should link to it
|
||||||
|
more prominently
|
||||||
|
- [ ] Audit and update all error messages for consistency and helpfulness
|
||||||
|
- [ ] Document the signature scheme more thoroughly (canonical string format,
|
||||||
|
verification steps)
|
||||||
|
|
||||||
|
### Release
|
||||||
|
|
||||||
|
- [ ] Finalize Go module path
|
||||||
|
- [ ] Update version constant in `mfer/constants.go`
|
||||||
|
- [ ] Add `--version` output matching SemVer
|
||||||
|
- [ ] Tag `v1.0.0`
|
||||||
|
|
||||||
# See Also
|
# See Also
|
||||||
|
|
||||||
@@ -340,9 +462,9 @@ Open work, open design questions included, is tracked in this repo's issues:
|
|||||||
- Issues:
|
- Issues:
|
||||||
[https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
[https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
||||||
|
|
||||||
# Author
|
# Authors
|
||||||
|
|
||||||
- [@sneak](https://sneak.berlin)
|
- [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
|
||||||
|
|
||||||
# License
|
# License
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
# Workflow
|
||||||
|
|
||||||
|
- branch (from `main`)
|
||||||
|
- do the work in Next Step
|
||||||
|
- move Next Step to the top of Completed Steps
|
||||||
|
- move the top item of Future Steps into Next Step
|
||||||
|
- commit (`TODO.md` changes in the same commit as the work)
|
||||||
|
- merge to `main` if the branch is not protected, otherwise open a PR
|
||||||
|
- push
|
||||||
|
|
||||||
|
# Status
|
||||||
|
|
||||||
|
pre-1.0. No git tags. README section "TODO: Remaining Work for 1.0" lists open
|
||||||
|
design questions and implementation tasks; policy compliance work is in flight
|
||||||
|
and unmerged.
|
||||||
|
|
||||||
|
# Next Step
|
||||||
|
|
||||||
|
Work through the remaining compliance items folded from the 2026-07-02 audit
|
||||||
|
(the first group under Future Steps): `.editorconfig`, `.gitignore` coverage,
|
||||||
|
gofumpt-based `fmt-check`, README "Getting Started", and the rest.
|
||||||
|
`.golangci.yml` and `TODO.md` are tracked and committed as of 2026-08-07, so the
|
||||||
|
only thing left of the `chore/align-repo-policies` branch is the list below.
|
||||||
|
|
||||||
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-03: `fetch` refuses any manifest path that runs through a symlink
|
||||||
|
already in the destination directory, checked before each of its writes
|
||||||
|
(directories, temp file, rename), so such a symlink cannot send a write
|
||||||
|
outside it (#86)
|
||||||
|
- 2026-10-02: a plain `docker build .` of a clone now stamps the tag or short
|
||||||
|
commit into `mfer version` instead of nothing: `.dockerignore` sends `.git`
|
||||||
|
(not `.git/config`), and the build stage takes the `VERSION` build argument,
|
||||||
|
otherwise `git describe --tags --always`, failing if `.git` is present and no
|
||||||
|
version comes out. `script/docker` is the canonical copy, which passes
|
||||||
|
`VERSION`; `bin/gitrev.sh` uses `--tags` too (#112)
|
||||||
|
- 2026-09-21: validate manifest entry paths on deserialize so untrusted `.mf`
|
||||||
|
files cannot make `Checker` stat or read outside `basePath` (#61)
|
||||||
|
- 2026-09-21: rewrote `script/test` to the canonical pattern (30s timeout,
|
||||||
|
`-race -cover`, quiet-first with verbose-on-failure rerun) and fixed the
|
||||||
|
process-global logger data race it surfaced (#67)
|
||||||
|
- 2026-09-21: added the canonical `.editorconfig`, made `.gitignore` cover
|
||||||
|
secrets, OS, editor, and Go artifacts, and removed the dead Drone CI
|
||||||
|
references from `.gitignore` and `bin/gitrev.sh` (#72)
|
||||||
|
- 2026-08-09: added `.prettierrc`/`.prettierignore`, gave `script/fmt` and
|
||||||
|
`script/fmt-check` one shared prettier file set via `script/prettier`, dropped
|
||||||
|
the `|| true` that hid prettier failures, and added a node-based Dockerfile
|
||||||
|
stage so a markdown formatting violation fails `docker build .` (#69)
|
||||||
|
- 2026-08-07: updated golangci-lint to v2.12.2 everywhere it is pinned
|
||||||
|
(`Makefile`, `Dockerfile`), added the canonical `.golangci.yml`
|
||||||
|
(`default: all`), and fixed all resulting lint findings across the codebase
|
||||||
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
||||||
|
shims, README Entrypoints section
|
||||||
|
- 2026-07-03: aligned repo tooling, docs, and config with standardized policies
|
||||||
|
(7d9a138, on chore/align-repo-policies, unmerged)
|
||||||
|
- 2026-06-28: moved to standardized repo policies (#56, on main)
|
||||||
|
- 2026-04-07: added 1.0 roadmap as README TODO section, removed old TODO.md
|
||||||
|
(#54)
|
||||||
|
- 2026-03-20: added Gitea Actions CI workflow (#53)
|
||||||
|
- 2026-03-17: added REPO_POLICIES.md, renamed CLAUDE.md to AGENTS.md (#51);
|
||||||
|
removed committed .index.mf (#52)
|
||||||
|
- 2026-03-15: split Dockerfile with pre-built golangci-lint stage for faster CI
|
||||||
|
(#45)
|
||||||
|
- 2026-03-01: 1.0 quality polish: code review, tests, bug fixes, docs (#32)
|
||||||
|
- 2026-02-20: deterministic file ordering in Builder.Build() (#28); removed
|
||||||
|
committed vendor/modcache archives (#35)
|
||||||
|
- 2026-02-08: added --seed flag for deterministic manifest UUID
|
||||||
|
|
||||||
|
# Future Steps
|
||||||
|
|
||||||
|
- Compliance (fold of TODO.md audit 2026-07-02; verify which items the in-flight
|
||||||
|
branch already closes, then check off):
|
||||||
|
- Add .editorconfig (canonical copy from sneak/prompts)
|
||||||
|
- Make .gitignore cover secrets (.env, _.key, _.pem), OS files (.DS_Store),
|
||||||
|
and editor files (_.swp, _~)
|
||||||
|
- Make fmt-check/lint verify with gofumpt, not gofmt -l, so `make check`
|
||||||
|
matches what `make fmt` writes
|
||||||
|
- Add README "Getting Started" section with copy-pasteable install/usage
|
||||||
|
block
|
||||||
|
- Move FORMAT.md from repo root to docs/ and update the AGENTS.md reference
|
||||||
|
- Pin Makefile-installed Go tools (`protoc-gen-go@v1.28.1`,
|
||||||
|
`golangci-lint@v2.12.2`) by module hash, not mutable tag
|
||||||
|
- Add explicit README "Rationale" heading (content exists under other
|
||||||
|
names); name the author in the README Description first line
|
||||||
|
- Reconcile root-level AGENTS.md with directory-hygiene policy (keep or
|
||||||
|
relocate)
|
||||||
|
- Add a `make build` target
|
||||||
|
- Rewrite `make hooks` to use printf or a heredoc instead of non-portable
|
||||||
|
`echo '...\n...'`
|
||||||
|
- Answer the 14 owner design questions in the README 1.0 roadmap:
|
||||||
|
- Format: simplify MFFileChecksum; store file mode; drop atime; specify path
|
||||||
|
normalization rules; version byte after magic; length-prefix after magic
|
||||||
|
- Signatures: hash covers compressed or uncompressed data; sign raw bytes vs
|
||||||
|
hex canonical string; detached .mf.sig support; GPG subprocess vs pure-Go
|
||||||
|
crypto
|
||||||
|
- Implementation: deterministic manifests by default; consolidate duplicate
|
||||||
|
scanner/checker implementations; export the manifest type; canonical Go
|
||||||
|
module path for 1.0
|
||||||
|
- Format and correctness:
|
||||||
|
- Resolve proto go_package vs go.mod module path inconsistency
|
||||||
|
- Specify and validate path invariants (UTF-8, forward-slash, relative, no
|
||||||
|
.., no leading /)
|
||||||
|
- Remove or deprecate atime; reserve mode field; add version byte (all
|
||||||
|
pending design answers)
|
||||||
|
- Write a standalone format specification document
|
||||||
|
- Library:
|
||||||
|
- Delete internal/scanner and internal/checker; consolidate on the mfer/
|
||||||
|
package versions (pending design answer)
|
||||||
|
- Add decompression size limit via io.LimitReader in deserializeInner()
|
||||||
|
- Fix errors.Is dead code in checker; make AddFile verify totalRead == size
|
||||||
|
- Export manifest type or define a public interface (pending)
|
||||||
|
- Replace GPG subprocess with pure-Go crypto (pending); add timeouts to
|
||||||
|
remaining subprocess calls
|
||||||
|
- CLI:
|
||||||
|
- Kebab-case primary flag names; fix fetch URL construction with
|
||||||
|
url.JoinPath; add http.Client timeout and retry with backoff to fetch;
|
||||||
|
rate-limit Checker progress output; add --deterministic flag or default;
|
||||||
|
wire top-level --version properly
|
||||||
|
- Testing:
|
||||||
|
- Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch
|
||||||
|
- Documentation:
|
||||||
|
- Promote docs/FORMAT.md as primary spec reference; audit error messages;
|
||||||
|
document the signature scheme fully
|
||||||
|
- Release:
|
||||||
|
- Finalize module path, bump version constant, SemVer --version output, tag
|
||||||
|
v1.0.0
|
||||||
@@ -12,13 +12,13 @@ require (
|
|||||||
github.com/pterm/pterm v0.12.35
|
github.com/pterm/pterm v0.12.35
|
||||||
github.com/spf13/afero v1.8.0
|
github.com/spf13/afero v1.8.0
|
||||||
github.com/stretchr/testify v1.8.1
|
github.com/stretchr/testify v1.8.1
|
||||||
github.com/urfave/cli/v2 v2.27.7
|
github.com/urfave/cli/v2 v2.23.6
|
||||||
google.golang.org/protobuf v1.28.1
|
google.golang.org/protobuf v1.28.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/atomicgo/cursor v0.0.1 // indirect
|
github.com/atomicgo/cursor v0.0.1 // indirect
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||||
github.com/fatih/color v1.7.0 // indirect
|
github.com/fatih/color v1.7.0 // indirect
|
||||||
github.com/gookit/color v1.4.2 // indirect
|
github.com/gookit/color v1.4.2 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||||
@@ -34,7 +34,7 @@ require (
|
|||||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||||
golang.org/x/sys v0.1.0 // indirect
|
golang.org/x/sys v0.1.0 // indirect
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||||
|
|||||||
@@ -61,8 +61,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
|||||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
|
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@@ -231,12 +231,12 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
|
|||||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||||
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
|
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
|
||||||
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
|
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
|
||||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
|
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
|
||||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
||||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
||||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
|||||||
+17
-33
@@ -2,7 +2,6 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -11,7 +10,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
@@ -76,22 +74,25 @@ func safeRateUint64(rate float64) uint64 {
|
|||||||
return uint64(rate)
|
return uint64(rate)
|
||||||
}
|
}
|
||||||
|
|
||||||
// findManifest returns the path of the manifest with the default name in
|
// findManifest looks for a manifest file in the given directory.
|
||||||
// dir, or an error if there is none.
|
// It checks for index.mf and .index.mf, returning the first one found.
|
||||||
func findManifest(fs afero.Fs, dir string) (string, error) {
|
func findManifest(fs afero.Fs, dir string) (string, error) {
|
||||||
path := filepath.Join(dir, defaultManifestName)
|
candidates := []string{"index.mf", ".index.mf"}
|
||||||
|
for _, name := range candidates {
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
|
||||||
exists, err := afero.Exists(fs, path)
|
exists, err := afero.Exists(fs, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists {
|
if exists {
|
||||||
return "", fmt.Errorf("%w in %s (looked for %s)",
|
return path, nil
|
||||||
errNoManifestFound, dir, defaultManifestName)
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, nil
|
return "", fmt.Errorf(
|
||||||
|
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
||||||
@@ -125,9 +126,7 @@ func (mfa *CLIApp) fetchManifestToTemp(url string) (string, error) {
|
|||||||
|
|
||||||
// verifyRequiredSigner enforces the --require-signature fingerprint
|
// verifyRequiredSigner enforces the --require-signature fingerprint
|
||||||
// against the manifest's embedded signing key.
|
// against the manifest's embedded signing key.
|
||||||
func verifyRequiredSigner(
|
func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
|
||||||
ctx context.Context, chk *mfer.Checker, requiredSigner string,
|
|
||||||
) error {
|
|
||||||
// Validate fingerprint format: must be exactly 40 hex characters
|
// Validate fingerprint format: must be exactly 40 hex characters
|
||||||
if len(requiredSigner) != fingerprintHexLen {
|
if len(requiredSigner) != fingerprintHexLen {
|
||||||
return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
|
return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
|
||||||
@@ -146,7 +145,7 @@ func verifyRequiredSigner(
|
|||||||
// Extract fingerprint from the embedded public key (not from the
|
// Extract fingerprint from the embedded public key (not from the
|
||||||
// signer field). This validates the key is importable and gets its
|
// signer field). This validates the key is importable and gets its
|
||||||
// actual fingerprint.
|
// actual fingerprint.
|
||||||
embeddedFP, err := chk.ExtractEmbeddedSigningKeyFP(ctx)
|
embeddedFP, err := chk.ExtractEmbeddedSigningKeyFP()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"failed to extract fingerprint from embedded signing key: %w", err)
|
"failed to extract fingerprint from embedded signing key: %w", err)
|
||||||
@@ -164,9 +163,7 @@ func verifyRequiredSigner(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// reportCheckProgress renders progress updates until the channel closes.
|
// reportCheckProgress renders progress updates until the channel closes.
|
||||||
func reportCheckProgress(progress <-chan mfer.CheckStatus, wg *sync.WaitGroup) {
|
func reportCheckProgress(progress <-chan mfer.CheckStatus) {
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
for status := range progress {
|
for status := range progress {
|
||||||
if status.ETA > 0 {
|
if status.ETA > 0 {
|
||||||
log.Progressf("Checking: %d/%d files, %s/s, ETA %s, %d failures",
|
log.Progressf("Checking: %d/%d files, %s/s, ETA %s, %d failures",
|
||||||
@@ -238,17 +235,11 @@ func runCheck(ctx *cli.Context, chk *mfer.Checker, showProgress bool) (int64, er
|
|||||||
results := make(chan mfer.Result, 1)
|
results := make(chan mfer.Result, 1)
|
||||||
|
|
||||||
// Set up progress channel
|
// Set up progress channel
|
||||||
var (
|
var progress chan mfer.CheckStatus
|
||||||
progress chan mfer.CheckStatus
|
|
||||||
progressWg sync.WaitGroup
|
|
||||||
)
|
|
||||||
|
|
||||||
if showProgress {
|
if showProgress {
|
||||||
progress = make(chan mfer.CheckStatus, 1)
|
progress = make(chan mfer.CheckStatus, 1)
|
||||||
|
|
||||||
progressWg.Add(1)
|
go reportCheckProgress(progress)
|
||||||
|
|
||||||
go reportCheckProgress(progress, &progressWg)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process results in a goroutine
|
// Process results in a goroutine
|
||||||
@@ -260,9 +251,6 @@ func runCheck(ctx *cli.Context, chk *mfer.Checker, showProgress bool) (int64, er
|
|||||||
|
|
||||||
// Run check
|
// Run check
|
||||||
err := chk.Check(ctx.Context, results, progress)
|
err := chk.Check(ctx.Context, results, progress)
|
||||||
|
|
||||||
progressWg.Wait()
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("check failed: %w", err)
|
return 0, fmt.Errorf("check failed: %w", err)
|
||||||
}
|
}
|
||||||
@@ -307,11 +295,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
|||||||
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
|
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
|
||||||
|
|
||||||
// Create checker
|
// Create checker
|
||||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
|
||||||
ManifestPath: manifestPath,
|
|
||||||
BasePath: basePath,
|
|
||||||
Fs: mfa.Fs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to load manifest: %w", err)
|
return fmt.Errorf("failed to load manifest: %w", err)
|
||||||
}
|
}
|
||||||
@@ -319,7 +303,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
|||||||
// Check signature requirement
|
// Check signature requirement
|
||||||
requiredSigner := ctx.String("require-signature")
|
requiredSigner := ctx.String("require-signature")
|
||||||
if requiredSigner != "" {
|
if requiredSigner != "" {
|
||||||
err = verifyRequiredSigner(ctx.Context, chk, requiredSigner)
|
err = verifyRequiredSigner(chk, requiredSigner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-396
@@ -5,21 +5,15 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
urfcli "github.com/urfave/cli/v2"
|
urfcli "github.com/urfave/cli/v2"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
"sneak.berlin/go/mfer/mfer"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,11 +23,10 @@ const (
|
|||||||
testFile1 = "/testdir/file1.txt"
|
testFile1 = "/testdir/file1.txt"
|
||||||
testMF = "/testdir/test.mf"
|
testMF = "/testdir/test.mf"
|
||||||
testOutput = "/output.mf"
|
testOutput = "/output.mf"
|
||||||
|
testOutputTmp = "/output.mf.tmp"
|
||||||
testManifest = "/manifest.mf"
|
testManifest = "/manifest.mf"
|
||||||
testFlagBase = "--base"
|
testFlagBase = "--base"
|
||||||
testFlagNoExtra = "--no-extra-files"
|
testFlagNoExtra = "--no-extra-files"
|
||||||
testFlagVersion = "--version"
|
|
||||||
testFlagVerbose = "--verbose"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var errSimulatedWrite = errors.New("simulated write failure")
|
var errSimulatedWrite = errors.New("simulated write failure")
|
||||||
@@ -46,32 +39,12 @@ var errSimulatedWrite = errors.New("simulated write failure")
|
|||||||
var runMu sync.Mutex
|
var runMu sync.Mutex
|
||||||
|
|
||||||
// runCLI invokes RunWithOptions while holding runMu so parallel tests
|
// runCLI invokes RunWithOptions while holding runMu so parallel tests
|
||||||
// capture their own output, and returns its exit code.
|
// capture their own output.
|
||||||
func runCLI(opts *RunOptions) int {
|
func runCLI(opts *RunOptions) int {
|
||||||
exitCode, _ := runCLIWithLevel(opts)
|
|
||||||
|
|
||||||
return exitCode
|
|
||||||
}
|
|
||||||
|
|
||||||
// runCLIWithLevel is runCLI that also returns the log level the run left
|
|
||||||
// set, read while runMu still keeps other runs from changing it. Each run
|
|
||||||
// starts at the default level, as a new process does. Before releasing the
|
|
||||||
// lock it points the process-global logger at io.Discard: other tests log
|
|
||||||
// outside the lock (manifest loads, scans), and those lines must not land in
|
|
||||||
// this run's buffers once it has returned and its test is reading them.
|
|
||||||
func runCLIWithLevel(opts *RunOptions) (int, log.Level) {
|
|
||||||
runMu.Lock()
|
runMu.Lock()
|
||||||
defer runMu.Unlock()
|
defer runMu.Unlock()
|
||||||
|
|
||||||
log.SetLevel(log.InfoLevel)
|
return RunWithOptions(opts)
|
||||||
|
|
||||||
exitCode := RunWithOptions(opts)
|
|
||||||
level := log.GetLevel()
|
|
||||||
|
|
||||||
log.SetOutput(io.Discard, io.Discard)
|
|
||||||
log.Init()
|
|
||||||
|
|
||||||
return exitCode, level
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
func TestMain(m *testing.M) {
|
||||||
@@ -129,7 +102,7 @@ func TestVersionCommand(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
opts := testOpts([]string{testApp, cmdVersion}, fs)
|
opts := testOpts([]string{testApp, "version"}, fs)
|
||||||
|
|
||||||
exitCode := runCLI(opts)
|
exitCode := runCLI(opts)
|
||||||
|
|
||||||
@@ -140,201 +113,6 @@ func TestVersionCommand(t *testing.T) {
|
|||||||
assert.Contains(t, stdout, "abc123")
|
assert.Contains(t, stdout, "abc123")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestVFlagCollision covers the -v/--verbose vs --version flag interaction
|
|
||||||
// (issue #64). Verbose owns -v; version answers to --version and -V. None of
|
|
||||||
// these invocations may produce a parser error, and the two ways of asking
|
|
||||||
// for the version must print the same thing.
|
|
||||||
func TestVFlagCollision(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Invocations that must print the version and exit 0.
|
|
||||||
versionCases := map[string][]string{
|
|
||||||
"long version flag": {testApp, testFlagVersion},
|
|
||||||
"short version flag": {testApp, "-V"},
|
|
||||||
"verbose then version": {testApp, "-v", testFlagVersion},
|
|
||||||
"long verbose and version": {testApp, "--verbose", testFlagVersion},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, args := range versionCases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
opts := testOpts(args, afero.NewMemMapFs())
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
|
||||||
assert.Contains(t, testStdout(t, opts), mfer.Version)
|
|
||||||
assert.NotContains(t, testStderr(t, opts), "two forms of the same flag")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Invocations that must enable verbose and exit 0 without a parser error.
|
|
||||||
verboseCases := map[string][]string{
|
|
||||||
"short verbose flag": {testApp, "-v"},
|
|
||||||
"long verbose flag": {testApp, "--verbose"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, args := range verboseCases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
opts := testOpts(args, afero.NewMemMapFs())
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
|
||||||
assert.Contains(t, testStdout(t, opts), cmdGenerate,
|
|
||||||
"root should show help listing subcommands")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVersionFlagAndCommandMatch asserts that "mfer --version" and
|
|
||||||
// "mfer version" produce identical output (issue #64).
|
|
||||||
func TestVersionFlagAndCommandMatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
flagOpts := testOpts([]string{testApp, testFlagVersion}, afero.NewMemMapFs())
|
|
||||||
require.Equal(t, 0, runCLI(flagOpts))
|
|
||||||
|
|
||||||
cmdOpts := testOpts([]string{testApp, cmdVersion}, afero.NewMemMapFs())
|
|
||||||
require.Equal(t, 0, runCLI(cmdOpts))
|
|
||||||
|
|
||||||
assert.Equal(t, testStdout(t, flagOpts), testStdout(t, cmdOpts))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRootVerbosityFlags asserts that -q and -v given before any subcommand
|
|
||||||
// name take effect: in the root itself, and in the fetch and export
|
|
||||||
// subcommands (issue #64). It does not run in parallel: other tests log
|
|
||||||
// outside runMu (manifest loads, scans), and a line logged while one of these
|
|
||||||
// runs is in progress lands in its output.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // see above
|
|
||||||
func TestRootVerbosityFlags(t *testing.T) {
|
|
||||||
t.Run("quiet with no subcommand hides the banner", func(t *testing.T) {
|
|
||||||
loud := testOpts([]string{testApp}, afero.NewMemMapFs())
|
|
||||||
require.Equal(t, 0, runCLI(loud))
|
|
||||||
assert.Contains(t, testStdout(t, loud), banner)
|
|
||||||
|
|
||||||
quiet := testOpts([]string{testApp, "-q"}, afero.NewMemMapFs())
|
|
||||||
require.Equal(t, 0, runCLI(quiet))
|
|
||||||
assert.Contains(t, testStdout(t, quiet), cmdGenerate)
|
|
||||||
assert.NotContains(t, testStdout(t, quiet), banner)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("quiet before fetch hides the banner", func(t *testing.T) {
|
|
||||||
opts := testOpts([]string{testApp, "-q", cmdFetch}, afero.NewMemMapFs())
|
|
||||||
|
|
||||||
assert.Equal(t, 1, runCLI(opts))
|
|
||||||
assert.Contains(t, testStderr(t, opts), errURLRequired.Error())
|
|
||||||
assert.NotContains(t, testStdout(t, opts), banner)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("verbose twice before fetch enables debug logging", func(t *testing.T) {
|
|
||||||
opts := testOpts([]string{testApp, "-v", "-v", cmdFetch}, afero.NewMemMapFs())
|
|
||||||
|
|
||||||
assert.Equal(t, 1, runCLI(opts))
|
|
||||||
assert.Contains(t, testStderr(t, opts), "fetchManifestOperation()")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("quiet before export hides the manifest summary", func(t *testing.T) {
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
manifest := buildTestManifest(t, map[string][]byte{"a.txt": []byte("a")})
|
|
||||||
require.NoError(t, afero.WriteFile(fs, testManifest, manifest, 0o644))
|
|
||||||
|
|
||||||
loud := testOpts([]string{testApp, cmdExport, testManifest}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(loud))
|
|
||||||
assert.Contains(t, testStderr(t, loud), "loaded manifest")
|
|
||||||
|
|
||||||
quiet := testOpts([]string{testApp, "-q", cmdExport, testManifest}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(quiet))
|
|
||||||
assert.NotContains(t, testStderr(t, quiet), "loaded manifest")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// commandsTakingVerbose returns the command lines -v can follow: the root and
|
|
||||||
// the generate, check, freshen and fetch subcommands.
|
|
||||||
func commandsTakingVerbose() [][]string {
|
|
||||||
return [][]string{
|
|
||||||
{testApp},
|
|
||||||
{testApp, cmdGenerate},
|
|
||||||
{testApp, cmdCheck},
|
|
||||||
{testApp, cmdFreshen},
|
|
||||||
{testApp, cmdFetch},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVerboseCount asserts that one -v or --verbose gives verbose output and
|
|
||||||
// two -v give debug output (issue #125). urfave/cli before v2.25.5 counted a
|
|
||||||
// flag given by its alias twice, so one -v gave debug output.
|
|
||||||
func TestVerboseCount(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
flags []string
|
|
||||||
want log.Level
|
|
||||||
}{
|
|
||||||
{[]string{"-v"}, log.VerboseLevel},
|
|
||||||
{[]string{testFlagVerbose}, log.VerboseLevel},
|
|
||||||
{[]string{"-v", "-v"}, log.DebugLevel},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, command := range commandsTakingVerbose() {
|
|
||||||
for _, tc := range cases {
|
|
||||||
args := slices.Concat(command, tc.flags)
|
|
||||||
|
|
||||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, level := runCLIWithLevel(testOpts(args, afero.NewMemMapFs()))
|
|
||||||
assert.Equal(t, tc.want, level)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCombinedShortVerboseRefused asserts that -vv is refused (issue #125):
|
|
||||||
// single-letter flags do not combine, so the -v help text says -v -v.
|
|
||||||
func TestCombinedShortVerboseRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, command := range commandsTakingVerbose() {
|
|
||||||
args := slices.Concat(command, []string{"-vv"})
|
|
||||||
|
|
||||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
opts := testOpts(args, afero.NewMemMapFs())
|
|
||||||
exitCode, level := runCLIWithLevel(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 1, exitCode)
|
|
||||||
assert.Contains(t, testStderr(t, opts), "flag provided but not defined: -vv")
|
|
||||||
assert.Equal(t, log.InfoLevel, level)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShortAndLongVerboseRefused asserts that -v and --verbose given together
|
|
||||||
// are refused (issue #125): urfave/cli v2 refuses a flag given under two of its
|
|
||||||
// names, and one flag with an alias keeps help and parsing simple.
|
|
||||||
func TestShortAndLongVerboseRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, command := range commandsTakingVerbose() {
|
|
||||||
args := slices.Concat(command, []string{"-v", testFlagVerbose})
|
|
||||||
|
|
||||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
opts := testOpts(args, afero.NewMemMapFs())
|
|
||||||
exitCode, level := runCLIWithLevel(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 1, exitCode)
|
|
||||||
assert.Contains(t, testStderr(t, opts), "Cannot use two forms of the same flag")
|
|
||||||
assert.Equal(t, log.InfoLevel, level)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHelpCommand(t *testing.T) {
|
func TestHelpCommand(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -348,7 +126,7 @@ func TestHelpCommand(t *testing.T) {
|
|||||||
stdout := testStdout(t, opts)
|
stdout := testStdout(t, opts)
|
||||||
assert.Contains(t, stdout, cmdGenerate)
|
assert.Contains(t, stdout, cmdGenerate)
|
||||||
assert.Contains(t, stdout, cmdCheck)
|
assert.Contains(t, stdout, cmdCheck)
|
||||||
assert.Contains(t, stdout, cmdFetch)
|
assert.Contains(t, stdout, "fetch")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateCommand(t *testing.T) {
|
func TestGenerateCommand(t *testing.T) {
|
||||||
@@ -394,67 +172,6 @@ func TestGenerateAndCheckCommand(t *testing.T) {
|
|||||||
assert.Equal(t, 0, exitCode, "check failed: %s", testStderr(t, opts))
|
assert.Equal(t, 0, exitCode, "check failed: %s", testStderr(t, opts))
|
||||||
}
|
}
|
||||||
|
|
||||||
// sharedWriter appends to a buffer shared with other sharedWriters, so
|
|
||||||
// output written to stdout and stderr is kept in the order it was written.
|
|
||||||
// Each write first waits for delay.
|
|
||||||
type sharedWriter struct {
|
|
||||||
mu *sync.Mutex
|
|
||||||
buf *bytes.Buffer
|
|
||||||
delay time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w sharedWriter) Write(p []byte) (int, error) {
|
|
||||||
time.Sleep(w.delay)
|
|
||||||
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
return w.buf.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCheckClearsProgressBeforeSummary asserts that check --progress writes
|
|
||||||
// its last progress line and clears it before it logs the summary. Progress
|
|
||||||
// writes are slowed down, so a progress goroutine that check did not wait for
|
|
||||||
// would write after the summary, or after the run has returned.
|
|
||||||
func TestCheckClearsProgressBeforeSummary(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello world")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testMF, testDir}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "generate failed: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
var (
|
|
||||||
mu sync.Mutex
|
|
||||||
output bytes.Buffer
|
|
||||||
)
|
|
||||||
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "--progress", testFlagBase, testDir, testMF,
|
|
||||||
}, fs)
|
|
||||||
opts.Stdout = sharedWriter{mu: &mu, buf: &output, delay: 100 * time.Millisecond}
|
|
||||||
opts.Stderr = sharedWriter{mu: &mu, buf: &output}
|
|
||||||
require.Equal(t, 0, runCLI(opts))
|
|
||||||
|
|
||||||
mu.Lock()
|
|
||||||
got := output.String()
|
|
||||||
mu.Unlock()
|
|
||||||
|
|
||||||
lastProgress := strings.Index(got, "Checking: 1/1 files")
|
|
||||||
progressDone := strings.Index(got, "\r\033[K")
|
|
||||||
summary := strings.Index(got, "checked 1 files")
|
|
||||||
|
|
||||||
require.NotEqual(t, -1, lastProgress, "no last progress line in %q", got)
|
|
||||||
require.NotEqual(t, -1, progressDone, "progress line never cleared in %q", got)
|
|
||||||
require.NotEqual(t, -1, summary, "no summary in %q", got)
|
|
||||||
assert.Less(t, lastProgress, progressDone,
|
|
||||||
"progress cleared before its last line in %q", got)
|
|
||||||
assert.Less(t, progressDone, summary,
|
|
||||||
"summary logged before progress was cleared in %q", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckCommandWithMissingFile(t *testing.T) {
|
func TestCheckCommandWithMissingFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -566,10 +283,7 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
|
|||||||
assert.True(t, exists)
|
assert.True(t, exists)
|
||||||
|
|
||||||
// Verify manifest only has 1 file (the non-dotfile)
|
// Verify manifest only has 1 file (the non-dotfile)
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(fs, testMF)
|
||||||
Path: testMF,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, manifest.Files(), 1)
|
assert.Len(t, manifest.Files(), 1)
|
||||||
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
|
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
|
||||||
@@ -593,10 +307,7 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
|||||||
require.Equal(t, 0, exitCode)
|
require.Equal(t, 0, exitCode)
|
||||||
|
|
||||||
// Verify manifest has 2 files (including dotfile)
|
// Verify manifest has 2 files (including dotfile)
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(fs, testMF)
|
||||||
Path: testMF,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, manifest.Files(), 2)
|
assert.Len(t, manifest.Files(), 2)
|
||||||
}
|
}
|
||||||
@@ -744,7 +455,7 @@ func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
|||||||
assert.True(t, exists, "output file should exist")
|
assert.True(t, exists, "output file should exist")
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
// Verify temp file does NOT exist
|
||||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should not exist after successful generation")
|
"temp file should not exist after successful generation")
|
||||||
@@ -776,7 +487,7 @@ func TestGenerateAtomicWriteOverwriteWithForce(t *testing.T) {
|
|||||||
"manifest should be overwritten")
|
"manifest should be overwritten")
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
// Verify temp file does NOT exist
|
||||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should not exist after successful generation")
|
"temp file should not exist after successful generation")
|
||||||
@@ -805,102 +516,6 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
|||||||
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
||||||
}
|
}
|
||||||
|
|
||||||
// manifestPaths returns the file paths listed by the manifest at path.
|
|
||||||
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
|
||||||
Path: path,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
paths := make([]string, 0, len(manifest.Files()))
|
|
||||||
for _, f := range manifest.Files() {
|
|
||||||
paths = append(paths, f.GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
|
|
||||||
// scanned tree with --force: the old file is not listed, even when the tree
|
|
||||||
// is named through a symlink, while a file named index.mf in a subdirectory
|
|
||||||
// still is. The output file is recognized by file identity, which needs
|
|
||||||
// the real filesystem.
|
|
||||||
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Paths are relative to a temp dir holding data/tree and link, a
|
|
||||||
// symlink to data.
|
|
||||||
for name, tc := range map[string]struct{ input, output string }{
|
|
||||||
"default name": {"data/tree", "data/tree/index.mf"},
|
|
||||||
"other name in a subdirectory": {"data/tree", "data/tree/sub/listing.mf"},
|
|
||||||
"tree named through a symlink": {"link/tree", "data/tree/index.mf"},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
tree := filepath.Join(root, "data", "tree")
|
|
||||||
output := filepath.Join(root, tc.output)
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
|
|
||||||
require.NoError(t,
|
|
||||||
os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
|
|
||||||
writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
|
|
||||||
writeTestFile(t, fs, filepath.Join(tree, "sub", "index.mf"), "an ordinary file")
|
|
||||||
writeTestFile(t, fs, output, "previous manifest")
|
|
||||||
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "--force", "-o", output, filepath.Join(root, tc.input),
|
|
||||||
}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{testFileTxt, "sub/index.mf"},
|
|
||||||
manifestPaths(t, fs, output))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateDefaultOutputLeftOutOfListing runs gen with --force and no
|
|
||||||
// other arguments, so it scans the current directory and writes the
|
|
||||||
// relative path index.mf: the index.mf already there is not listed.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
|
|
||||||
chdirTemp(t)
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
writeTestFile(t, fs, testFileTxt, "hello")
|
|
||||||
writeTestFile(t, fs, "index.mf", "previous manifest")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "--force"}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, "index.mf"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
|
|
||||||
// interrupted run left its temp file beside the output: the leftover is
|
|
||||||
// not listed, and gen does not fail when it overwrites it.
|
|
||||||
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
output := filepath.Join(root, "index.mf")
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
|
||||||
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -923,7 +538,7 @@ func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
|||||||
exists, _ := afero.Exists(fs, testOutput)
|
exists, _ := afero.Exists(fs, testOutput)
|
||||||
assert.True(t, exists, "output file should exist")
|
assert.True(t, exists, "output file should exist")
|
||||||
|
|
||||||
tmpExists, _ := afero.Exists(fs, manifestTempPath(testOutput))
|
tmpExists, _ := afero.Exists(fs, testOutputTmp)
|
||||||
assert.False(t, tmpExists, "temp file should be cleaned up")
|
assert.False(t, tmpExists, "temp file should be cleaned up")
|
||||||
|
|
||||||
// Verify manifest is valid (not empty)
|
// Verify manifest is valid (not empty)
|
||||||
@@ -989,7 +604,7 @@ func TestGenerateAtomicWriteCleansUpOnError(t *testing.T) {
|
|||||||
"output file should not exist after failed generation (atomic write)")
|
"output file should not exist after failed generation (atomic write)")
|
||||||
|
|
||||||
// Temp file should also not exist
|
// Temp file should also not exist
|
||||||
tmpExists, _ := afero.Exists(baseFs, manifestTempPath(testOutput))
|
tmpExists, _ := afero.Exists(baseFs, testOutputTmp)
|
||||||
assert.False(t, tmpExists,
|
assert.False(t, tmpExists,
|
||||||
"temp file should be cleaned up after failed generation")
|
"temp file should be cleaned up after failed generation")
|
||||||
}
|
}
|
||||||
|
|||||||
+134
-329
@@ -2,362 +2,167 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"fmt"
|
||||||
"context"
|
|
||||||
"flag"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
urfcli "github.com/urfave/cli/v2"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// These tests pin the exact rendered text of the CLI's user-visible error
|
// errMsgCase is one pinned user-visible error message.
|
||||||
// messages. The messages are grepped for in CI pipelines and quoted in bug
|
type errMsgCase struct {
|
||||||
// reports, so a reword is a deliberate change, never a refactoring side
|
name string
|
||||||
// effect.
|
err error
|
||||||
//
|
want string
|
||||||
// Every case drives the real function that emits the message and asserts on
|
}
|
||||||
// what it returns. No production format string is restated here: a test that
|
|
||||||
// only re-rendered a copied format string would keep passing after the real
|
|
||||||
// message changed, which is exactly the regression these tests exist to
|
|
||||||
// catch.
|
|
||||||
|
|
||||||
// Full 40-hex fingerprints used where a message embeds one.
|
|
||||||
const (
|
const (
|
||||||
msgFpA = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
msgFpA = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||||
msgFpB = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
|
msgFpB = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
|
||||||
)
|
)
|
||||||
|
|
||||||
// runLocked runs fn while holding runMu, so operations that write to the
|
func checkErrMsgCases(t *testing.T, cases []errMsgCase) {
|
||||||
// process-global logger do not race the other CLI runs.
|
|
||||||
func runLocked(fn func() error) error {
|
|
||||||
runMu.Lock()
|
|
||||||
defer runMu.Unlock()
|
|
||||||
|
|
||||||
return fn()
|
|
||||||
}
|
|
||||||
|
|
||||||
// unsignedChecker builds a Checker over a freshly scanned, unsigned manifest.
|
|
||||||
func unsignedChecker(t *testing.T) *mfer.Checker {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
for _, tc := range cases {
|
||||||
require.NoError(t, fs.MkdirAll("/d", 0o755))
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
|
t.Parallel()
|
||||||
|
assert.Equal(t, tc.want, tc.err.Error())
|
||||||
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: fs})
|
|
||||||
require.NoError(t, s.EnumeratePath("/d", nil))
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
|
|
||||||
|
|
||||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
|
||||||
ManifestPath: "/d/index.mf",
|
|
||||||
BasePath: "/d",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
}
|
||||||
require.False(t, chk.IsSigned())
|
|
||||||
|
|
||||||
return chk
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNoManifestFoundMessage(t *testing.T) {
|
// TestErrorMessagesVerbatim pins the exact rendered text of the CLI's
|
||||||
t.Parallel()
|
// user-visible error messages.
|
||||||
|
|
||||||
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
|
||||||
require.ErrorIs(t, err, errNoManifestFound)
|
|
||||||
assert.EqualError(t, err,
|
|
||||||
"no manifest found in /tmp/x (looked for index.mf)")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
t.Run("invalid fingerprint length", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := verifyRequiredSigner(context.Background(),
|
|
||||||
unsignedChecker(t), "12345678")
|
|
||||||
require.ErrorIs(t, err, errInvalidFingerprint)
|
|
||||||
assert.EqualError(t, err,
|
|
||||||
"invalid fingerprint: must be exactly 40 hex characters, got 8")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("manifest not signed", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := verifyRequiredSigner(context.Background(),
|
|
||||||
unsignedChecker(t), msgFpA)
|
|
||||||
require.ErrorIs(t, err, errManifestNotSigned)
|
|
||||||
assert.EqualError(t, err,
|
|
||||||
"manifest is not signed, but signature from "+msgFpA+" is required")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSignerMismatchMessage drives verifyRequiredSigner against a real signed
|
|
||||||
// manifest. The embedded fingerprint is whatever the generated key produced,
|
|
||||||
// so it is read back from the checker and substituted into the expected
|
|
||||||
// string; the required signer is a fixed value that cannot match it. Requires
|
|
||||||
// gpg and is skipped where it is absent, as the other signing tests are.
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // signedChecker calls t.Setenv, which bars t.Parallel
|
// These strings are an interface: they are grepped for in CI pipelines
|
||||||
func TestSignerMismatchMessage(t *testing.T) {
|
// and quoted in bug reports. The messages are assembled by wrapping
|
||||||
chk := signedChecker(t)
|
// static sentinels, and it is easy to change what a user sees while
|
||||||
|
// only meaning to make an error matchable with errors.Is - which is
|
||||||
embeddedFP, err := chk.ExtractEmbeddedSigningKeyFP(context.Background())
|
// precisely what happened once already. Any change to a string below is
|
||||||
require.NoError(t, err)
|
// therefore a deliberate, separately stated change, never a side effect
|
||||||
|
// of a refactor.
|
||||||
err = verifyRequiredSigner(context.Background(), chk, msgFpB)
|
func TestErrorMessagesVerbatim(t *testing.T) {
|
||||||
require.ErrorIs(t, err, errSignerMismatch)
|
|
||||||
assert.EqualError(t, err,
|
|
||||||
"embedded signing key fingerprint "+embeddedFP+
|
|
||||||
" does not match required "+msgFpB)
|
|
||||||
}
|
|
||||||
|
|
||||||
// signedChecker builds a Checker over a manifest signed by a throwaway GPG
|
|
||||||
// key generated in a temporary GNUPGHOME.
|
|
||||||
func signedChecker(t *testing.T) *mfer.Checker {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := exec.LookPath("gpg")
|
|
||||||
if err != nil {
|
|
||||||
t.Skip("gpg not installed, skipping signing test")
|
|
||||||
}
|
|
||||||
|
|
||||||
gpgHome := t.TempDir()
|
|
||||||
params := "%no-protection\n" +
|
|
||||||
"Key-Type: RSA\nKey-Length: 2048\n" +
|
|
||||||
"Name-Real: MFER Test Key\nName-Email: test@mfer.test\n" +
|
|
||||||
"Expire-Date: 0\n%commit\n"
|
|
||||||
paramsFile := filepath.Join(gpgHome, "key-params")
|
|
||||||
require.NoError(t, os.WriteFile(paramsFile, []byte(params), 0o600))
|
|
||||||
|
|
||||||
//nolint:gosec // paramsFile is a test-controlled path inside t.TempDir()
|
|
||||||
cmd := exec.CommandContext(context.Background(), "gpg",
|
|
||||||
"--batch", "--gen-key", paramsFile)
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
|
||||||
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Skipf("failed to generate test GPG key: %v: %s", err, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
|
||||||
b.SetSigningOptions(&mfer.SigningOptions{KeyID: mfer.GPGKeyID("test@mfer.test")})
|
|
||||||
|
|
||||||
content := []byte("signed file")
|
|
||||||
_, err = b.AddFile("f.txt", mfer.FileSize(len(content)), mfer.ModTime{},
|
|
||||||
bytes.NewReader(content), nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644))
|
|
||||||
|
|
||||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
|
||||||
ManifestPath: "/index.mf",
|
|
||||||
BasePath: "/",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.True(t, chk.IsSigned())
|
|
||||||
|
|
||||||
return chk
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPathDoesNotExistMessage(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
set := flag.NewFlagSet("gen", flag.ContinueOnError)
|
checkErrMsgCases(t, []errMsgCase{
|
||||||
require.NoError(t, set.Parse([]string{"nope"}))
|
{
|
||||||
|
name: "check: no manifest found",
|
||||||
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
|
err: fmt.Errorf("%w in %s (looked for index.mf and .index.mf)",
|
||||||
ctx := urfcli.NewContext(nil, set, nil)
|
errNoManifestFound, "/tmp/x"),
|
||||||
|
want: "no manifest found in /tmp/x " +
|
||||||
_, err := mfa.collectInputPaths(ctx.Args())
|
"(looked for index.mf and .index.mf)",
|
||||||
require.ErrorIs(t, err, errPathNotExist)
|
},
|
||||||
assert.EqualError(t, err, "path does not exist: nope")
|
{
|
||||||
}
|
name: "check: invalid fingerprint length",
|
||||||
|
err: fmt.Errorf("%w, got %d", errInvalidFingerprint, 8),
|
||||||
func TestOutputFileExistsMessage(t *testing.T) {
|
want: "invalid fingerprint: must be exactly 40 hex characters, got 8",
|
||||||
t.Parallel()
|
},
|
||||||
|
{
|
||||||
fs := afero.NewMemMapFs()
|
name: "check: manifest not signed",
|
||||||
require.NoError(t, fs.MkdirAll("/d", 0o755))
|
err: fmt.Errorf("%w, but signature from %s is required",
|
||||||
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
|
errManifestNotSigned, msgFpA),
|
||||||
require.NoError(t, afero.WriteFile(fs, "/out.mf", []byte("old"), 0o644))
|
want: "manifest is not signed, but signature from " + msgFpA +
|
||||||
|
" is required",
|
||||||
set := flag.NewFlagSet("gen", flag.ContinueOnError)
|
},
|
||||||
set.String("output", "", "")
|
{
|
||||||
set.Bool("force", false, "")
|
name: "check: signer mismatch",
|
||||||
require.NoError(t, set.Parse([]string{"/d"}))
|
err: fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
||||||
require.NoError(t, set.Set("output", "/out.mf"))
|
msgFpA, errSignerMismatch, msgFpB),
|
||||||
|
want: "embedded signing key fingerprint " + msgFpA +
|
||||||
mfa := &CLIApp{Fs: fs}
|
" does not match required " + msgFpB,
|
||||||
ctx := urfcli.NewContext(nil, set, nil)
|
},
|
||||||
|
{
|
||||||
// generateManifestOperation writes to the process-global logger during
|
name: "gen: path does not exist",
|
||||||
// enumeration, so serialize with the other CLI runs.
|
err: fmt.Errorf("%w: %s", errPathNotExist, "nope"),
|
||||||
err := runLocked(func() error { return mfa.generateManifestOperation(ctx) })
|
want: "path does not exist: nope",
|
||||||
require.ErrorIs(t, err, errOutputExists)
|
},
|
||||||
assert.EqualError(t, err,
|
{
|
||||||
"output file /out.mf already exists (use --force to overwrite)")
|
name: "gen: output file exists",
|
||||||
}
|
err: fmt.Errorf("output file %s %w", "index.mf", errOutputExists),
|
||||||
|
want: "output file index.mf already exists " +
|
||||||
// TestUnknownCommandMessage drives the root command's action. run only logs
|
"(use --force to overwrite)",
|
||||||
// the error that action returns, so the test lets run build the app with no
|
},
|
||||||
// command given and then runs that same app on an unknown command to get the
|
{
|
||||||
// error itself.
|
name: "mfer: unknown command",
|
||||||
func TestUnknownCommandMessage(t *testing.T) {
|
err: fmt.Errorf("%w %q", errUnknownCommand, "bogus"),
|
||||||
t.Parallel()
|
want: `unknown command "bogus"`,
|
||||||
|
},
|
||||||
mfa := &CLIApp{
|
|
||||||
appname: testApp,
|
|
||||||
Stdout: &bytes.Buffer{},
|
|
||||||
Stderr: &bytes.Buffer{},
|
|
||||||
Fs: afero.NewMemMapFs(),
|
|
||||||
}
|
|
||||||
|
|
||||||
// run points the process-global logger at this app's output, so
|
|
||||||
// serialize with the other CLI runs.
|
|
||||||
err := runLocked(func() error {
|
|
||||||
mfa.run([]string{testApp})
|
|
||||||
|
|
||||||
return mfa.app.Run([]string{testApp, "bogus"})
|
|
||||||
})
|
|
||||||
require.ErrorIs(t, err, errUnknownCommand)
|
|
||||||
assert.EqualError(t, err, `unknown command "bogus"`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestLoaderHTTPStatusMessage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
|
|
||||||
|
|
||||||
_, err := mfa.openManifestReader(server.URL + "/foo.mf")
|
|
||||||
require.ErrorIs(t, err, errHTTPStatus)
|
|
||||||
assert.EqualError(t, err,
|
|
||||||
"failed to fetch "+server.URL+"/foo.mf: HTTP 404")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetchManifestHTTPStatusMessage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
set := flag.NewFlagSet("fetch", flag.ContinueOnError)
|
|
||||||
require.NoError(t, set.Parse([]string{server.URL}))
|
|
||||||
|
|
||||||
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
|
|
||||||
ctx := urfcli.NewContext(nil, set, nil)
|
|
||||||
|
|
||||||
// fetchManifestOperation logs to the process-global logger.
|
|
||||||
err := runLocked(func() error { return mfa.fetchManifestOperation(ctx) })
|
|
||||||
require.ErrorIs(t, err, errHTTPStatus)
|
|
||||||
assert.EqualError(t, err, "failed to fetch manifest: HTTP 404")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFetchFileHTTPStatusMessage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
err := downloadFile(context.Background(), server.URL+"/x", "x",
|
|
||||||
&mfer.MFFilePath{}, nil)
|
|
||||||
require.ErrorIs(t, err, errHTTPStatus)
|
|
||||||
assert.EqualError(t, err, "HTTP 500")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestURLRequiredMessage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
set := flag.NewFlagSet("fetch", flag.ContinueOnError)
|
|
||||||
require.NoError(t, set.Parse([]string{}))
|
|
||||||
|
|
||||||
mfa := &CLIApp{Fs: afero.NewMemMapFs()}
|
|
||||||
ctx := urfcli.NewContext(nil, set, nil)
|
|
||||||
|
|
||||||
// fetchManifestOperation logs to the process-global logger.
|
|
||||||
err := runLocked(func() error { return mfa.fetchManifestOperation(ctx) })
|
|
||||||
require.ErrorIs(t, err, errURLRequired)
|
|
||||||
assert.EqualError(t, err, "URL argument required")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSanitizePathMessages(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
t.Run("empty", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := sanitizePath("")
|
|
||||||
require.ErrorIs(t, err, errEmptyPath)
|
|
||||||
assert.EqualError(t, err, "empty path")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("absolute", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := sanitizePath("/etc/passwd")
|
|
||||||
require.ErrorIs(t, err, errAbsolutePath)
|
|
||||||
assert.EqualError(t, err, "absolute path not allowed: /etc/passwd")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("traversal", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := sanitizePath("../x")
|
|
||||||
require.ErrorIs(t, err, errPathTraversal)
|
|
||||||
assert.EqualError(t, err, "path traversal not allowed: ../x")
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSizeMismatchMessage(t *testing.T) {
|
// TestFetchErrorMessagesVerbatim pins the fetch and manifest-loader
|
||||||
|
// messages; see TestErrorMessagesVerbatim for why.
|
||||||
|
func TestFetchErrorMessagesVerbatim(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// finishDownload returns the size-mismatch error before it touches the
|
checkErrMsgCases(t, []errMsgCase{
|
||||||
// paths, digest, or entry, so those can be zero here.
|
{
|
||||||
err := finishDownload("", "", 9, 10, nil, nil, nil, nil)
|
name: "manifest_loader: http status",
|
||||||
require.ErrorIs(t, err, errSizeMismatch)
|
err: fmt.Errorf("failed to fetch %s: %w %d",
|
||||||
assert.EqualError(t, err, "size mismatch: expected 10 bytes, got 9")
|
"https://example.com/index.mf", errHTTPStatus, 404),
|
||||||
|
want: "failed to fetch https://example.com/index.mf: HTTP 404",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: manifest http status",
|
||||||
|
err: fmt.Errorf("failed to fetch manifest: %w %d",
|
||||||
|
errHTTPStatus, 404),
|
||||||
|
want: "failed to fetch manifest: HTTP 404",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: file http status",
|
||||||
|
err: fmt.Errorf("%w %d", errHTTPStatus, 500),
|
||||||
|
want: "HTTP 500",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: empty path",
|
||||||
|
err: errEmptyPath,
|
||||||
|
want: "empty path",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: absolute path",
|
||||||
|
err: fmt.Errorf("%w: %s", errAbsolutePath, "/etc/passwd"),
|
||||||
|
want: "absolute path not allowed: /etc/passwd",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: path traversal",
|
||||||
|
err: fmt.Errorf("%w: %s", errPathTraversal, "../x"),
|
||||||
|
want: "path traversal not allowed: ../x",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: size mismatch",
|
||||||
|
err: fmt.Errorf("%w: expected %d bytes, got %d",
|
||||||
|
errSizeMismatch, 10, 9),
|
||||||
|
want: "size mismatch: expected 10 bytes, got 9",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: url required",
|
||||||
|
err: errURLRequired,
|
||||||
|
want: "URL argument required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fetch: hash mismatch",
|
||||||
|
err: errHashMismatch,
|
||||||
|
want: "hash mismatch",
|
||||||
|
},
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHashMismatchMessage(t *testing.T) {
|
// TestSentinelsAreMatchable checks that the wrapped forms of the
|
||||||
|
// messages above remain matchable with errors.Is, which is the reason
|
||||||
|
// the sentinels exist at all.
|
||||||
|
func TestSentinelsAreMatchable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// A 32-byte digest that matches none of the (empty) manifest hashes.
|
wrapped := fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
||||||
err := verifyDownloadedHash(make([]byte, 32), &mfer.MFFilePath{})
|
"a", errSignerMismatch, "b")
|
||||||
require.ErrorIs(t, err, errHashMismatch)
|
require.ErrorIs(t, wrapped, errSignerMismatch)
|
||||||
require.NotErrorIs(t, err, errSizeMismatch)
|
|
||||||
assert.EqualError(t, err, "hash mismatch")
|
wrapped = fmt.Errorf("output file %s %w", "index.mf", errOutputExists)
|
||||||
|
require.ErrorIs(t, wrapped, errOutputExists)
|
||||||
|
|
||||||
|
wrapped = fmt.Errorf("failed to fetch manifest: %w %d", errHTTPStatus, 404)
|
||||||
|
require.ErrorIs(t, wrapped, errHTTPStatus)
|
||||||
|
|
||||||
|
assert.NotErrorIs(t, errHashMismatch, errSizeMismatch)
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-17
@@ -36,11 +36,6 @@ const (
|
|||||||
// traversal bit for group and other must stay set.
|
// traversal bit for group and other must stay set.
|
||||||
dirPerms os.FileMode = 0o755
|
dirPerms os.FileMode = 0o755
|
||||||
|
|
||||||
// filePerms is the permission mode, before the umask, for downloaded
|
|
||||||
// files. It is the mode os.Create uses; like dirPerms, it keeps group
|
|
||||||
// and other read access.
|
|
||||||
filePerms os.FileMode = 0o666
|
|
||||||
|
|
||||||
// Bitrate unit thresholds in bits per second.
|
// Bitrate unit thresholds in bits per second.
|
||||||
bpsPerGbps = 1e9
|
bpsPerGbps = 1e9
|
||||||
bpsPerMbps = 1e6
|
bpsPerMbps = 1e6
|
||||||
@@ -313,7 +308,7 @@ func checkNoSymlinks(p string) error {
|
|||||||
|
|
||||||
// resolveManifestURL takes a URL and returns the manifest URL.
|
// resolveManifestURL takes a URL and returns the manifest URL.
|
||||||
// If the URL already ends with .mf, it's returned as-is.
|
// If the URL already ends with .mf, it's returned as-is.
|
||||||
// Otherwise, the default manifest name is appended.
|
// Otherwise, index.mf is appended.
|
||||||
func resolveManifestURL(inputURL string) (string, error) {
|
func resolveManifestURL(inputURL string) (string, error) {
|
||||||
parsed, err := url.Parse(inputURL)
|
parsed, err := url.Parse(inputURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -330,7 +325,8 @@ func resolveManifestURL(inputURL string) (string, error) {
|
|||||||
parsed.Path += "/"
|
parsed.Path += "/"
|
||||||
}
|
}
|
||||||
|
|
||||||
parsed.Path += defaultManifestName
|
// Append index.mf
|
||||||
|
parsed.Path += "index.mf"
|
||||||
|
|
||||||
return parsed.String(), nil
|
return parsed.String(), nil
|
||||||
}
|
}
|
||||||
@@ -493,20 +489,12 @@ func downloadFile(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove whatever is at tmpPath, such as a leftover from an
|
// Create temp file.
|
||||||
// interrupted run, rather than write into it: it may be a hard link
|
|
||||||
// to a file outside the target directory, and removing a hard link
|
|
||||||
// removes only this name. If the removal fails, O_EXCL below makes
|
|
||||||
// the create fail.
|
|
||||||
_ = os.Remove(tmpPath)
|
|
||||||
|
|
||||||
// Create the temp file only if nothing is at tmpPath (O_EXCL).
|
|
||||||
//
|
//
|
||||||
// G304: tmpPath is a relative path that sanitizePath keeps inside the
|
// G304: tmpPath is a relative path that sanitizePath keeps inside the
|
||||||
// target directory as text, and checkNoSymlinks just found no symlink
|
// target directory as text, and checkNoSymlinks just found no symlink
|
||||||
// in it.
|
// in it.
|
||||||
out, err := os.OpenFile( //nolint:gosec // G304: see comment above
|
out, err := os.Create(tmpPath) //nolint:gosec // G304: see comment above
|
||||||
tmpPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, filePerms)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create temp file: %w", err)
|
return fmt.Errorf("failed to create temp file: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -478,7 +478,7 @@ func TestFetchRefusesSymlinks(t *testing.T) {
|
|||||||
require.NoError(t, os.MkdirAll(filepath.Dir(tt.link), 0o750))
|
require.NoError(t, os.MkdirAll(filepath.Dir(tt.link), 0o750))
|
||||||
require.NoError(t, os.Symlink(filepath.Join(outside, tt.target), tt.link))
|
require.NoError(t, os.Symlink(filepath.Join(outside, tt.target), tt.link))
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
|
opts := testOpts([]string{testApp, "fetch", "-q", server.URL}, afero.NewOsFs())
|
||||||
assert.Equal(t, 1, runCLI(opts))
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
assert.Contains(t, testStderr(t, opts), "failed to download "+tt.entry+
|
assert.Contains(t, testStderr(t, opts), "failed to download "+tt.entry+
|
||||||
": symlink in path not allowed: "+tt.link)
|
": symlink in path not allowed: "+tt.link)
|
||||||
@@ -489,37 +489,3 @@ func TestFetchRefusesSymlinks(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFetchReplacesHardLinkAtTempName runs fetch into a destination
|
|
||||||
// directory that holds, at the temp file's name, a hard link to a file
|
|
||||||
// outside it. To fetch that is an ordinary leftover from an interrupted
|
|
||||||
// earlier run: it must replace it and succeed, and the outside file must
|
|
||||||
// not change.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestFetchReplacesHardLinkAtTempName(t *testing.T) {
|
|
||||||
content := []byte("fetched")
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, "/"+testFileTxt, content, 0o644))
|
|
||||||
|
|
||||||
server := httptest.NewServer(fetchTestHandler(
|
|
||||||
scanToManifest(t, sourceFs), map[string][]byte{testFileTxt: content}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
outsideFile := filepath.Join(t.TempDir(), "secret.txt")
|
|
||||||
require.NoError(t, os.WriteFile(outsideFile, []byte("outside"), 0o600))
|
|
||||||
|
|
||||||
chdirTemp(t)
|
|
||||||
require.NoError(t, os.Link(outsideFile, ".file.txt.tmp"))
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
|
|
||||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
|
||||||
|
|
||||||
fetched, err := os.ReadFile(testFileTxt)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, content, fetched)
|
|
||||||
|
|
||||||
outside, err := os.ReadFile(outsideFile) //nolint:gosec // test-controlled path
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "outside", string(outside), "fetch wrote outside the destination")
|
|
||||||
}
|
|
||||||
|
|||||||
+13
-39
@@ -1,13 +1,11 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -57,7 +55,7 @@ type freshenEntry struct {
|
|||||||
type freshenScanner struct {
|
type freshenScanner struct {
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
absBase string
|
absBase string
|
||||||
excluded []fs.FileInfo // files left out of the listing
|
manifestBase string
|
||||||
includeDotfiles bool
|
includeDotfiles bool
|
||||||
followSymlinks bool
|
followSymlinks bool
|
||||||
showProgress bool
|
showProgress bool
|
||||||
@@ -157,6 +155,11 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
|||||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
"freshen: failed to compute relative path for %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Skip the manifest file itself
|
||||||
|
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Handle dotfiles
|
// Handle dotfiles
|
||||||
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
@@ -181,12 +184,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
|||||||
info = realInfo
|
info = realInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, excluded := range s.excluded {
|
|
||||||
if os.SameFile(info, excluded) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s.scanCount++
|
s.scanCount++
|
||||||
|
|
||||||
// Check against existing manifest
|
// Check against existing manifest
|
||||||
@@ -307,16 +304,16 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
|||||||
// writeFreshenedManifest writes the manifest atomically (write to a
|
// writeFreshenedManifest writes the manifest atomically (write to a
|
||||||
// temp file, then rename over the target).
|
// temp file, then rename over the target).
|
||||||
func writeFreshenedManifest(
|
func writeFreshenedManifest(
|
||||||
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
||||||
) error {
|
) error {
|
||||||
tmpPath := manifestTempPath(manifestPath)
|
tmpPath := manifestPath + ".tmp"
|
||||||
|
|
||||||
outFile, err := afs.Create(tmpPath)
|
outFile, err := afs.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create temp file: %w", err)
|
return fmt.Errorf("failed to create temp file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = builder.Build(ctx, outFile)
|
err = builder.Build(outFile)
|
||||||
_ = outFile.Close()
|
_ = outFile.Close()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -366,22 +363,10 @@ func (mfa *CLIApp) freshenScan(
|
|||||||
startScan := time.Now()
|
startScan := time.Now()
|
||||||
showProgress := ctx.Bool("progress")
|
showProgress := ctx.Bool("progress")
|
||||||
|
|
||||||
// Leave out the manifest and a temp file left by an interrupted run,
|
|
||||||
// as gen does. A path that cannot be stat'd, normally because no file
|
|
||||||
// is there, needs no leaving out.
|
|
||||||
var excluded []fs.FileInfo
|
|
||||||
|
|
||||||
for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
|
|
||||||
info, err := mfa.Fs.Stat(p)
|
|
||||||
if err == nil {
|
|
||||||
excluded = append(excluded, info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
scanner := &freshenScanner{
|
scanner := &freshenScanner{
|
||||||
fs: mfa.Fs,
|
fs: mfa.Fs,
|
||||||
absBase: absBase,
|
absBase: absBase,
|
||||||
excluded: excluded,
|
manifestBase: filepath.Base(manifestPath),
|
||||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
includeDotfiles: ctx.Bool("include-dotfiles"),
|
||||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
followSymlinks: ctx.Bool("follow-symlinks"),
|
||||||
showProgress: showProgress,
|
showProgress: showProgress,
|
||||||
@@ -459,10 +444,7 @@ func (mfa *CLIApp) loadExistingEntries(
|
|||||||
log.Infof("loading manifest from %s", manifestPath)
|
log.Infof("loading manifest from %s", manifestPath)
|
||||||
|
|
||||||
// Load existing manifest
|
// Load existing manifest
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
|
||||||
Path: manifestPath,
|
|
||||||
Fs: mfa.Fs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to load manifest: %w", err)
|
return nil, fmt.Errorf("failed to load manifest: %w", err)
|
||||||
}
|
}
|
||||||
@@ -548,7 +530,7 @@ func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Write updated manifest atomically (write to temp, then rename)
|
// Write updated manifest atomically (write to temp, then rename)
|
||||||
err = writeFreshenedManifest(ctx.Context, mfa.Fs, hasher.builder, manifestPath)
|
err = writeFreshenedManifest(mfa.Fs, hasher.builder, manifestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -629,15 +611,7 @@ func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
return b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||||
entry.GetHashes()[0].GetMultiHash())
|
entry.GetHashes()[0].GetMultiHash())
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
|
||||||
entry.GetPath(), err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,13 +4,10 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -32,7 +29,7 @@ func (s stubFileInfo) IsDir() bool { return false }
|
|||||||
func (s stubFileInfo) Sys() any { return nil }
|
func (s stubFileInfo) Sys() any { return nil }
|
||||||
|
|
||||||
// setupFreshenDir populates /testdir with two files, scans it, and
|
// setupFreshenDir populates /testdir with two files, scans it, and
|
||||||
// writes the resulting manifest to /testdir/index.mf.
|
// writes the resulting manifest to /testdir/.index.mf.
|
||||||
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -51,7 +48,7 @@ func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
|||||||
|
|
||||||
// Write manifest to filesystem
|
// Write manifest to filesystem
|
||||||
require.NoError(t,
|
require.NoError(t,
|
||||||
afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644))
|
afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFreshenUnchanged(t *testing.T) {
|
func TestFreshenUnchanged(t *testing.T) {
|
||||||
@@ -61,10 +58,7 @@ func TestFreshenUnchanged(t *testing.T) {
|
|||||||
setupFreshenDir(t, fs)
|
setupFreshenDir(t, fs)
|
||||||
|
|
||||||
// Parse manifest to verify
|
// Parse manifest to verify
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
|
||||||
Path: "/testdir/index.mf",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, manifest.Files(), 2)
|
assert.Len(t, manifest.Files(), 2)
|
||||||
}
|
}
|
||||||
@@ -76,10 +70,7 @@ func TestFreshenWithChanges(t *testing.T) {
|
|||||||
setupFreshenDir(t, fs)
|
setupFreshenDir(t, fs)
|
||||||
|
|
||||||
// Verify initial manifest has 2 files
|
// Verify initial manifest has 2 files
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
|
||||||
Path: "/testdir/index.mf",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Len(t, manifest.Files(), 2)
|
assert.Len(t, manifest.Files(), 2)
|
||||||
|
|
||||||
@@ -104,64 +95,6 @@ func TestFreshenWithChanges(t *testing.T) {
|
|||||||
assert.Equal(t, "modified content2", string(content))
|
assert.Equal(t, "modified content2", string(content))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
|
||||||
// subdirectory of the tree it lists: the manifest is not listed, while an
|
|
||||||
// ordinary file of the same name at the top of the tree is. The manifest
|
|
||||||
// is recognized by file identity, which needs the real filesystem.
|
|
||||||
func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
manifestPath := filepath.Join(root, "sub", "listing.mf")
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
require.NoError(t, fs.MkdirAll(filepath.Join(root, "sub"), 0o750))
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, "listing.mf"), "an ordinary file")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
// A new file gives freshen something to write.
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
|
||||||
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
|
||||||
}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{testFileTxt, "listing.mf", "added.txt"},
|
|
||||||
manifestPaths(t, fs, manifestPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
|
|
||||||
// an interrupted run left its temp file beside it: the leftover is not
|
|
||||||
// listed.
|
|
||||||
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := t.TempDir()
|
|
||||||
manifestPath := filepath.Join(root, "index.mf")
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
writeTestFile(t, fs, manifestTempPath(manifestPath), "part of a manifest")
|
|
||||||
// A new file gives freshen something to write.
|
|
||||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
|
||||||
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
|
||||||
}, fs)
|
|
||||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{testFileTxt, "added.txt"},
|
|
||||||
manifestPaths(t, fs, manifestPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
||||||
// with respect to MFFilePath.Mtime, which is a message pointer with
|
// with respect to MFFilePath.Mtime, which is a message pointer with
|
||||||
// proto3 field presence and may legitimately be absent.
|
// proto3 field presence and may legitimately be absent.
|
||||||
@@ -177,10 +110,7 @@ func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
|||||||
|
|
||||||
const relPath = "file1.txt"
|
const relPath = "file1.txt"
|
||||||
|
|
||||||
// The scanned file's mtime is the Unix epoch. If recordEntry ever misreads
|
mtime := time.Unix(1_700_000_000, 0)
|
||||||
// an absent manifest mtime as the epoch, the "absent" case below would
|
|
||||||
// compare equal to this and be classified unchanged, so the test fails.
|
|
||||||
mtime := time.Unix(0, 0)
|
|
||||||
info := stubFileInfo{size: 8, mtime: mtime}
|
info := stubFileInfo{size: 8, mtime: mtime}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
@@ -236,50 +166,21 @@ func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
|||||||
func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
b := mfer.NewBuilder()
|
||||||
entry := &mfer.MFFilePath{
|
entry := &mfer.MFFilePath{
|
||||||
Path: "file1.txt",
|
Path: "file1.txt",
|
||||||
Size: 8,
|
Size: 8,
|
||||||
Mtime: nil,
|
Mtime: nil,
|
||||||
Hashes: []*mfer.MFFileChecksum{
|
Hashes: []*mfer.MFFileChecksum{
|
||||||
{MultiHash: hash},
|
{MultiHash: []byte{0x12, 0x20}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err = addExistingToBuilder(b, entry)
|
err := addExistingToBuilder(b, entry)
|
||||||
require.ErrorIs(t, err, errEntryMissingMtime)
|
require.ErrorIs(t, err, errEntryMissingMtime)
|
||||||
assert.Contains(t, err.Error(), "file1.txt")
|
assert.Contains(t, err.Error(), "file1.txt")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
|
|
||||||
// entry whose hash the builder refuses is reported as a problem with the
|
|
||||||
// manifest, with the command that fixes it.
|
|
||||||
func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
|
||||||
entry := &mfer.MFFilePath{
|
|
||||||
Path: "old.txt",
|
|
||||||
Size: 8,
|
|
||||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
|
||||||
Hashes: []*mfer.MFFileChecksum{
|
|
||||||
{MultiHash: sha1Hash},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
err = addExistingToBuilder(b, entry)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "manifest entry old.txt")
|
|
||||||
assert.Contains(t, err.Error(), "mfer generate")
|
|
||||||
assert.Zero(t, b.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntryMtime pins the presence semantics the callers depend on.
|
// TestEntryMtime pins the presence semantics the callers depend on.
|
||||||
func TestEntryMtime(t *testing.T) {
|
func TestEntryMtime(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
+1
-5
@@ -89,15 +89,11 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
|||||||
|
|
||||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
||||||
output := ctx.String("output")
|
|
||||||
opts := &mfer.ScannerOptions{
|
opts := &mfer.ScannerOptions{
|
||||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
||||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
||||||
Fs: mfa.Fs,
|
Fs: mfa.Fs,
|
||||||
// Neither a manifest being replaced nor a temp file left by an
|
|
||||||
// interrupted run belongs in the new manifest.
|
|
||||||
ExcludePaths: []string{output, manifestTempPath(output)},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set seed for deterministic UUID if provided
|
// Set seed for deterministic UUID if provided
|
||||||
@@ -221,7 +217,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create temp file for atomic write
|
// Create temp file for atomic write
|
||||||
tmpPath := manifestTempPath(outputPath)
|
tmpPath := outputPath + ".tmp"
|
||||||
|
|
||||||
outFile, err := mfa.Fs.Create(tmpPath)
|
outFile, err := mfa.Fs.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// resolveManifestArg resolves the manifest path from CLI arguments.
|
// resolveManifestArg resolves the manifest path from CLI arguments.
|
||||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
|
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
|
||||||
// If no argument is given, the current directory is searched.
|
// If no argument is given, the current directory is searched.
|
||||||
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
||||||
if ctx.Args().Len() > 0 {
|
if ctx.Args().Len() > 0 {
|
||||||
|
|||||||
+9
-64
@@ -17,27 +17,13 @@ import (
|
|||||||
const (
|
const (
|
||||||
cmdGenerate = "generate"
|
cmdGenerate = "generate"
|
||||||
cmdCheck = "check"
|
cmdCheck = "check"
|
||||||
cmdFreshen = "freshen"
|
|
||||||
cmdExport = "export"
|
cmdExport = "export"
|
||||||
cmdFetch = "fetch"
|
|
||||||
cmdVersion = "version"
|
|
||||||
|
|
||||||
flagProgress = "progress"
|
flagProgress = "progress"
|
||||||
|
|
||||||
manifestArgsUsage = "[manifest file]"
|
manifestArgsUsage = "[manifest file]"
|
||||||
|
|
||||||
// defaultManifestName is the filename gen writes by default, the one
|
|
||||||
// looked for when a command is given a directory, and the one fetch
|
|
||||||
// appends to a directory URL.
|
|
||||||
defaultManifestName = "index.mf"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// manifestTempPath returns the temp file gen and freshen write a manifest
|
|
||||||
// to before renaming it to out.
|
|
||||||
func manifestTempPath(out string) string {
|
|
||||||
return out + ".tmp"
|
|
||||||
}
|
|
||||||
|
|
||||||
// errUnknownCommand indicates an unrecognized command argument.
|
// errUnknownCommand indicates an unrecognized command argument.
|
||||||
var errUnknownCommand = errors.New("unknown command")
|
var errUnknownCommand = errors.New("unknown command")
|
||||||
|
|
||||||
@@ -81,12 +67,6 @@ func (mfa *CLIApp) VersionString() string {
|
|||||||
return mfer.Version
|
return mfer.Version
|
||||||
}
|
}
|
||||||
|
|
||||||
// printVersion writes the version line shared by the --version flag and the
|
|
||||||
// version subcommand, so both produce identical output.
|
|
||||||
func (mfa *CLIApp) printVersion() {
|
|
||||||
_, _ = fmt.Fprintf(mfa.Stdout, "%s version %s\n", mfa.appname, mfa.VersionString())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) printBanner() {
|
func (mfa *CLIApp) printBanner() {
|
||||||
if log.GetLevel() <= log.InfoLevel {
|
if log.GetLevel() <= log.InfoLevel {
|
||||||
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
||||||
@@ -97,40 +77,26 @@ func (mfa *CLIApp) printBanner() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// setVerbosity sets the log level from -v and -q, given before the subcommand
|
|
||||||
// name (the root's copies), after it (the subcommand's own copies), or both.
|
|
||||||
// urfave/cli reads a flag from the nearest command that defines it, so each
|
|
||||||
// command in the lineage is asked. The highest -v count wins rather than the
|
|
||||||
// sum, because a subcommand without its own copies reads the root's.
|
|
||||||
func (mfa *CLIApp) setVerbosity(c *cli.Context) {
|
func (mfa *CLIApp) setVerbosity(c *cli.Context) {
|
||||||
_, present := os.LookupEnv("MFER_DEBUG")
|
_, present := os.LookupEnv("MFER_DEBUG")
|
||||||
|
|
||||||
verbosity := 0
|
|
||||||
quiet := false
|
|
||||||
|
|
||||||
for _, ctx := range c.Lineage() {
|
|
||||||
verbosity = max(verbosity, ctx.Count("verbose"))
|
|
||||||
quiet = quiet || ctx.Bool("quiet")
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case present:
|
case present:
|
||||||
log.EnableDebugLogging()
|
log.EnableDebugLogging()
|
||||||
case quiet:
|
case c.Bool("quiet"):
|
||||||
log.SetLevel(log.ErrorLevel)
|
log.SetLevel(log.ErrorLevel)
|
||||||
default:
|
default:
|
||||||
log.SetLevelFromVerbosity(verbosity)
|
log.SetLevelFromVerbosity(c.Count("verbose"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// commonFlags returns the -v and -q flags taken by the root and by the
|
// commonFlags returns the flags shared by most commands (-v, -q)
|
||||||
// generate, check, freshen and fetch subcommands.
|
|
||||||
func commonFlags() []cli.Flag {
|
func commonFlags() []cli.Flag {
|
||||||
return []cli.Flag{
|
return []cli.Flag{
|
||||||
&cli.BoolFlag{
|
&cli.BoolFlag{
|
||||||
Name: "verbose",
|
Name: "verbose",
|
||||||
Aliases: []string{"v"},
|
Aliases: []string{"v"},
|
||||||
Usage: "Increase verbosity (-v for verbose, -v -v for debug)",
|
Usage: "Increase verbosity (-v for verbose, -vv for debug)",
|
||||||
Count: new(int),
|
Count: new(int),
|
||||||
},
|
},
|
||||||
&cli.BoolFlag{
|
&cli.BoolFlag{
|
||||||
@@ -146,7 +112,6 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
|||||||
Name: cmdGenerate,
|
Name: cmdGenerate,
|
||||||
Aliases: []string{"gen"},
|
Aliases: []string{"gen"},
|
||||||
Usage: "Generate manifest file",
|
Usage: "Generate manifest file",
|
||||||
ArgsUsage: "[path ...]",
|
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
mfa.setVerbosity(c)
|
mfa.setVerbosity(c)
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
@@ -167,7 +132,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
|||||||
},
|
},
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
Value: defaultManifestName,
|
Value: "./.index.mf",
|
||||||
Aliases: []string{"o"},
|
Aliases: []string{"o"},
|
||||||
Usage: "Specify output filename",
|
Usage: "Specify output filename",
|
||||||
},
|
},
|
||||||
@@ -240,7 +205,7 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdFreshen,
|
Name: "freshen",
|
||||||
Usage: "Update manifest with changed, new, and removed files",
|
Usage: "Update manifest with changed, new, and removed files",
|
||||||
ArgsUsage: manifestArgsUsage,
|
ArgsUsage: manifestArgsUsage,
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
@@ -293,8 +258,6 @@ func (mfa *CLIApp) exportCommand() *cli.Command {
|
|||||||
Usage: "Export manifest contents as JSON",
|
Usage: "Export manifest contents as JSON",
|
||||||
ArgsUsage: "[manifest file or URL]",
|
ArgsUsage: "[manifest file or URL]",
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
mfa.setVerbosity(c)
|
|
||||||
|
|
||||||
return mfa.exportManifestOperation(c)
|
return mfa.exportManifestOperation(c)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -302,10 +265,10 @@ func (mfa *CLIApp) exportCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) versionCommand() *cli.Command {
|
func (mfa *CLIApp) versionCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdVersion,
|
Name: "version",
|
||||||
Usage: "Show version",
|
Usage: "Show version",
|
||||||
Action: func(_ *cli.Context) error {
|
Action: func(_ *cli.Context) error {
|
||||||
mfa.printVersion()
|
_, _ = fmt.Fprintln(mfa.Stdout, mfa.VersionString())
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
@@ -337,9 +300,8 @@ func (mfa *CLIApp) listCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdFetch,
|
Name: "fetch",
|
||||||
Usage: "fetch manifest and referenced files",
|
Usage: "fetch manifest and referenced files",
|
||||||
ArgsUsage: "URL",
|
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
mfa.setVerbosity(c)
|
mfa.setVerbosity(c)
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
@@ -362,21 +324,6 @@ func (mfa *CLIApp) run(args []string) {
|
|||||||
log.SetOutput(mfa.Stdout, mfa.Stderr)
|
log.SetOutput(mfa.Stdout, mfa.Stderr)
|
||||||
log.Init()
|
log.Init()
|
||||||
|
|
||||||
// -v means verbose, not version. urfave/cli's built-in version flag
|
|
||||||
// claims -v by default, which made "mfer -v --version" fail to parse and
|
|
||||||
// gave -v a different meaning at the root than on the generate, check,
|
|
||||||
// freshen and fetch subcommands, where it means verbose. Verbose is the
|
|
||||||
// more common meaning of -v in tools that offer both, so -v means verbose
|
|
||||||
// at the root too and the version flag takes the capital -V.
|
|
||||||
// VersionFlag and VersionPrinter are urfave/cli package globals; run() is
|
|
||||||
// serialized in tests, so assigning them here is safe.
|
|
||||||
cli.VersionFlag = &cli.BoolFlag{
|
|
||||||
Name: cmdVersion,
|
|
||||||
Aliases: []string{"V"},
|
|
||||||
Usage: "print the version",
|
|
||||||
}
|
|
||||||
cli.VersionPrinter = func(_ *cli.Context) { mfa.printVersion() }
|
|
||||||
|
|
||||||
mfa.app = &cli.App{
|
mfa.app = &cli.App{
|
||||||
Name: mfa.appname,
|
Name: mfa.appname,
|
||||||
Usage: "Manifest generator",
|
Usage: "Manifest generator",
|
||||||
@@ -384,13 +331,11 @@ func (mfa *CLIApp) run(args []string) {
|
|||||||
EnableBashCompletion: true,
|
EnableBashCompletion: true,
|
||||||
Writer: mfa.Stdout,
|
Writer: mfa.Stdout,
|
||||||
ErrWriter: mfa.Stderr,
|
ErrWriter: mfa.Stderr,
|
||||||
Flags: commonFlags(),
|
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
if c.Args().Len() > 0 {
|
if c.Args().Len() > 0 {
|
||||||
return fmt.Errorf("%w %q", errUnknownCommand, c.Args().First())
|
return fmt.Errorf("%w %q", errUnknownCommand, c.Args().First())
|
||||||
}
|
}
|
||||||
|
|
||||||
mfa.setVerbosity(c)
|
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
|
|
||||||
return cli.ShowAppHelp(c)
|
return cli.ShowAppHelp(c)
|
||||||
|
|||||||
+8
-23
@@ -3,7 +3,6 @@
|
|||||||
package mfer
|
package mfer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -35,8 +34,7 @@ var (
|
|||||||
errPathDotDot = errors.New("contains '..' segment")
|
errPathDotDot = errors.New("contains '..' segment")
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
errSizeMismatch = errors.New("size mismatch")
|
||||||
errNegativeSize = errors.New("size cannot be negative")
|
errNegativeSize = errors.New("size cannot be negative")
|
||||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
errEmptyHash = errors.New("hash cannot be nil or empty")
|
||||||
errHashTooShort = errors.New("hash digest is too short")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ValidatePath checks that a file path conforms to manifest path invariants:
|
// ValidatePath checks that a file path conforms to manifest path invariants:
|
||||||
@@ -229,8 +227,7 @@ func (b *Builder) FileCount() int {
|
|||||||
|
|
||||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
// AddFileWithHash adds a file entry with a pre-computed hash.
|
||||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||||
// Returns an error if path is invalid, size is negative, or hash is not a
|
// Returns an error if path is empty, size is negative, or hash is nil/empty.
|
||||||
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
|
|
||||||
func (b *Builder) AddFileWithHash(
|
func (b *Builder) AddFileWithHash(
|
||||||
path RelFilePath,
|
path RelFilePath,
|
||||||
size FileSize,
|
size FileSize,
|
||||||
@@ -246,19 +243,8 @@ func (b *Builder) AddFileWithHash(
|
|||||||
return errNegativeSize
|
return errNegativeSize
|
||||||
}
|
}
|
||||||
|
|
||||||
decoded, err := multihash.Decode(hash)
|
if len(hash) == 0 {
|
||||||
if err != nil {
|
return errEmptyHash
|
||||||
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
|
|
||||||
// hash is at least as long as a SHA-256 multihash, so a manifest of
|
|
||||||
// shorter ones could fail to load.
|
|
||||||
if len(decoded.Digest) < sha256.Size {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %d bytes, at least %d needed",
|
|
||||||
errHashTooShort, len(decoded.Digest), sha256.Size,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
entry := &MFFilePath{
|
entry := &MFFilePath{
|
||||||
@@ -295,9 +281,8 @@ func (b *Builder) SetSigningOptions(opts *SigningOptions) {
|
|||||||
b.signingOptions = opts
|
b.signingOptions = opts
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build finalizes the manifest and writes it to the writer. ctx bounds the
|
// Build finalizes the manifest and writes it to the writer.
|
||||||
// gpg runs that sign the manifest when signing options are set.
|
func (b *Builder) Build(w io.Writer) error {
|
||||||
func (b *Builder) Build(ctx context.Context, w io.Writer) error {
|
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
defer b.mu.Unlock()
|
defer b.mu.Unlock()
|
||||||
|
|
||||||
@@ -323,13 +308,13 @@ func (b *Builder) Build(ctx context.Context, w io.Writer) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Generate outer wrapper
|
// Generate outer wrapper
|
||||||
err := m.generateOuter(ctx)
|
err := m.generateOuter()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("build: generate outer: %w", err)
|
return fmt.Errorf("build: generate outer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate final output
|
// Generate final output
|
||||||
err = m.generate(ctx)
|
err = m.generate()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("build: generate: %w", err)
|
return fmt.Errorf("build: generate: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
+34
-108
@@ -3,16 +3,10 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
@@ -46,10 +40,9 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
hash := make([]byte, 34) // SHA256 multihash is 34 bytes
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 1, b.FileCount())
|
assert.Equal(t, 1, b.FileCount())
|
||||||
}
|
}
|
||||||
@@ -57,14 +50,12 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
|||||||
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Run("empty path", func(t *testing.T) {
|
t.Run("empty path", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), sha256Hash)
|
hash := make([]byte, 34)
|
||||||
|
err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "path")
|
assert.Contains(t, err.Error(), "path")
|
||||||
})
|
})
|
||||||
@@ -73,58 +64,41 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), sha256Hash)
|
hash := make([]byte, 34)
|
||||||
|
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Contains(t, err.Error(), "size")
|
assert.Contains(t, err.Error(), "size")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("nil hash", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
b := NewBuilder()
|
||||||
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "hash")
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty hash", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
b := NewBuilder()
|
||||||
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "hash")
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid inputs", func(t *testing.T) {
|
t.Run("valid inputs", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), sha256Hash)
|
hash := make([]byte, 34)
|
||||||
|
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 1, b.FileCount())
|
assert.Equal(t, 1, b.FileCount())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
hash Multihash
|
|
||||||
want error
|
|
||||||
}{
|
|
||||||
{"nil hash", nil, errHashNotMultihash},
|
|
||||||
{"empty hash", []byte{}, errHashNotMultihash},
|
|
||||||
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
|
|
||||||
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
|
||||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
|
||||||
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
|
||||||
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
|
||||||
// A valid multihash whose 31-byte digest is one byte short.
|
|
||||||
{"31-byte digest", truncatedHash, errHashTooShort},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), tt.hash)
|
|
||||||
require.ErrorIs(t, err, tt.want)
|
|
||||||
assert.Equal(t, 0, b.FileCount())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderBuild(t *testing.T) {
|
func TestBuilderBuild(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -139,7 +113,7 @@ func TestBuilderBuild(t *testing.T) {
|
|||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err = b.Build(context.Background(), &buf)
|
err = b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Should have magic bytes
|
// Should have magic bytes
|
||||||
@@ -203,7 +177,7 @@ func TestBuilderDeterministicOutput(t *testing.T) {
|
|||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err := b.Build(context.Background(), &buf)
|
err := b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return buf.Bytes()
|
return buf.Bytes()
|
||||||
@@ -351,7 +325,7 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
require.NoError(t, b.Build(&buf))
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
m, err := NewManifestFromReader(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -374,31 +348,6 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A manifest whose payload is larger than one zstd block (128 KiB) is
|
|
||||||
// written as a frame asking for the writer's whole window, zstdWindowSize,
|
|
||||||
// which is the most the parser accepts.
|
|
||||||
func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
|
|
||||||
for i := range 4000 {
|
|
||||||
path := RelFilePath(fmt.Sprintf("dir/file-%05d.txt", i))
|
|
||||||
require.NoError(t, b.AddFileWithHash(path, FileSize(i), ModTime{}, hash))
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Greater(t, m.pbOuter.GetSize(), int64(128<<10))
|
|
||||||
assert.Len(t, m.Files(), 4000)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewManifestFromReaderInvalidMagic(t *testing.T) {
|
func TestNewManifestFromReaderInvalidMagic(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -422,29 +371,6 @@ func TestNewManifestFromReaderTruncated(t *testing.T) {
|
|||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewManifestFromFileRequiresPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := NewManifestFromFile(nil)
|
|
||||||
require.ErrorIs(t, err, errManifestPathEmpty)
|
|
||||||
|
|
||||||
_, err = NewManifestFromFile(&ManifestFromFileOptions{Fs: afero.NewMemMapFs()})
|
|
||||||
require.ErrorIs(t, err, errManifestPathEmpty)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewManifestFromFileNilFsUsesOsFs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "index.mf")
|
|
||||||
createTestManifest(t, afero.NewOsFs(), path, map[string][]byte{
|
|
||||||
testFileName: []byte("hello"),
|
|
||||||
})
|
|
||||||
|
|
||||||
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, m.Files(), 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestString(t *testing.T) {
|
func TestManifestString(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -457,7 +383,7 @@ func TestManifestString(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
require.NoError(t, b.Build(&buf))
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
m, err := NewManifestFromReader(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -471,7 +397,7 @@ func TestBuilderBuildEmpty(t *testing.T) {
|
|||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err := b.Build(context.Background(), &buf)
|
err := b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Should still produce valid manifest with 0 files
|
// Should still produce valid manifest with 0 files
|
||||||
@@ -490,7 +416,7 @@ func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
require.NoError(t, b.Build(&buf))
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
m, err := NewManifestFromReader(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -512,7 +438,7 @@ func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
require.NoError(t, b.Build(&buf))
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
m, err := NewManifestFromReader(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -538,7 +464,7 @@ func TestBuilderDeterministicFileOrder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
require.NoError(t, b.Build(&buf))
|
||||||
m, err := NewManifestFromReader(&buf)
|
m, err := NewManifestFromReader(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
|||||||
+10
-36
@@ -14,11 +14,7 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var errNoSigningPubKey = errors.New("manifest has no signing public key")
|
||||||
errNoSigningPubKey = errors.New("manifest has no signing public key")
|
|
||||||
errManifestPathEmpty = errors.New("manifest path cannot be empty")
|
|
||||||
errBasePathEmpty = errors.New("base path cannot be empty")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Result represents the outcome of checking a single file.
|
// Result represents the outcome of checking a single file.
|
||||||
type Result struct {
|
type Result struct {
|
||||||
@@ -86,42 +82,20 @@ type Checker struct {
|
|||||||
signingPubKey []byte
|
signingPubKey []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// CheckerOptions configures a Checker.
|
// NewChecker creates a new Checker for the given manifest, base path, and filesystem.
|
||||||
type CheckerOptions struct {
|
// The basePath is the directory relative to which manifest paths are resolved.
|
||||||
// ManifestPath is the manifest file to check against (required).
|
// If fs is nil, the real filesystem (OsFs) is used.
|
||||||
ManifestPath string
|
func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, error) {
|
||||||
// BasePath is the directory relative to which manifest paths are
|
|
||||||
// resolved (required).
|
|
||||||
BasePath string
|
|
||||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
|
||||||
Fs afero.Fs
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewChecker creates a new Checker with the given options. It returns an
|
|
||||||
// error if opts is nil or either path is empty.
|
|
||||||
func NewChecker(opts *CheckerOptions) (*Checker, error) {
|
|
||||||
if opts == nil || opts.ManifestPath == "" {
|
|
||||||
return nil, errManifestPathEmpty
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.BasePath == "" {
|
|
||||||
return nil, errBasePathEmpty
|
|
||||||
}
|
|
||||||
|
|
||||||
fs := opts.Fs
|
|
||||||
if fs == nil {
|
if fs == nil {
|
||||||
fs = afero.NewOsFs()
|
fs = afero.NewOsFs()
|
||||||
}
|
}
|
||||||
|
|
||||||
m, err := NewManifestFromFile(&ManifestFromFileOptions{
|
m, err := NewManifestFromFile(fs, manifestPath)
|
||||||
Path: opts.ManifestPath,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
abs, err := filepath.Abs(opts.BasePath)
|
abs, err := filepath.Abs(basePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -134,7 +108,7 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
|
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
|
||||||
absManifest, err := filepath.Abs(opts.ManifestPath)
|
absManifest, err := filepath.Abs(manifestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -190,12 +164,12 @@ func (c *Checker) SigningPubKey() []byte {
|
|||||||
// ExtractEmbeddedSigningKeyFP imports the manifest's embedded public key into a
|
// ExtractEmbeddedSigningKeyFP imports the manifest's embedded public key into a
|
||||||
// temporary keyring and extracts its fingerprint. This validates the key and
|
// temporary keyring and extracts its fingerprint. This validates the key and
|
||||||
// returns its actual fingerprint from the key material itself.
|
// returns its actual fingerprint from the key material itself.
|
||||||
func (c *Checker) ExtractEmbeddedSigningKeyFP(ctx context.Context) (string, error) {
|
func (c *Checker) ExtractEmbeddedSigningKeyFP() (string, error) {
|
||||||
if len(c.signingPubKey) == 0 {
|
if len(c.signingPubKey) == 0 {
|
||||||
return "", errNoSigningPubKey
|
return "", errNoSigningPubKey
|
||||||
}
|
}
|
||||||
|
|
||||||
return gpgExtractPubKeyFingerprint(ctx, c.signingPubKey)
|
return gpgExtractPubKeyFingerprint(c.signingPubKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check verifies all files against the manifest.
|
// Check verifies all files against the manifest.
|
||||||
|
|||||||
+48
-194
@@ -5,8 +5,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -19,10 +17,6 @@ const (
|
|||||||
testFile1 = "file1.txt"
|
testFile1 = "file1.txt"
|
||||||
testFile2 = "file2.txt"
|
testFile2 = "file2.txt"
|
||||||
testExistsFile = "exists.txt"
|
testExistsFile = "exists.txt"
|
||||||
testManifestPath = "/manifest.mf"
|
|
||||||
testDataDir = "/data"
|
|
||||||
// testDataManifestPath is a manifest kept inside the checked tree.
|
|
||||||
testDataManifestPath = testDataDir + "/index.mf"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestStatusString(t *testing.T) {
|
func TestStatusString(t *testing.T) {
|
||||||
@@ -67,18 +61,20 @@ func createTestManifest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, builder.Build(context.Background(), &buf))
|
require.NoError(t, builder.Build(&buf))
|
||||||
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0o644))
|
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0o644))
|
||||||
}
|
}
|
||||||
|
|
||||||
// createFilesOnDisk creates the given files on the filesystem under
|
// createFilesOnDisk creates the given files on the filesystem under
|
||||||
// testDataDir.
|
// /data.
|
||||||
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
|
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
basePath := "/data"
|
||||||
|
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
fullPath := testDataDir + "/" + path
|
fullPath := basePath + "/" + path
|
||||||
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
|
require.NoError(t, fs.MkdirAll(basePath, 0o755))
|
||||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,13 +90,9 @@ func TestNewChecker(t *testing.T) {
|
|||||||
testFile1: []byte("hello"),
|
testFile1: []byte("hello"),
|
||||||
testFile2: []byte("world"),
|
testFile2: []byte("world"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: "/",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NotNil(t, chk)
|
assert.NotNil(t, chk)
|
||||||
assert.Equal(t, FileCount(2), chk.FileCount())
|
assert.Equal(t, FileCount(2), chk.FileCount())
|
||||||
@@ -110,11 +102,7 @@ func TestNewChecker(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
_, err := NewChecker(&CheckerOptions{
|
_, err := NewChecker("/nonexistent.mf", "/", fs)
|
||||||
ManifestPath: "/nonexistent.mf",
|
|
||||||
BasePath: "/",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -123,73 +111,11 @@ func TestNewChecker(t *testing.T) {
|
|||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
|
||||||
_, err := NewChecker(&CheckerOptions{
|
_, err := NewChecker("/bad.mf", "/", fs)
|
||||||
ManifestPath: "/bad.mf",
|
|
||||||
BasePath: "/",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewCheckerRequiredPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
opts *CheckerOptions
|
|
||||||
want string
|
|
||||||
is error
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "nil options",
|
|
||||||
opts: nil,
|
|
||||||
want: "manifest path cannot be empty",
|
|
||||||
is: errManifestPathEmpty,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty manifest path",
|
|
||||||
opts: &CheckerOptions{BasePath: testDataDir},
|
|
||||||
want: "manifest path cannot be empty",
|
|
||||||
is: errManifestPathEmpty,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty base path",
|
|
||||||
opts: &CheckerOptions{ManifestPath: testManifestPath},
|
|
||||||
want: "base path cannot be empty",
|
|
||||||
is: errBasePathEmpty,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
chk, err := NewChecker(tc.opts)
|
|
||||||
require.ErrorIs(t, err, tc.is)
|
|
||||||
require.EqualError(t, err, tc.want)
|
|
||||||
assert.Nil(t, chk)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
manifestPath := filepath.Join(dir, "index.mf")
|
|
||||||
content := []byte("hello")
|
|
||||||
createTestManifest(t, afero.NewOsFs(), manifestPath, map[string][]byte{
|
|
||||||
testFile1: content,
|
|
||||||
})
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, testFile1), content, 0o600))
|
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{ManifestPath: manifestPath, BasePath: dir})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 1)
|
|
||||||
require.NoError(t, chk.Check(context.Background(), results, nil))
|
|
||||||
assert.Equal(t, StatusOK, (<-results).Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -199,13 +125,9 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
|||||||
"medium.txt": []byte("hello world"),
|
"medium.txt": []byte("hello world"),
|
||||||
"large.txt": bytes.Repeat([]byte("x"), 1000),
|
"large.txt": bytes.Repeat([]byte("x"), 1000),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: "/",
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.Equal(t, FileCount(3), chk.FileCount())
|
assert.Equal(t, FileCount(3), chk.FileCount())
|
||||||
@@ -220,14 +142,10 @@ func TestCheckAllFilesOK(t *testing.T) {
|
|||||||
testFile1: []byte("content one"),
|
testFile1: []byte("content one"),
|
||||||
testFile2: []byte("content two"),
|
testFile2: []byte("content two"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -254,17 +172,13 @@ func TestCheckMissingFile(t *testing.T) {
|
|||||||
testExistsFile: []byte("I exist"),
|
testExistsFile: []byte("I exist"),
|
||||||
"missing.txt": []byte("I don't exist on disk"),
|
"missing.txt": []byte("I don't exist on disk"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
// Only create one file
|
// Only create one file
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
createFilesOnDisk(t, fs, map[string][]byte{
|
||||||
testExistsFile: []byte("I exist"),
|
testExistsFile: []byte("I exist"),
|
||||||
})
|
})
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -297,17 +211,13 @@ func TestCheckSizeMismatch(t *testing.T) {
|
|||||||
files := map[string][]byte{
|
files := map[string][]byte{
|
||||||
testFileName: []byte("original content"),
|
testFileName: []byte("original content"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
// Create file with different size
|
// Create file with different size
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
createFilesOnDisk(t, fs, map[string][]byte{
|
||||||
testFileName: []byte("short"),
|
testFileName: []byte("short"),
|
||||||
})
|
})
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -327,7 +237,7 @@ func TestCheckHashMismatch(t *testing.T) {
|
|||||||
files := map[string][]byte{
|
files := map[string][]byte{
|
||||||
testFileName: originalContent,
|
testFileName: originalContent,
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
// Create file with same size but different content
|
// Create file with same size but different content
|
||||||
differentContent := []byte("different contnt") // same length (16 bytes) but different
|
differentContent := []byte("different contnt") // same length (16 bytes) but different
|
||||||
require.Len(t, differentContent, len(originalContent), "test requires same length")
|
require.Len(t, differentContent, len(originalContent), "test requires same length")
|
||||||
@@ -335,11 +245,7 @@ func TestCheckHashMismatch(t *testing.T) {
|
|||||||
testFileName: differentContent,
|
testFileName: differentContent,
|
||||||
})
|
})
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -359,14 +265,10 @@ func TestCheckWithProgress(t *testing.T) {
|
|||||||
testFile1: bytes.Repeat([]byte("a"), 100),
|
testFile1: bytes.Repeat([]byte("a"), 100),
|
||||||
testFile2: bytes.Repeat([]byte("b"), 200),
|
testFile2: bytes.Repeat([]byte("b"), 200),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -403,14 +305,10 @@ func TestCheckContextCancellation(t *testing.T) {
|
|||||||
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
|
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
|
||||||
}
|
}
|
||||||
|
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -429,7 +327,7 @@ func TestFindExtraFiles(t *testing.T) {
|
|||||||
manifestFiles := map[string][]byte{
|
manifestFiles := map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, manifestFiles)
|
createTestManifest(t, fs, "/manifest.mf", manifestFiles)
|
||||||
|
|
||||||
// Disk has file1 and file2
|
// Disk has file1 and file2
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
createFilesOnDisk(t, fs, map[string][]byte{
|
||||||
@@ -437,11 +335,7 @@ func TestFindExtraFiles(t *testing.T) {
|
|||||||
testFile2: []byte("extra file"),
|
testFile2: []byte("extra file"),
|
||||||
})
|
})
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -466,7 +360,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|||||||
manifestFiles := map[string][]byte{
|
manifestFiles := map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testDataManifestPath, manifestFiles)
|
createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
createFilesOnDisk(t, fs, map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
})
|
})
|
||||||
@@ -474,14 +368,10 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|||||||
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
|
||||||
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
|
||||||
// Create a real extra file
|
// Create a real extra file
|
||||||
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
|
require.NoError(t, fs.MkdirAll("/data", 0o755))
|
||||||
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/data/.index.mf", "/data", fs)
|
||||||
ManifestPath: testDataManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -493,7 +383,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|||||||
extras = append(extras, r)
|
extras = append(extras, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Should only report extra.txt, not .hidden, .config/settings, or index.mf
|
// Should only report extra.txt, not .hidden, .config/settings, or .index.mf
|
||||||
for _, e := range extras {
|
for _, e := range extras {
|
||||||
t.Logf("extra: %s", e.Path)
|
t.Logf("extra: %s", e.Path)
|
||||||
}
|
}
|
||||||
@@ -510,14 +400,10 @@ func TestFindExtraFilesContextCancellation(t *testing.T) {
|
|||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
files := map[string][]byte{testFileName: []byte("data")}
|
files := map[string][]byte{testFileName: []byte("data")}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -533,14 +419,10 @@ func TestCheckNilChannels(t *testing.T) {
|
|||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
files := map[string][]byte{testFileName: []byte("data")}
|
files := map[string][]byte{testFileName: []byte("data")}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Should not panic with nil channels
|
// Should not panic with nil channels
|
||||||
@@ -553,14 +435,10 @@ func TestFindExtraFilesNilChannel(t *testing.T) {
|
|||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
files := map[string][]byte{testFileName: []byte("data")}
|
files := map[string][]byte{testFileName: []byte("data")}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Should not panic with nil channel
|
// Should not panic with nil channel
|
||||||
@@ -577,7 +455,7 @@ func TestCheckSubdirectories(t *testing.T) {
|
|||||||
"dir1/dir2/file2.txt": []byte("content2"),
|
"dir1/dir2/file2.txt": []byte("content2"),
|
||||||
"dir1/dir2/dir3/deep.txt": []byte("deep content"),
|
"dir1/dir2/dir3/deep.txt": []byte("deep content"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
|
|
||||||
// Create files with full directory structure
|
// Create files with full directory structure
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
@@ -587,11 +465,7 @@ func TestCheckSubdirectories(t *testing.T) {
|
|||||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
||||||
}
|
}
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -619,17 +493,13 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
|
|||||||
testExistsFile: []byte("here"),
|
testExistsFile: []byte("here"),
|
||||||
"missing.txt": []byte("not on disk"),
|
"missing.txt": []byte("not on disk"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
// Only create one file on disk
|
// Only create one file on disk
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
createFilesOnDisk(t, fs, map[string][]byte{
|
||||||
testExistsFile: []byte("here"),
|
testExistsFile: []byte("here"),
|
||||||
})
|
})
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -658,7 +528,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
|||||||
files := map[string][]byte{
|
files := map[string][]byte{
|
||||||
testFile1: []byte("in manifest"),
|
testFile1: []byte("in manifest"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testDataManifestPath, files)
|
createTestManifest(t, fs, "/data/.index.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
// Add dotfiles and manifest file on disk
|
// Add dotfiles and manifest file on disk
|
||||||
@@ -667,11 +537,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
|||||||
require.NoError(t,
|
require.NoError(t,
|
||||||
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/data/.index.mf", "/data", fs)
|
||||||
ManifestPath: testDataManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -697,14 +563,10 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
|
|||||||
files := map[string][]byte{
|
files := map[string][]byte{
|
||||||
testFile1: []byte("content"),
|
testFile1: []byte("content"),
|
||||||
}
|
}
|
||||||
createTestManifest(t, fs, testDataManifestPath, files)
|
createTestManifest(t, fs, "/data/index.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/data/index.mf", "/data", fs)
|
||||||
ManifestPath: testDataManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
results := make(chan Result, 10)
|
||||||
@@ -725,13 +587,9 @@ func TestCheckEmptyManifest(t *testing.T) {
|
|||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
// Create manifest with no files
|
// Create manifest with no files
|
||||||
createTestManifest(t, fs, testManifestPath, map[string][]byte{})
|
createTestManifest(t, fs, "/manifest.mf", map[string][]byte{})
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.Equal(t, FileCount(0), chk.FileCount())
|
assert.Equal(t, FileCount(0), chk.FileCount())
|
||||||
@@ -763,14 +621,10 @@ func TestCheckProgressRateLimited(t *testing.T) {
|
|||||||
files[name] = []byte("content")
|
files[name] = []byte("content")
|
||||||
}
|
}
|
||||||
|
|
||||||
createTestManifest(t, fs, testManifestPath, files)
|
createTestManifest(t, fs, "/manifest.mf", files)
|
||||||
createFilesOnDisk(t, fs, files)
|
createFilesOnDisk(t, fs, files)
|
||||||
|
|
||||||
chk, err := NewChecker(&CheckerOptions{
|
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||||
ManifestPath: testManifestPath,
|
|
||||||
BasePath: testDataDir,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
results := make(chan Result, 200)
|
results := make(chan Result, 200)
|
||||||
|
|||||||
@@ -12,33 +12,6 @@ const (
|
|||||||
// memory.
|
// memory.
|
||||||
MaxDecompressedSize int64 = 256 * 1024 * 1024
|
MaxDecompressedSize int64 = 256 * 1024 * 1024
|
||||||
|
|
||||||
// zstdWindowSize is the zstd window zstd.SpeedBestCompression gives mfer's writer.
|
|
||||||
zstdWindowSize = 8 << 20
|
|
||||||
|
|
||||||
// uuidLength is the length in bytes of a binary UUID.
|
// uuidLength is the length in bytes of a binary UUID.
|
||||||
uuidLength = 16
|
uuidLength = 16
|
||||||
|
|
||||||
// Numbers in mf.proto of MFFile.files and of the MFFilePath fields
|
|
||||||
// that decoding sets aside a fixed amount of memory for.
|
|
||||||
filesFieldNumber = 101
|
|
||||||
hashesFieldNumber = 3
|
|
||||||
mimeTypeFieldNumber = 301
|
|
||||||
mtimeFieldNumber = 302
|
|
||||||
ctimeFieldNumber = 303
|
|
||||||
|
|
||||||
// Bytes decoding sets aside for each file entry, hash, timestamp and
|
|
||||||
// MIME type, however short its encoding. checkDecodedSize refuses an
|
|
||||||
// inner message for which these add up to more than maxDecodedGrowth
|
|
||||||
// times its size.
|
|
||||||
decodedFileEntrySize = 160
|
|
||||||
decodedHashSize = 112
|
|
||||||
decodedTimestampSize = 64
|
|
||||||
decodedMIMETypeSize = 16
|
|
||||||
|
|
||||||
// Each file entry mfer writes holds a path of at least one byte, a
|
|
||||||
// multihash at least as long as SHA-256's 34 bytes (AddFileWithHash
|
|
||||||
// refuses shorter ones) and a modification time: at least 47 bytes,
|
|
||||||
// counted at 336. So its manifests add up to at most about 7.15 times
|
|
||||||
// their size, and this limit is about 12% above that.
|
|
||||||
maxDecodedGrowth = 8
|
|
||||||
)
|
)
|
||||||
|
|||||||
+7
-123
@@ -2,7 +2,6 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -11,7 +10,6 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"google.golang.org/protobuf/encoding/protowire"
|
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
"sneak.berlin/go/mfer/internal/bork"
|
"sneak.berlin/go/mfer/internal/bork"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
"sneak.berlin/go/mfer/internal/log"
|
||||||
@@ -28,8 +26,6 @@ var (
|
|||||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
||||||
errInvalidFileFormat = errors.New("invalid file format")
|
errInvalidFileFormat = errors.New("invalid file format")
|
||||||
errInvalidManifestPath = errors.New("manifest contains invalid path")
|
errInvalidManifestPath = errors.New("manifest contains invalid path")
|
||||||
errDecodedTooLarge = errors.New(
|
|
||||||
"manifest would take too much memory to decode")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
|
// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
|
||||||
@@ -96,9 +92,7 @@ func (m *manifest) verifyOuterIntegrity() error {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Loading a manifest takes no context; gpgTimeout still bounds gpg.
|
|
||||||
err = gpgVerify(
|
err = gpgVerify(
|
||||||
context.Background(),
|
|
||||||
[]byte(sigString),
|
[]byte(sigString),
|
||||||
m.pbOuter.GetSignature(),
|
m.pbOuter.GetSignature(),
|
||||||
m.pbOuter.GetSigningPubKey(),
|
m.pbOuter.GetSigningPubKey(),
|
||||||
@@ -117,18 +111,7 @@ func (m *manifest) verifyOuterIntegrity() error {
|
|||||||
func (m *manifest) decompressInner() ([]byte, error) {
|
func (m *manifest) decompressInner() ([]byte, error) {
|
||||||
bb := bytes.NewBuffer(m.pbOuter.GetInnerMessage())
|
bb := bytes.NewBuffer(m.pbOuter.GetInnerMessage())
|
||||||
|
|
||||||
// By default the decoder decodes a payload under 128 KiB in full,
|
zr, err := zstd.NewReader(bb)
|
||||||
// each frame up to the decoder's limit, before the LimitReader below
|
|
||||||
// reads any of it. Decoding synchronously and never in full makes it
|
|
||||||
// decode only what the LimitReader asks for. It also sets aside a new
|
|
||||||
// buffer for each frame that asks for a larger window than the frames
|
|
||||||
// before it, even a frame holding no data. Refusing windows above
|
|
||||||
// zstdWindowSize keeps each buffer to a little over zstdWindowSize, and
|
|
||||||
// the buffers of frames holding no data to about 16 times it in total.
|
|
||||||
zr, err := zstd.NewReader(bb,
|
|
||||||
zstd.WithDecoderConcurrency(1),
|
|
||||||
zstd.WithDecodeBuffersBelow(0),
|
|
||||||
zstd.WithDecoderMaxWindow(zstdWindowSize))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
|
return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
|
||||||
}
|
}
|
||||||
@@ -157,85 +140,6 @@ func (m *manifest) decompressInner() ([]byte, error) {
|
|||||||
return dat, nil
|
return dat, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkDecodedSize refuses an encoded inner message whose file entries,
|
|
||||||
// hashes, timestamps and MIME types would take more than maxDecodedGrowth
|
|
||||||
// times its size to decode. Decoding sets aside a fixed amount for each,
|
|
||||||
// however short its encoding, so a message of empty ones would take about
|
|
||||||
// 50 times its size.
|
|
||||||
func checkDecodedSize(inner []byte) error {
|
|
||||||
limit := maxDecodedGrowth * int64(len(inner))
|
|
||||||
|
|
||||||
var decoded int64
|
|
||||||
|
|
||||||
add := func(size int64) error {
|
|
||||||
decoded += size
|
|
||||||
if decoded > limit {
|
|
||||||
return errDecodedTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return forEachBytesField(inner, func(num protowire.Number, entry []byte) error {
|
|
||||||
if num != filesFieldNumber {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err := add(decodedFileEntrySize)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return forEachBytesField(entry, func(num protowire.Number, _ []byte) error {
|
|
||||||
if num == hashesFieldNumber {
|
|
||||||
return add(decodedHashSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
if num == mtimeFieldNumber || num == ctimeFieldNumber {
|
|
||||||
return add(decodedTimestampSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
if num == mimeTypeFieldNumber {
|
|
||||||
return add(decodedMIMETypeSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// forEachBytesField calls fn with the number and value of each
|
|
||||||
// length-delimited field in the encoded message msg, and fails if msg is
|
|
||||||
// malformed.
|
|
||||||
func forEachBytesField(
|
|
||||||
msg []byte, fn func(num protowire.Number, value []byte) error,
|
|
||||||
) error {
|
|
||||||
for len(msg) > 0 {
|
|
||||||
num, wireType, tagLen := protowire.ConsumeTag(msg)
|
|
||||||
if tagLen < 0 {
|
|
||||||
return protowire.ParseError(tagLen)
|
|
||||||
}
|
|
||||||
|
|
||||||
valueLen := protowire.ConsumeFieldValue(num, wireType, msg[tagLen:])
|
|
||||||
if valueLen < 0 {
|
|
||||||
return protowire.ParseError(valueLen)
|
|
||||||
}
|
|
||||||
|
|
||||||
if wireType == protowire.BytesType {
|
|
||||||
value, _ := protowire.ConsumeBytes(msg[tagLen:])
|
|
||||||
|
|
||||||
err := fn(num, value)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
msg = msg[tagLen+valueLen:]
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *manifest) deserializeInner() error {
|
func (m *manifest) deserializeInner() error {
|
||||||
err := m.validateOuterHeader()
|
err := m.validateOuterHeader()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -259,16 +163,10 @@ func (m *manifest) deserializeInner() error {
|
|||||||
return bork.ErrFileTruncated
|
return bork.ErrFileTruncated
|
||||||
}
|
}
|
||||||
|
|
||||||
err = checkDecodedSize(dat)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deserialize inner message
|
// Deserialize inner message
|
||||||
m.pbInner = new(MFFile)
|
m.pbInner = new(MFFile)
|
||||||
|
|
||||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
err = proto.Unmarshal(dat, m.pbInner)
|
||||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbInner)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
||||||
}
|
}
|
||||||
@@ -337,8 +235,7 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|||||||
// deserialize outer:
|
// deserialize outer:
|
||||||
m.pbOuter = new(MFFileOuter)
|
m.pbOuter = new(MFFileOuter)
|
||||||
|
|
||||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
err = proto.Unmarshal(dat, m.pbOuter)
|
||||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbOuter)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -352,29 +249,16 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ManifestFromFileOptions configures NewManifestFromFile.
|
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
||||||
type ManifestFromFileOptions struct {
|
// If fs is nil, the real filesystem (OsFs) is used.
|
||||||
// Path is the manifest file to read (required).
|
|
||||||
Path string
|
|
||||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
|
||||||
Fs afero.Fs
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewManifestFromFile reads a manifest from a file. It returns an error if
|
|
||||||
// opts is nil or its path is empty.
|
|
||||||
//
|
//
|
||||||
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
||||||
func NewManifestFromFile(opts *ManifestFromFileOptions) (*manifest, error) {
|
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
||||||
if opts == nil || opts.Path == "" {
|
|
||||||
return nil, errManifestPathEmpty
|
|
||||||
}
|
|
||||||
|
|
||||||
fs := opts.Fs
|
|
||||||
if fs == nil {
|
if fs == nil {
|
||||||
fs = afero.NewOsFs()
|
fs = afero.NewOsFs()
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := fs.Open(opts.Path)
|
f, err := fs.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"google.golang.org/protobuf/proto"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FuzzNewManifestFromReader feeds arbitrary bytes to the manifest parser.
|
|
||||||
// `make test` runs it on the seed corpus in
|
|
||||||
// testdata/fuzz/FuzzNewManifestFromReader; `make fuzz` searches for new
|
|
||||||
// inputs.
|
|
||||||
//
|
|
||||||
// For every input the parser must return a manifest or an error, not both
|
|
||||||
// and not neither, and must not allocate more than a fixed multiple of its
|
|
||||||
// input and of the decompressed data it may read, plus room for the
|
|
||||||
// decoder's window buffers. A panic or a hang fails the test on its own.
|
|
||||||
func FuzzNewManifestFromReader(f *testing.F) {
|
|
||||||
// A signed manifest makes the parser write the key and signature to a
|
|
||||||
// temporary directory and run gpg on them. With gpg off the PATH and
|
|
||||||
// temporary files kept in the test's own directory, no process is
|
|
||||||
// started and nothing is written elsewhere; such input ends in an
|
|
||||||
// error instead.
|
|
||||||
f.Setenv("PATH", "")
|
|
||||||
f.Setenv("TMPDIR", f.TempDir())
|
|
||||||
|
|
||||||
f.Fuzz(func(t *testing.T, data []byte) {
|
|
||||||
var before, after runtime.MemStats
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&before)
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&after)
|
|
||||||
|
|
||||||
if (m == nil) == (err == nil) {
|
|
||||||
t.Fatalf("got manifest %p and error %v, want exactly one", m, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The parser reads at most the declared size plus one byte of
|
|
||||||
// decompressed data, and never more than MaxDecompressedSize.
|
|
||||||
decompressed := uint64(MaxDecompressedSize)
|
|
||||||
|
|
||||||
outer := new(MFFileOuter)
|
|
||||||
if validateMagic(data) &&
|
|
||||||
proto.Unmarshal(data[len(MAGIC):], outer) == nil {
|
|
||||||
size := outer.GetSize()
|
|
||||||
if size > 0 && size < MaxDecompressedSize {
|
|
||||||
decompressed = uint64(size) + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// It also keeps a few copies of its input. Buffers grow by
|
|
||||||
// copying, so reaching those sizes allocates up to about six times
|
|
||||||
// them in total. Decoding the decompressed data takes up to
|
|
||||||
// maxDecodedGrowth times its size for file entries, hashes,
|
|
||||||
// timestamps and MIME types, and drops fields it does not know.
|
|
||||||
// The strings and bytes it copies out of it, such as many one-byte
|
|
||||||
// values in one hash, take up to about five times more under the
|
|
||||||
// race detector, which pads every small copy to 16 bytes, and about
|
|
||||||
// half that without it. Twenty times the input and the
|
|
||||||
// decompressed data leaves room for all of that.
|
|
||||||
//
|
|
||||||
// The decoder also sets aside a new buffer of one to two times the
|
|
||||||
// window for each frame that asks for a larger window than the
|
|
||||||
// frames before it, and refuses windows above zstdWindowSize. A
|
|
||||||
// frame that gives its content size instead of a window has that
|
|
||||||
// size as its window, refused above zstdWindowSize like any other.
|
|
||||||
// Frames asking for every window size up to that make it set aside
|
|
||||||
// about 16 times zstdWindowSize in total; 24 times leaves room.
|
|
||||||
//
|
|
||||||
// The seed whose frame claims 8 GiB fails if the decoder sets that
|
|
||||||
// size aside; the seed whose frames ask for ever larger windows
|
|
||||||
// fails if the decoder accepts windows of twice zstdWindowSize; the
|
|
||||||
// seed whose two frames together exceed MaxDecompressedSize fails
|
|
||||||
// if the decoder decodes them in full instead of stopping at the
|
|
||||||
// declared size; the seeds of empty file entries and of a file
|
|
||||||
// entry of empty hashes fail if the parser decodes them.
|
|
||||||
limit := 20*(uint64(len(data))+decompressed) + 24*zstdWindowSize
|
|
||||||
|
|
||||||
allocated := after.TotalAlloc - before.TotalAlloc
|
|
||||||
if allocated > limit {
|
|
||||||
t.Fatalf("allocated %d bytes for %d bytes of input, limit %d",
|
|
||||||
allocated, len(data), limit)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -3,17 +3,12 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"google.golang.org/protobuf/encoding/protowire"
|
"google.golang.org/protobuf/encoding/protowire"
|
||||||
@@ -118,125 +113,16 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
|
||||||
// and change times are counted at 416 bytes each (160 + 112 + 16 + 64 + 64)
|
|
||||||
// and take 16 bytes plus the path to encode. A 35-character path makes that
|
|
||||||
// 51 bytes, about 8.2 times: refused, and leaving any one of the five
|
|
||||||
// uncounted, even the MIME type, brings it under 8. A 37-character path makes
|
|
||||||
// it 53 bytes, about 7.8 times: loaded.
|
|
||||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
pathLen int
|
|
||||||
refused bool
|
|
||||||
}{
|
|
||||||
{35, true},
|
|
||||||
{37, false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
||||||
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen))
|
|
||||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
|
||||||
entry = protowire.AppendBytes(entry, nil)
|
|
||||||
|
|
||||||
id := uuid.New()
|
|
||||||
inner := protowire.AppendTag(nil, 102, protowire.BytesType) // MFFile.uuid
|
|
||||||
inner = protowire.AppendBytes(inner, id[:])
|
|
||||||
|
|
||||||
for range 1000 {
|
|
||||||
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
|
|
||||||
inner = protowire.AppendBytes(inner, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
|
||||||
if tt.refused {
|
|
||||||
require.ErrorIs(t, err, errDecodedTooLarge)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fields the decoder does not know are dropped, in the outer message, the inner
|
|
||||||
// message and a file entry, so that they take no memory once loaded.
|
|
||||||
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
|
||||||
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
|
||||||
|
|
||||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
|
||||||
entry = protowire.AppendString(entry, "a")
|
|
||||||
entry = append(entry, unknown...)
|
|
||||||
|
|
||||||
id := uuid.New()
|
|
||||||
inner := protowire.AppendTag(nil, 101, protowire.BytesType) // MFFile.files
|
|
||||||
inner = protowire.AppendBytes(inner, entry)
|
|
||||||
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
|
||||||
inner = protowire.AppendBytes(inner, id[:])
|
|
||||||
inner = append(inner, unknown...)
|
|
||||||
|
|
||||||
data := wrapInner(t, id, inner)
|
|
||||||
data = append(data, unknown...) // the outer message ends the file
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(bytes.NewReader(data))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, m.Files(), 1)
|
|
||||||
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
|
||||||
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
|
||||||
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Many empty files with names of at most three characters and modification
|
|
||||||
// times at the epoch make about the densest manifest mfer writes: it takes
|
|
||||||
// about 7 times its size to decode, and still loads. A signature would not
|
|
||||||
// change the inner message, so none is added.
|
|
||||||
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
b.SetIncludeTimestamps(true)
|
|
||||||
|
|
||||||
const files = 10000
|
|
||||||
for i := range files {
|
|
||||||
name := RelFilePath(strconv.FormatInt(int64(i), 36))
|
|
||||||
require.NoError(t, b.AddFileWithHash(name, 0, ModTime(time.Unix(0, 0)), hash))
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, m.Files(), files)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
b := NewBuilder()
|
b := NewBuilder()
|
||||||
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
require.NoError(t, b.Build(context.Background(), &buf))
|
require.NoError(t, b.Build(&buf))
|
||||||
|
|
||||||
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
+2
-4
@@ -2,7 +2,6 @@
|
|||||||
package mfer
|
package mfer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -81,7 +80,6 @@ func TestSerializeInternalErrorMessagesVerbatim(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
m := &manifest{}
|
m := &manifest{}
|
||||||
require.EqualError(t, m.generate(context.Background()),
|
require.EqualError(t, m.generate(), "internal error: pbInner not set")
|
||||||
"internal error: pbInner not set")
|
require.EqualError(t, m.generateOuter(), "internal error")
|
||||||
require.EqualError(t, m.generateOuter(context.Background()), "internal error")
|
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-48
@@ -10,21 +10,9 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// gpgTimeout bounds every gpg run, which can otherwise wait forever on
|
|
||||||
// a passphrase prompt or a stalled gpg-agent. A minute leaves a person
|
|
||||||
// time to type a passphrase or touch a smartcard.
|
|
||||||
gpgTimeout = time.Minute
|
|
||||||
|
|
||||||
// gpgWaitDelay is how long a gpg run keeps waiting for gpg's stdout
|
|
||||||
// and stderr to close once gpg has been killed or has exited. Reading
|
|
||||||
// what gpg itself wrote takes far less; only a process gpg left behind
|
|
||||||
// holds them open longer.
|
|
||||||
gpgWaitDelay = time.Second
|
|
||||||
|
|
||||||
// privateDirPerms is the permission mode for temporary GPG home
|
// privateDirPerms is the permission mode for temporary GPG home
|
||||||
// directories.
|
// directories.
|
||||||
privateDirPerms os.FileMode = 0o700
|
privateDirPerms os.FileMode = 0o700
|
||||||
@@ -78,20 +66,8 @@ func gpgArgs(opts []string, positional ...string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// runGPG runs the gpg binary in batch mode with the given arguments and
|
// runGPG runs the gpg binary in batch mode with the given arguments and
|
||||||
// optional stdin, returning captured stdout and stderr. gpg is killed when
|
// optional stdin, returning captured stdout and stderr.
|
||||||
// ctx ends or gpgTimeout passes, whichever comes first.
|
func runGPG(stdin io.Reader, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
|
||||||
func runGPG(
|
|
||||||
ctx context.Context, stdin io.Reader, args ...string,
|
|
||||||
) (*bytes.Buffer, *bytes.Buffer, error) {
|
|
||||||
// exec.CommandContext kills only gpg itself. A gpg-agent that gpg
|
|
||||||
// starts runs detached and holds none of gpg's output, but another
|
|
||||||
// process gpg leaves behind (a wrapper script that runs the real gpg
|
|
||||||
// without exec, for example) can keep gpg's stdout or stderr open, and
|
|
||||||
// Run would wait for it to exit. WaitDelay stops that wait
|
|
||||||
// gpgWaitDelay after the kill; that process is left running.
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, gpgTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
fullArgs := append([]string{"--batch", "--no-tty"}, args...)
|
fullArgs := append([]string{"--batch", "--no-tty"}, args...)
|
||||||
|
|
||||||
// G204: the executable name is a compile-time constant. The arguments
|
// G204: the executable name is a compile-time constant. The arguments
|
||||||
@@ -100,8 +76,7 @@ func runGPG(
|
|||||||
// option or after the "--" end-of-options marker inserted by gpgArgs,
|
// option or after the "--" end-of-options marker inserted by gpgArgs,
|
||||||
// and therefore cannot be reinterpreted by gpg as an option.
|
// and therefore cannot be reinterpreted by gpg as an option.
|
||||||
cmd := exec.CommandContext( //nolint:gosec // G204: see comment above
|
cmd := exec.CommandContext( //nolint:gosec // G204: see comment above
|
||||||
ctx, "gpg", fullArgs...)
|
context.Background(), "gpg", fullArgs...)
|
||||||
cmd.WaitDelay = gpgWaitDelay
|
|
||||||
cmd.Stdin = stdin
|
cmd.Stdin = stdin
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
@@ -110,14 +85,6 @@ func runGPG(
|
|||||||
cmd.Stderr = &stderr
|
cmd.Stderr = &stderr
|
||||||
|
|
||||||
err := cmd.Run()
|
err := cmd.Run()
|
||||||
if err != nil && ctx.Err() != nil {
|
|
||||||
// gpg was killed because ctx ended, which Run reports only as
|
|
||||||
// "signal: killed"; return the reason instead.
|
|
||||||
err = ctx.Err()
|
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
err = fmt.Errorf("gpg timed out: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &stdout, &stderr, err
|
return &stdout, &stderr, err
|
||||||
}
|
}
|
||||||
@@ -138,8 +105,8 @@ func parseFingerprint(colonOutput string) (string, bool) {
|
|||||||
|
|
||||||
// gpgSign creates a detached signature of the data using the specified key.
|
// gpgSign creates a detached signature of the data using the specified key.
|
||||||
// Returns the armored detached signature.
|
// Returns the armored detached signature.
|
||||||
func gpgSign(ctx context.Context, data []byte, keyID GPGKeyID) ([]byte, error) {
|
func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
|
||||||
stdout, stderr, err := runGPG(ctx, bytes.NewReader(data),
|
stdout, stderr, err := runGPG(bytes.NewReader(data),
|
||||||
"--detach-sign",
|
"--detach-sign",
|
||||||
gpgOptArmor,
|
gpgOptArmor,
|
||||||
"--local-user", string(keyID),
|
"--local-user", string(keyID),
|
||||||
@@ -153,8 +120,8 @@ func gpgSign(ctx context.Context, data []byte, keyID GPGKeyID) ([]byte, error) {
|
|||||||
|
|
||||||
// gpgExportPublicKey exports the public key for the specified key ID.
|
// gpgExportPublicKey exports the public key for the specified key ID.
|
||||||
// Returns the armored public key.
|
// Returns the armored public key.
|
||||||
func gpgExportPublicKey(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) {
|
||||||
stdout, stderr, err := runGPG(ctx, nil,
|
stdout, stderr, err := runGPG(nil,
|
||||||
gpgArgs([]string{"--export", gpgOptArmor}, string(keyID))...,
|
gpgArgs([]string{"--export", gpgOptArmor}, string(keyID))...,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -169,8 +136,8 @@ func gpgExportPublicKey(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// gpgGetKeyFingerprint gets the full fingerprint for a key ID.
|
// gpgGetKeyFingerprint gets the full fingerprint for a key ID.
|
||||||
func gpgGetKeyFingerprint(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
func gpgGetKeyFingerprint(keyID GPGKeyID) ([]byte, error) {
|
||||||
stdout, stderr, err := runGPG(ctx, nil,
|
stdout, stderr, err := runGPG(nil,
|
||||||
gpgArgs([]string{"--with-colons", "--fingerprint"}, string(keyID))...,
|
gpgArgs([]string{"--with-colons", "--fingerprint"}, string(keyID))...,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -190,7 +157,7 @@ func gpgGetKeyFingerprint(ctx context.Context, keyID GPGKeyID) ([]byte, error) {
|
|||||||
// gpgExtractPubKeyFingerprint imports a public key into a temporary keyring
|
// gpgExtractPubKeyFingerprint imports a public key into a temporary keyring
|
||||||
// and extracts its fingerprint. This verifies the key is valid and returns
|
// and extracts its fingerprint. This verifies the key is valid and returns
|
||||||
// the actual fingerprint from the key material.
|
// the actual fingerprint from the key material.
|
||||||
func gpgExtractPubKeyFingerprint(ctx context.Context, pubKey []byte) (string, error) {
|
func gpgExtractPubKeyFingerprint(pubKey []byte) (string, error) {
|
||||||
// Create temporary directory for GPG operations
|
// Create temporary directory for GPG operations
|
||||||
tmpDir, err := os.MkdirTemp("", "mfer-gpg-fingerprint-*")
|
tmpDir, err := os.MkdirTemp("", "mfer-gpg-fingerprint-*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -214,7 +181,7 @@ func gpgExtractPubKeyFingerprint(ctx context.Context, pubKey []byte) (string, er
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Import the public key into the temporary keyring
|
// Import the public key into the temporary keyring
|
||||||
_, importStderr, err := runGPG(ctx, nil,
|
_, importStderr, err := runGPG(nil,
|
||||||
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
|
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -224,7 +191,7 @@ func gpgExtractPubKeyFingerprint(ctx context.Context, pubKey []byte) (string, er
|
|||||||
}
|
}
|
||||||
|
|
||||||
// List keys to get fingerprint
|
// List keys to get fingerprint
|
||||||
listStdout, listStderr, err := runGPG(ctx, nil,
|
listStdout, listStderr, err := runGPG(nil,
|
||||||
"--homedir", tmpDir,
|
"--homedir", tmpDir,
|
||||||
"--with-colons",
|
"--with-colons",
|
||||||
"--fingerprint",
|
"--fingerprint",
|
||||||
@@ -245,7 +212,7 @@ func gpgExtractPubKeyFingerprint(ctx context.Context, pubKey []byte) (string, er
|
|||||||
|
|
||||||
// gpgVerify verifies a detached signature against data using the provided public key.
|
// gpgVerify verifies a detached signature against data using the provided public key.
|
||||||
// It creates a temporary keyring to import the public key for verification.
|
// It creates a temporary keyring to import the public key for verification.
|
||||||
func gpgVerify(ctx context.Context, data, signature, pubKey []byte) error {
|
func gpgVerify(data, signature, pubKey []byte) error {
|
||||||
// Create temporary directory for GPG operations
|
// Create temporary directory for GPG operations
|
||||||
tmpDir, err := os.MkdirTemp("", "mfer-gpg-verify-*")
|
tmpDir, err := os.MkdirTemp("", "mfer-gpg-verify-*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -285,7 +252,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Import the public key into the temporary keyring
|
// Import the public key into the temporary keyring
|
||||||
_, importStderr, err := runGPG(ctx, nil,
|
_, importStderr, err := runGPG(nil,
|
||||||
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
|
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -295,7 +262,7 @@ func gpgVerify(ctx context.Context, data, signature, pubKey []byte) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify the signature
|
// Verify the signature
|
||||||
_, verifyStderr, err := runGPG(ctx, nil,
|
_, verifyStderr, err := runGPG(nil,
|
||||||
gpgArgs([]string{gpgOptHomedir, tmpDir, gpgOptVerify},
|
gpgArgs([]string{gpgOptHomedir, tmpDir, gpgOptVerify},
|
||||||
sigFile, dataFile)...,
|
sigFile, dataFile)...,
|
||||||
)
|
)
|
||||||
|
|||||||
+20
-116
@@ -4,15 +4,11 @@ package mfer
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -47,11 +43,8 @@ Expire-Date: 0
|
|||||||
paramsFile := filepath.Join(gpgHome, "key-params")
|
paramsFile := filepath.Join(gpgHome, "key-params")
|
||||||
require.NoError(t, os.WriteFile(paramsFile, []byte(keyParams), 0o600))
|
require.NoError(t, os.WriteFile(paramsFile, []byte(keyParams), 0o600))
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), gpgTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
//nolint:gosec // paramsFile is a test-controlled path inside t.TempDir()
|
//nolint:gosec // paramsFile is a test-controlled path inside t.TempDir()
|
||||||
cmd := exec.CommandContext(ctx, "gpg",
|
cmd := exec.CommandContext(context.Background(), "gpg",
|
||||||
"--batch", "--gen-key", paramsFile)
|
"--batch", "--gen-key", paramsFile)
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
||||||
@@ -62,7 +55,7 @@ Expire-Date: 0
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get the key fingerprint
|
// Get the key fingerprint
|
||||||
cmd = exec.CommandContext(ctx, "gpg",
|
cmd = exec.CommandContext(context.Background(), "gpg",
|
||||||
"--list-keys", "--with-colons", "test@mfer.test")
|
"--list-keys", "--with-colons", "test@mfer.test")
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
||||||
@@ -97,7 +90,7 @@ func TestGPGSign(t *testing.T) {
|
|||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
data := []byte("test data to sign")
|
data := []byte("test data to sign")
|
||||||
sig, err := gpgSign(context.Background(), data, keyID)
|
sig, err := gpgSign(data, keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NotEmpty(t, sig)
|
assert.NotEmpty(t, sig)
|
||||||
assert.Contains(t, string(sig), "-----BEGIN PGP SIGNATURE-----")
|
assert.Contains(t, string(sig), "-----BEGIN PGP SIGNATURE-----")
|
||||||
@@ -108,7 +101,7 @@ func TestGPGExportPublicKey(t *testing.T) {
|
|||||||
keyID, gpgHome := testGPGEnv(t)
|
keyID, gpgHome := testGPGEnv(t)
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(context.Background(), keyID)
|
pubKey, err := gpgExportPublicKey(keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NotEmpty(t, pubKey)
|
assert.NotEmpty(t, pubKey)
|
||||||
assert.Contains(t, string(pubKey), "-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
assert.Contains(t, string(pubKey), "-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
||||||
@@ -119,7 +112,7 @@ func TestGPGGetKeyFingerprint(t *testing.T) {
|
|||||||
keyID, gpgHome := testGPGEnv(t)
|
keyID, gpgHome := testGPGEnv(t)
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
fingerprint, err := gpgGetKeyFingerprint(context.Background(), keyID)
|
fingerprint, err := gpgGetKeyFingerprint(keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NotEmpty(t, fingerprint)
|
assert.NotEmpty(t, fingerprint)
|
||||||
// The fingerprint should be 40 hex chars
|
// The fingerprint should be 40 hex chars
|
||||||
@@ -153,12 +146,12 @@ func TestGPGOptionLikeKeyIDIsNotAnOption(t *testing.T) {
|
|||||||
_, gpgHome := testGPGEnv(t)
|
_, gpgHome := testGPGEnv(t)
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(context.Background(), GPGKeyID("--version"))
|
pubKey, err := gpgExportPublicKey(GPGKeyID("--version"))
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
require.ErrorIs(t, err, errGPGKeyNotFound)
|
require.ErrorIs(t, err, errGPGKeyNotFound)
|
||||||
assert.NotContains(t, string(pubKey), "gpg (GnuPG)")
|
assert.NotContains(t, string(pubKey), "gpg (GnuPG)")
|
||||||
|
|
||||||
fpr, err := gpgGetKeyFingerprint(context.Background(), GPGKeyID("--version"))
|
fpr, err := gpgGetKeyFingerprint(GPGKeyID("--version"))
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.NotContains(t, string(fpr), "gpg (GnuPG)")
|
assert.NotContains(t, string(fpr), "gpg (GnuPG)")
|
||||||
}
|
}
|
||||||
@@ -169,8 +162,7 @@ func TestGPGSignInvalidKey(t *testing.T) {
|
|||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
data := []byte("test data")
|
data := []byte("test data")
|
||||||
_, err := gpgSign(context.Background(), data,
|
_, err := gpgSign(data, GPGKeyID("NONEXISTENT_KEY_ID_12345"))
|
||||||
GPGKeyID("NONEXISTENT_KEY_ID_12345"))
|
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,7 +185,7 @@ func TestBuilderWithSigning(t *testing.T) {
|
|||||||
// Build the manifest
|
// Build the manifest
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err = b.Build(context.Background(), &buf)
|
err = b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Parse the manifest and verify signature fields are populated
|
// Parse the manifest and verify signature fields are populated
|
||||||
@@ -259,14 +251,14 @@ func TestGPGVerify(t *testing.T) {
|
|||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
data := []byte("test data to sign and verify")
|
data := []byte("test data to sign and verify")
|
||||||
sig, err := gpgSign(context.Background(), data, keyID)
|
sig, err := gpgSign(data, keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(context.Background(), keyID)
|
pubKey, err := gpgExportPublicKey(keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Verify the signature
|
// Verify the signature
|
||||||
err = gpgVerify(context.Background(), data, sig, pubKey)
|
err = gpgVerify(data, sig, pubKey)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,15 +267,15 @@ func TestGPGVerifyInvalidSignature(t *testing.T) {
|
|||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
data := []byte("test data to sign")
|
data := []byte("test data to sign")
|
||||||
sig, err := gpgSign(context.Background(), data, keyID)
|
sig, err := gpgSign(data, keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(context.Background(), keyID)
|
pubKey, err := gpgExportPublicKey(keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Try to verify with different data - should fail
|
// Try to verify with different data - should fail
|
||||||
wrongData := []byte("different data")
|
wrongData := []byte("different data")
|
||||||
err = gpgVerify(context.Background(), wrongData, sig, pubKey)
|
err = gpgVerify(wrongData, sig, pubKey)
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -292,12 +284,12 @@ func TestGPGVerifyBadPublicKey(t *testing.T) {
|
|||||||
t.Setenv("GNUPGHOME", gpgHome)
|
t.Setenv("GNUPGHOME", gpgHome)
|
||||||
|
|
||||||
data := []byte("test data")
|
data := []byte("test data")
|
||||||
sig, err := gpgSign(context.Background(), data, keyID)
|
sig, err := gpgSign(data, keyID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Try to verify with invalid public key - should fail
|
// Try to verify with invalid public key - should fail
|
||||||
badPubKey := []byte("not a valid public key")
|
badPubKey := []byte("not a valid public key")
|
||||||
err = gpgVerify(context.Background(), data, sig, badPubKey)
|
err = gpgVerify(data, sig, badPubKey)
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -320,7 +312,7 @@ func TestManifestSignatureVerification(t *testing.T) {
|
|||||||
// Build the manifest
|
// Build the manifest
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err = b.Build(context.Background(), &buf)
|
err = b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Parse the manifest - signature should be verified during load
|
// Parse the manifest - signature should be verified during load
|
||||||
@@ -349,7 +341,7 @@ func TestManifestTamperedSignatureFails(t *testing.T) {
|
|||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err = b.Build(context.Background(), &buf)
|
err = b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Tamper with the signature by replacing some bytes
|
// Tamper with the signature by replacing some bytes
|
||||||
@@ -383,7 +375,7 @@ func TestBuilderWithoutSigning(t *testing.T) {
|
|||||||
// Build the manifest
|
// Build the manifest
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
err = b.Build(context.Background(), &buf)
|
err = b.Build(&buf)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Parse the manifest and verify signature fields are empty
|
// Parse the manifest and verify signature fields are empty
|
||||||
@@ -398,91 +390,3 @@ func TestBuilderWithoutSigning(t *testing.T) {
|
|||||||
assert.Empty(t, manifest.pbOuter.GetSigningPubKey(),
|
assert.Empty(t, manifest.pbOuter.GetSigningPubKey(),
|
||||||
"signing public key should be empty when not signing")
|
"signing public key should be empty when not signing")
|
||||||
}
|
}
|
||||||
|
|
||||||
// fakeGPGPath writes script as an executable named gpg into a temporary
|
|
||||||
// directory and returns a PATH value with that directory first.
|
|
||||||
func fakeGPGPath(t *testing.T, script string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
binDir := t.TempDir()
|
|
||||||
//nolint:gosec // G306: the fake gpg has to be executable
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(binDir, "gpg"),
|
|
||||||
[]byte(script), 0o700))
|
|
||||||
|
|
||||||
return binDir + string(os.PathListSeparator) + os.Getenv("PATH")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGPGTimeoutKillsGPG puts a fake gpg that never finishes first on
|
|
||||||
// PATH and checks that a run past its deadline is killed and reported as
|
|
||||||
// a timeout of the named operation, instead of hanging.
|
|
||||||
func TestGPGTimeoutKillsGPG(t *testing.T) {
|
|
||||||
t.Setenv("PATH", fakeGPGPath(t, "#!/bin/sh\nexec sleep 10\n"))
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
_, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
|
||||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
|
||||||
assert.Contains(t, err.Error(), "gpg sign failed: gpg timed out")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGPGCancelWhenChildHoldsOutput uses a fake gpg that runs sleep as a
|
|
||||||
// child instead of exec-ing it, the way a wrapper script around the real
|
|
||||||
// gpg might. Killing the fake gpg leaves sleep holding its stdout and
|
|
||||||
// stderr open; the call must still return once ctx ends instead of waiting
|
|
||||||
// for sleep to exit. The fake gpg writes the process ID of sleep to a named
|
|
||||||
// pipe; the test ends ctx only after reading it, so sleep is running by
|
|
||||||
// then, and kills sleep before returning.
|
|
||||||
func TestGPGCancelWhenChildHoldsOutput(t *testing.T) {
|
|
||||||
pidPipe := filepath.Join(t.TempDir(), "sleep.pid")
|
|
||||||
require.NoError(t, syscall.Mkfifo(pidPipe, 0o600))
|
|
||||||
// sleep outlasts the 10 s wait below, so a call that waits for it fails.
|
|
||||||
t.Setenv("PATH", fakeGPGPath(t,
|
|
||||||
"#!/bin/sh\nsleep 60 &\necho $! >'"+pidPipe+"'\nwait\n"))
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
signErr := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
_, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
|
||||||
signErr <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
pid, err := os.ReadFile(pidPipe) //nolint:gosec // G304: path inside t.TempDir()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
n, err := strconv.Atoi(strings.TrimSpace(string(pid)))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
sleep, err := os.FindProcess(n)
|
|
||||||
require.NoError(t, err)
|
|
||||||
t.Cleanup(func() { require.NoError(t, sleep.Kill()) })
|
|
||||||
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
// The call should return about gpgWaitDelay (one second) after the
|
|
||||||
// cancel. 10 s is far above that and well under the 30 s test timeout,
|
|
||||||
// which would abort the whole package before the cleanup kills sleep.
|
|
||||||
select {
|
|
||||||
case err := <-signErr:
|
|
||||||
require.ErrorIs(t, err, context.Canceled)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("the call waited for the child holding gpg's output to exit")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBuildPassesContextToSigning checks that a caller can cancel the gpg
|
|
||||||
// runs that sign a manifest through the context given to Build.
|
|
||||||
func TestBuildPassesContextToSigning(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
b.SetSigningOptions(&SigningOptions{KeyID: "any"})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
require.ErrorIs(t, b.Build(ctx, io.Discard), context.Canceled)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
103901c42b94396aa7ae128fd503ef693a4b7a03b2169481f25fda3d2c254e00 mf.proto
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package mfer_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// mf.pb.go is generated from mf.proto and committed. `make generate`
|
|
||||||
// records the hash of the mf.proto it generated from in mf.proto.sha256.
|
|
||||||
func TestGeneratedCodeMatchesProto(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
proto, err := os.ReadFile("mf.proto")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
recorded, err := os.ReadFile("mf.proto.sha256")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
recordedHash, _, _ := strings.Cut(string(recorded), " ")
|
|
||||||
sum := sha256.Sum256(proto)
|
|
||||||
require.Equal(t, recordedHash, hex.EncodeToString(sum[:]),
|
|
||||||
"mfer/mf.proto has changed since mfer/mf.pb.go was generated "+
|
|
||||||
"from it: run `make generate` and commit the result")
|
|
||||||
}
|
|
||||||
+3
-23
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -58,8 +57,6 @@ type ScannerOptions struct {
|
|||||||
SigningOptions *SigningOptions
|
SigningOptions *SigningOptions
|
||||||
// Seed, if set, derives a deterministic UUID from this seed.
|
// Seed, if set, derives a deterministic UUID from this seed.
|
||||||
Seed string
|
Seed string
|
||||||
// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
|
|
||||||
ExcludePaths []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileEntry represents a file that has been enumerated.
|
// FileEntry represents a file that has been enumerated.
|
||||||
@@ -78,7 +75,6 @@ type Scanner struct {
|
|||||||
totalBytes FileSize // cached sum of all file sizes
|
totalBytes FileSize // cached sum of all file sizes
|
||||||
options *ScannerOptions
|
options *ScannerOptions
|
||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
excluded []fs.FileInfo // the files named in ExcludePaths that exist
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScanner creates a new Scanner with default options.
|
// NewScanner creates a new Scanner with default options.
|
||||||
@@ -97,22 +93,11 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|||||||
fs = afero.NewOsFs()
|
fs = afero.NewOsFs()
|
||||||
}
|
}
|
||||||
|
|
||||||
s := &Scanner{
|
return &Scanner{
|
||||||
files: make([]*FileEntry, 0),
|
files: make([]*FileEntry, 0),
|
||||||
options: opts,
|
options: opts,
|
||||||
fs: fs,
|
fs: fs,
|
||||||
}
|
}
|
||||||
|
|
||||||
// A path that cannot be stat'd, normally because no file is there,
|
|
||||||
// needs no leaving out.
|
|
||||||
for _, p := range opts.ExcludePaths {
|
|
||||||
info, err := s.fs.Stat(p)
|
|
||||||
if err == nil {
|
|
||||||
s.excluded = append(s.excluded, info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
||||||
@@ -298,7 +283,8 @@ func (s *Scanner) ToManifest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Build and write manifest
|
// Build and write manifest
|
||||||
return builder.Build(ctx, w)
|
//nolint:contextcheck // Build's GPG signing exec is not cancellable by design
|
||||||
|
return builder.Build(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
// configureBuilder constructs a manifest builder configured from the
|
// configureBuilder constructs a manifest builder configured from the
|
||||||
@@ -450,12 +436,6 @@ func (s *Scanner) enumerateFileWithInfo(
|
|||||||
info = realInfo
|
info = realInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, excluded := range s.excluded {
|
|
||||||
if os.SameFile(info, excluded) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := &FileEntry{
|
entry := &FileEntry{
|
||||||
Path: RelFilePath(cleanPath),
|
Path: RelFilePath(cleanPath),
|
||||||
AbsPath: AbsFilePath(absPath),
|
AbsPath: AbsFilePath(absPath),
|
||||||
|
|||||||
+29
-2
@@ -304,19 +304,46 @@ func TestScannerEnumerateFS(t *testing.T) {
|
|||||||
func TestSendEnumerateStatusNonBlocking(t *testing.T) {
|
func TestSendEnumerateStatusNonBlocking(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Nobody receives, so a blocking send would hang the test into its timeout.
|
// Channel with no buffer - send should not block
|
||||||
ch := make(chan EnumerateStatus)
|
ch := make(chan EnumerateStatus)
|
||||||
|
|
||||||
|
// This should not block
|
||||||
|
done := make(chan bool)
|
||||||
|
|
||||||
|
go func() {
|
||||||
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
|
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
|
||||||
|
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success - did not block
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
t.Fatal("sendEnumerateStatus blocked on full channel")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSendScanStatusNonBlocking(t *testing.T) {
|
func TestSendScanStatusNonBlocking(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Nobody receives, so a blocking send would hang the test into its timeout.
|
// Channel with no buffer - send should not block
|
||||||
ch := make(chan ScanStatus)
|
ch := make(chan ScanStatus)
|
||||||
|
|
||||||
|
done := make(chan bool)
|
||||||
|
|
||||||
|
go func() {
|
||||||
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
|
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
|
||||||
|
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Success - did not block
|
||||||
|
case <-time.After(100 * time.Millisecond):
|
||||||
|
t.Fatal("sendScanStatus blocked on full channel")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSendStatusNilChannel(t *testing.T) {
|
func TestSendStatusNilChannel(t *testing.T) {
|
||||||
|
|||||||
+8
-9
@@ -2,7 +2,6 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -51,13 +50,13 @@ func newTimestampFromTime(t time.Time) *Timestamp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifest) generate(ctx context.Context) error {
|
func (m *manifest) generate() error {
|
||||||
if m.pbInner == nil {
|
if m.pbInner == nil {
|
||||||
return errInnerNotSet
|
return errInnerNotSet
|
||||||
}
|
}
|
||||||
|
|
||||||
if m.pbOuter == nil {
|
if m.pbOuter == nil {
|
||||||
e := m.generateOuter(ctx)
|
e := m.generateOuter()
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
@@ -78,7 +77,7 @@ func (m *manifest) generate(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifest) generateOuter(ctx context.Context) error {
|
func (m *manifest) generateOuter() error {
|
||||||
if m.pbInner == nil {
|
if m.pbInner == nil {
|
||||||
return errInternal
|
return errInternal
|
||||||
}
|
}
|
||||||
@@ -136,7 +135,7 @@ func (m *manifest) generateOuter(ctx context.Context) error {
|
|||||||
|
|
||||||
// Sign the manifest if signing options are provided
|
// Sign the manifest if signing options are provided
|
||||||
if m.signingOptions != nil && m.signingOptions.KeyID != "" {
|
if m.signingOptions != nil && m.signingOptions.KeyID != "" {
|
||||||
return m.signOuter(ctx)
|
return m.signOuter()
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -144,27 +143,27 @@ func (m *manifest) generateOuter(ctx context.Context) error {
|
|||||||
|
|
||||||
// signOuter signs the outer message with the configured GPG key and
|
// signOuter signs the outer message with the configured GPG key and
|
||||||
// embeds the signature, signer fingerprint, and public key.
|
// embeds the signature, signer fingerprint, and public key.
|
||||||
func (m *manifest) signOuter(ctx context.Context) error {
|
func (m *manifest) signOuter() error {
|
||||||
sigString, err := m.signatureString()
|
sigString, err := m.signatureString()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to generate signature string: %w", err)
|
return fmt.Errorf("failed to generate signature string: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sig, err := gpgSign(ctx, []byte(sigString), m.signingOptions.KeyID)
|
sig, err := gpgSign([]byte(sigString), m.signingOptions.KeyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to sign manifest: %w", err)
|
return fmt.Errorf("failed to sign manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.pbOuter.Signature = sig
|
m.pbOuter.Signature = sig
|
||||||
|
|
||||||
fingerprint, err := gpgGetKeyFingerprint(ctx, m.signingOptions.KeyID)
|
fingerprint, err := gpgGetKeyFingerprint(m.signingOptions.KeyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to get key fingerprint: %w", err)
|
return fmt.Errorf("failed to get key fingerprint: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.pbOuter.Signer = fingerprint
|
m.pbOuter.Signer = fingerprint
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(ctx, m.signingOptions.KeyID)
|
pubKey, err := gpgExportPublicKey(m.signingOptions.KeyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to export public key: %w", err)
|
return fmt.Errorf("failed to export public key: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\xff\xff\xff\x03\xc2\x06 {\x16\xbdu\xa0\xa2\x11\xfcH\xef*\x1b7\r\x99\xefb\x04\x02g\n\xa9\xf3B5\xe5p\x96\x8c\x8c\xac\x0e\xca\x06\x10\x03Q\xb2\xd0\x19`F\xc1\xb1\xc0Z\xf4x\xf4g^\xba\f\xa1\x06(\xb5/\xfd\x04h\x04\x01\x00d\x01\xb2\x06\x10\x03Q\xb2\xd0\x19`F\xc1\xb1\xc0Z\xf4x\xf4g^\xaa\x06\x00\x01T\x13\x024\xce\xff\rL\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15M\x00\x00\x00\x01T\x00\x044\xfc\xff\x153\xea\a\xb4")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\xff\xff\xff\x03\xc2\x06 .\xcd\x11|0\xfcP\xe5\x1b\xe3\xc6Ӡ\xcdڤx\xcd\x169t\x1a9~ǽB\xc9\xe8G`\x05\xca\x06\x10\x11*!\x0e\x95EF\xb8\xbd\x9f\xde\x12MF\r\x99\xba\f\xa6\x06(\xb5/\xfd\x04h,\x01\x00\xb4\x01\xb2\x06\x10\x11*!\x0e\x95EF\xb8\xbd\x9f\xde\x12MF\r\x99\xaa\x06\xe6\xff\xff\x03\x1a\x00\x01T\x14\x024\x8b\xff\x17L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15L\x00\x00\x00\x01T\x00\x044\xfd\xff\x15M\x00\x00\x00\x01T\x00\x044\xfc\xff\x15\x02\xd1.\xe3")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFGy[i\x04\xe5O\x82A\x1d\xf4\xb0\xe2z7:U\xee\xa3\xf9\xd6m\xacZ\x9b\xce\x1d\xd9/{@\x1d\xa5y[i\x04\xe5O\x82A\x1d\xf4\xb0\xe2z7:U\xee\xa3\xf9\xd6m\xacZ\x9b\xce\x1d\xd9/{@\x1d\xa5")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG\x80")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG\x80\xca\f\xe8\x03-----BEGIN PGP SIGNATURE-----\n\niQEzBAABCgAdFiEET1Yr+4Y/3GtRtO6IhypRF2zvI64FAmrBIXAACgkQhypRF2zv\nI67BQAf/QrpX2MjY15YGMGkjR5oIhnx/YV96aGYZyZThzb+l/R/N75iVFVkhX21d\nZhQqdCsORrodTPAXic2g2UGVXP9PhNMh7n6Wm3LsvQYjrRQGrQnqtCkut+3tUt8K\n7pt4OAnnwRSieaVImA1COmzxIrQQKNOs6UkgmAstGuPV0XZoeDiSG8TUYJ/vieCn\np5hC0FFXtzfw4NtkxSmkewE0xBxIwFCA/RfSHCGH3m5K+tRz41vMEgGbL1iEp6+V\nuBaoEc4hqCgEt+Af2pA8VHfqeu2vKiwggOpYpaILXZKVqH9+tWHL1EBv9t0vTsYE\n9D57euuR9+kOdngYNPieP1yn5dOSHg==\n=kvgL\n-----END PGP SIGNATURE-----\n\xd2\f(4F562BFB863FDC6B51B4EE88872A51176CEF23AE\xda\f\xb5\a-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQENBGrBIW8BCADESetN5EdxIe7Fafgxl99Yoo5cOexf7wJyYT0wfUYlRaxt3neR\nhir7LOfH4PZWWoDx7qghxCS4+vs7yGypl6JOm7jnJlhn4HneDa2zeIlgGW2TamyE\nua9KPWBQqkFOYmKPmzp+KnL6ncnBLR5mDkNKFyON812KVvteu6Dp/DNk4Meufe44\nWWr49LSFZa9gEbmRCoQGKby9F0H0yIi4FAc74VdQudy0+fMKcfkKjEvByMzlbBEK\n92Hq3sRFzWd3kvPliNjZTmlh5n5m9aBhMpoy3GkKy8gpDdFc6NLA9iAJe7oNMriR\nkVoa5EjQL1xCXAiAWTYA9NScFfU/574sCTxZABEBAAG0Hk1GRVIgVGVzdCBLZXkg\nPHRlc3RAbWZlci50ZXN0PokBTwQTAQoAORYhBE9WK/uGP9xrUbTuiIcqURds7yOu\nBQJqwSFvAxsvBAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCHKlEXbO8jrjml\nCAC8wUK9wmvxq0+NZUpFyP+P29klLZYzBDaBrLPJFs0GjnG4kvfUAktWx0Ro80F7\ncjTJ4f44XjDj4glvSjbe2VaDnZl9FTfzUfG+xjD4462NgntQ4fHk/uG4F6d1ikWx\nkEoMpIn1PlSMas1jTQSGlxUr+zFwWuUbGq4n6hRxEnwLlwJlwQt/Aw1vPDYuPDE3\nOYDhJIAJyP+6e9W8ToaAG9byg/22KA1u1qxnNQqsx5Tped2VltAzdYub+yeCuNc8\nIUo5ILo/fQq3GM5sUEaHjPolv88WlDm3vcdbSbDoh5m2inDtg5zuUKJwu32UGu8l\nKoTjp4nxgQGy5WmeBzs4Hdm/\n=TCB8\n-----END PGP PUBLIC KEY BLOCK-----\n")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06!\xc2\x06 \x91\x90*\xa5>\fݐ \x87\xbeaL\xc1\x05?\x0eR\xc18\xa4eՕ\xa95\xb9KʺoZ\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\f-(\xb5/\xfd\x04\x00\x01\x01\x00\xa0\x06\x01\xaa\x06\a\n\x05a.txt\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3a[k'")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\x1f\xc2\x06 \x91\x90*\xa5>\fݐ \x87\xbeaL\xc1\x05?\x0eR\xc18\xa4eՕ\xa95\xb9KʺoZ\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\f-(\xb5/\xfd\x04\x00\x01\x01\x00\xa0\x06\x01\xaa\x06\a\n\x05a.txt\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3a[k'")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 ")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAV")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFX\xa8\x06\x01\xb0\x06\x01\xb8\x06V\xc2\x06 \xa3\xf7\x97\xa7\xf3\x87:\x90)\\ӊj\xb9\xf7\xfaTJ\x1b\xe7:\xee\xbe\"V\xe0:\x8d(Z\xd6%\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\fc(\xb5/\xfd\x04\x00\xb1\x02\x00\xa0\x06\x01\xaa\x06=\n\x05a.txt\x10\x01\x1a$\n\"\x12 ʗ\x81\x12\xca\x1b\xbd\xca\xfa\xc21\xb3\x9a#\xdcM\xa7\x86\xef\xf8\x14|Nr\xb9\x80w\x85\xaf\xeeH\xbb\xf2\x12\v\b\x80\x92\xb8Ø\xfe\xff\xff\xff\x01\xb2\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3s\xeeG\x80")
|
|
||||||
File diff suppressed because one or more lines are too long
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\x01\xc2\x06 \xd6aQ\xa4\x85\xc0+\xbb\xca\x11\x11\a<\x019\x97\xb3\xbb3\xd0 \xd5U\xfa!\xaeAf<N@\x9d\xca\x06\x10\x93\x85\vpu\x85\xe4\x04\xe4\x95\x1a=\xdc\x1f\x05\xa3\xba\f\x12(\xb5/\xfd\xc0\x00\x00\x00\x00\x00\x02\x00\x00\x00\v\x00\x00\x00")
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
go test fuzz v1
|
|
||||||
[]byte("ZNAVSRFG\xa8\x06\x01\xb0\x06\x01\xb8\x06\x01\xc2\x06 \xc8g?\xa0\xc9\xc5]\xb57M\xe5O#\x04\xb2\x12\xc6)@=\xd2\xf3f/͉~\x10\x15\xfbkD\xca\x06\x10o\x1c*N;]L~\x9a\x8b\x1d.?@Qb\xba\f\x89\t(\xb5/\xfd\x00\x00\x01\x00\x00(\xb5/\xfd\x00\x01\x01\x00\x00(\xb5/\xfd\x00\x02\x01\x00\x00(\xb5/\xfd\x00\x03\x01\x00\x00(\xb5/\xfd\x00\x04\x01\x00\x00(\xb5/\xfd\x00\x05\x01\x00\x00(\xb5/\xfd\x00\x06\x01\x00\x00(\xb5/\xfd\x00\a\x01\x00\x00(\xb5/\xfd\x00\b\x01\x00\x00(\xb5/\xfd\x00\t\x01\x00\x00(\xb5/\xfd\x00\n\x01\x00\x00(\xb5/\xfd\x00\v\x01\x00\x00(\xb5/\xfd\x00\f\x01\x00\x00(\xb5/\xfd\x00\r\x01\x00\x00(\xb5/\xfd\x00\x0e\x01\x00\x00(\xb5/\xfd\x00\x0f\x01\x00\x00(\xb5/\xfd\x00\x10\x01\x00\x00(\xb5/\xfd\x00\x11\x01\x00\x00(\xb5/\xfd\x00\x12\x01\x00\x00(\xb5/\xfd\x00\x13\x01\x00\x00(\xb5/\xfd\x00\x14\x01\x00\x00(\xb5/\xfd\x00\x15\x01\x00\x00(\xb5/\xfd\x00\x16\x01\x00\x00(\xb5/\xfd\x00\x17\x01\x00\x00(\xb5/\xfd\x00\x18\x01\x00\x00(\xb5/\xfd\x00\x19\x01\x00\x00(\xb5/\xfd\x00\x1a\x01\x00\x00(\xb5/\xfd\x00\x1b\x01\x00\x00(\xb5/\xfd\x00\x1c\x01\x00\x00(\xb5/\xfd\x00\x1d\x01\x00\x00(\xb5/\xfd\x00\x1e\x01\x00\x00(\xb5/\xfd\x00\x1f\x01\x00\x00(\xb5/\xfd\x00 \x01\x00\x00(\xb5/\xfd\x00!\x01\x00\x00(\xb5/\xfd\x00\"\x01\x00\x00(\xb5/\xfd\x00#\x01\x00\x00(\xb5/\xfd\x00$\x01\x00\x00(\xb5/\xfd\x00%\x01\x00\x00(\xb5/\xfd\x00&\x01\x00\x00(\xb5/\xfd\x00'\x01\x00\x00(\xb5/\xfd\x00(\x01\x00\x00(\xb5/\xfd\x00)\x01\x00\x00(\xb5/\xfd\x00*\x01\x00\x00(\xb5/\xfd\x00+\x01\x00\x00(\xb5/\xfd\x00,\x01\x00\x00(\xb5/\xfd\x00-\x01\x00\x00(\xb5/\xfd\x00.\x01\x00\x00(\xb5/\xfd\x00/\x01\x00\x00(\xb5/\xfd\x000\x01\x00\x00(\xb5/\xfd\x001\x01\x00\x00(\xb5/\xfd\x002\x01\x00\x00(\xb5/\xfd\x003\x01\x00\x00(\xb5/\xfd\x004\x01\x00\x00(\xb5/\xfd\x005\x01\x00\x00(\xb5/\xfd\x006\x01\x00\x00(\xb5/\xfd\x007\x01\x00\x00(\xb5/\xfd\x008\x01\x00\x00(\xb5/\xfd\x009\x01\x00\x00(\xb5/\xfd\x00:\x01\x00\x00(\xb5/\xfd\x00;\x01\x00\x00(\xb5/\xfd\x00<\x01\x00\x00(\xb5/\xfd\x00=\x01\x00\x00(\xb5/\xfd\x00>\x01\x00\x00(\xb5/\xfd\x00?\x01\x00\x00(\xb5/\xfd\x00@\x01\x00\x00(\xb5/\xfd\x00A\x01\x00\x00(\xb5/\xfd\x00B\x01\x00\x00(\xb5/\xfd\x00C\x01\x00\x00(\xb5/\xfd\x00D\x01\x00\x00(\xb5/\xfd\x00E\x01\x00\x00(\xb5/\xfd\x00F\x01\x00\x00(\xb5/\xfd\x00G\x01\x00\x00(\xb5/\xfd\x00H\x01\x00\x00(\xb5/\xfd\x00I\x01\x00\x00(\xb5/\xfd\x00J\x01\x00\x00(\xb5/\xfd\x00K\x01\x00\x00(\xb5/\xfd\x00L\x01\x00\x00(\xb5/\xfd\x00M\x01\x00\x00(\xb5/\xfd\x00N\x01\x00\x00(\xb5/\xfd\x00O\x01\x00\x00(\xb5/\xfd\x00P\x01\x00\x00(\xb5/\xfd\x00Q\x01\x00\x00(\xb5/\xfd\x00R\x01\x00\x00(\xb5/\xfd\x00S\x01\x00\x00(\xb5/\xfd\x00T\x01\x00\x00(\xb5/\xfd\x00U\x01\x00\x00(\xb5/\xfd\x00V\x01\x00\x00(\xb5/\xfd\x00W\x01\x00\x00(\xb5/\xfd\x00X\x01\x00\x00(\xb5/\xfd\x00Y\x01\x00\x00(\xb5/\xfd\x00Z\x01\x00\x00(\xb5/\xfd\x00[\x01\x00\x00(\xb5/\xfd\x00\\\x01\x00\x00(\xb5/\xfd\x00]\x01\x00\x00(\xb5/\xfd\x00^\x01\x00\x00(\xb5/\xfd\x00_\x01\x00\x00(\xb5/\xfd\x00`\x01\x00\x00(\xb5/\xfd\x00a\x01\x00\x00(\xb5/\xfd\x00b\x01\x00\x00(\xb5/\xfd\x00c\x01\x00\x00(\xb5/\xfd\x00d\x01\x00\x00(\xb5/\xfd\x00e\x01\x00\x00(\xb5/\xfd\x00f\x01\x00\x00(\xb5/\xfd\x00g\x01\x00\x00(\xb5/\xfd\x00h\x01\x00\x00(\xb5/\xfd\x00i\x01\x00\x00(\xb5/\xfd\x00j\x01\x00\x00(\xb5/\xfd\x00k\x01\x00\x00(\xb5/\xfd\x00l\x01\x00\x00(\xb5/\xfd\x00m\x01\x00\x00(\xb5/\xfd\x00n\x01\x00\x00(\xb5/\xfd\x00o\x01\x00\x00(\xb5/\xfd\x00p\x01\x00\x00(\xb5/\xfd\x00q\x01\x00\x00(\xb5/\xfd\x00r\x01\x00\x00(\xb5/\xfd\x00s\x01\x00\x00(\xb5/\xfd\x00t\x01\x00\x00(\xb5/\xfd\x00u\x01\x00\x00(\xb5/\xfd\x00v\x01\x00\x00(\xb5/\xfd\x00w\x01\x00\x00(\xb5/\xfd\x00x\x01\x00\x00(\xb5/\xfd\x00y\x01\x00\x00(\xb5/\xfd\x00z\x01\x00\x00(\xb5/\xfd\x00{\x01\x00\x00(\xb5/\xfd\x00|\x01\x00\x00(\xb5/\xfd\x00}\x01\x00\x00(\xb5/\xfd\x00~\x01\x00\x00(\xb5/\xfd\x00\x7f\x01\x00\x00(\xb5/\xfd\x00\x80\x01\x00\x00")
|
|
||||||
File diff suppressed because one or more lines are too long
+6
-1
@@ -140,7 +140,12 @@ main() {
|
|||||||
|
|
||||||
# ---- Go repos ----
|
# ---- Go repos ----
|
||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
# No golangci-lint: script/lint runs it in Docker only.
|
# golangci-lint: packaged in nix, brew, and apk. On apt there is no
|
||||||
|
# package: download a specific release archive from GitHub and
|
||||||
|
# verify its hash (verify_sha256), never curl | sh.
|
||||||
|
if missing golangci-lint; then
|
||||||
|
pkg_install golangci-lint golangci-lint golangci-lint golangci-lint
|
||||||
|
fi
|
||||||
go mod download
|
go mod download
|
||||||
|
|
||||||
# ---- Python repos ----
|
# ---- Python repos ----
|
||||||
|
|||||||
+5
-15
@@ -1,24 +1,14 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/cibuild: run the CI build; the Gitea workflow runs this on push.
|
# script/cibuild: run the CI build. The Dockerfile runs script/check
|
||||||
# It builds the image with the same command as script/docker. --no-cache
|
# (via make check), so a successful build implies all checks pass.
|
||||||
# because the checks the final stage depends on are RUN steps, and a
|
# Generic: needs no adaptation. The Gitea workflow runs this on push.
|
||||||
# cached one is a check that did not run.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
# Own line: a failing command substitution inside an argument does
|
docker build .
|
||||||
# not trip `set -e`, so the inline form degrades silently to an
|
|
||||||
# empty constant. The VERSION build argument takes precedence over
|
|
||||||
# the version a build stage derives from the .git in the context.
|
|
||||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
|
||||||
[ -n "$version" ] || version="unknown"
|
|
||||||
docker build --no-cache \
|
|
||||||
--build-arg VERSION="$version" \
|
|
||||||
-t "$("$SCRIPT_DIR/projectname")" .
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+12
@@ -5,9 +5,21 @@ set -eu
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||||
|
# (mirrors the old Makefile prerequisite; the generated file is
|
||||||
|
# committed, so this is normally a no-op).
|
||||||
|
ensure_pb() {
|
||||||
|
if [ ! -f mfer/mf.pb.go ] ||
|
||||||
|
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||||
|
(cd mfer && go generate .)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
ensure_pb
|
||||||
gofumpt -l -w mfer internal cmd
|
gofumpt -l -w mfer internal cmd
|
||||||
|
golangci-lint run --fix
|
||||||
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
||||||
"$SCRIPT_DIR/prettier" --write
|
"$SCRIPT_DIR/prettier" --write
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,8 +6,19 @@ set -eu
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||||
|
# (mirrors the old Makefile prerequisite; the generated file is
|
||||||
|
# committed, so this is normally a no-op).
|
||||||
|
ensure_pb() {
|
||||||
|
if [ ! -f mfer/mf.pb.go ] ||
|
||||||
|
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||||
|
(cd mfer && go generate .)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
ensure_pb
|
||||||
if [ -n "$(gofmt -l .)" ]; then
|
if [ -n "$(gofmt -l .)" ]; then
|
||||||
echo "gofmt: files need formatting:" >&2
|
echo "gofmt: files need formatting:" >&2
|
||||||
gofmt -l . >&2
|
gofmt -l . >&2
|
||||||
|
|||||||
-17
@@ -1,17 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/fuzz: fuzz the manifest parser for one minute. Run by hand only:
|
|
||||||
# script/test already runs the committed seed corpus as ordinary tests,
|
|
||||||
# and CI never fuzzes. An input that fails is written to
|
|
||||||
# mfer/testdata/fuzz/FuzzNewManifestFromReader/; once the parser is fixed,
|
|
||||||
# commit it there as a regression seed.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
go test -run '^$' -fuzz '^FuzzNewManifestFromReader$' \
|
|
||||||
-fuzztime 1m -parallel 2 ./mfer
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/generate: regenerate mfer/mf.pb.go from mfer/mf.proto, and record
|
|
||||||
# the hash of that mf.proto in mfer/mf.proto.sha256. Nothing else
|
|
||||||
# regenerates mf.pb.go: it is committed, so building and checking need no
|
|
||||||
# protoc. A test fails while mf.proto no longer matches the recorded hash.
|
|
||||||
#
|
|
||||||
# Needs exactly the protoc and protoc-gen-go versions named in the header of
|
|
||||||
# the committed mf.pb.go (README.md says how to install them). Another
|
|
||||||
# version writes a different mf.pb.go, so the script refuses to run.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
|
|
||||||
PROTOC_VERSION="33.4"
|
|
||||||
PROTOC_GEN_GO_VERSION="v1.36.11"
|
|
||||||
|
|
||||||
# require_version <command> <its exact --version output>
|
|
||||||
require_version() {
|
|
||||||
actual="$("$1" --version 2>/dev/null || true)"
|
|
||||||
if [ "$actual" != "$2" ]; then
|
|
||||||
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
|
|
||||||
echo " README.md says how to install it." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
|
|
||||||
# where there is no sha256sum.
|
|
||||||
sha256() {
|
|
||||||
if command -v sha256sum >/dev/null 2>&1; then
|
|
||||||
sha256sum "$1"
|
|
||||||
elif command -v shasum >/dev/null 2>&1; then
|
|
||||||
shasum -a 256 "$1"
|
|
||||||
else
|
|
||||||
echo "generate: needs sha256sum or shasum on PATH" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT/mfer"
|
|
||||||
require_version protoc "libprotoc $PROTOC_VERSION"
|
|
||||||
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
|
|
||||||
# Hashed before regenerating, so a missing hash tool stops the script
|
|
||||||
# before it changes anything. Regenerating leaves mf.proto as it is.
|
|
||||||
proto_hash="$(sha256 mf.proto)"
|
|
||||||
go generate .
|
|
||||||
echo "$proto_hash" >mf.proto.sha256
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
+8
-11
@@ -1,20 +1,17 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/lint: run golangci-lint, in Docker only. Builds the lint stage of
|
# script/lint: run the linter.
|
||||||
# the Dockerfile, whose build runs the linter, so a successful build is a
|
|
||||||
# clean lint. --no-cache because a cached build runs no linter. The image
|
|
||||||
# is removed afterwards, whatever the outcome.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
# Tagged per run, so concurrent runs never remove each other's image.
|
golangci-lint run
|
||||||
image="$("$SCRIPT_DIR/projectname")-lint:$$"
|
if [ -n "$(gofmt -l .)" ]; then
|
||||||
# A failed build leaves no image, so there is nothing to remove then.
|
echo "gofmt: files need formatting:" >&2
|
||||||
trap 'docker image rm "$image" >/dev/null 2>&1 || true' EXIT INT TERM
|
gofmt -l . >&2
|
||||||
docker build --no-cache --target lint -t "$image" .
|
exit 1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+11
@@ -4,8 +4,19 @@ set -eu
|
|||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||||
|
# (mirrors the old Makefile prerequisite; the generated file is
|
||||||
|
# committed, so this is normally a no-op).
|
||||||
|
ensure_pb() {
|
||||||
|
if [ ! -f mfer/mf.pb.go ] ||
|
||||||
|
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||||
|
(cd mfer && go generate .)
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
ensure_pb
|
||||||
go test -timeout 30s -race -cover ./... ||
|
go test -timeout 30s -race -cover ./... ||
|
||||||
{
|
{
|
||||||
echo "--- Rerunning with -v for details ---"
|
echo "--- Rerunning with -v for details ---"
|
||||||
|
|||||||
Reference in New Issue
Block a user