Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7331e8d11 |
+1
-1
@@ -8,7 +8,7 @@ vendor.tzst
|
||||
modcache.tzst
|
||||
|
||||
# Generated manifest files
|
||||
/index.mf
|
||||
.index.mf
|
||||
|
||||
# Secrets
|
||||
.env
|
||||
|
||||
+8
-9
@@ -8,12 +8,13 @@ RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
||||
RUN touch mfer/mf.pb.go
|
||||
|
||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||
# markdown half runs in the mdfmt stage below.
|
||||
RUN make fmt-check-go
|
||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
||||
# there is no docker inside this build.
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
RUN make lint
|
||||
|
||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||
@@ -43,6 +44,9 @@ RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
||||
RUN touch mfer/mf.pb.go
|
||||
|
||||
RUN make test
|
||||
|
||||
# A build context sent as a tar archive, as upaas sends it, keeps its files'
|
||||
@@ -63,13 +67,8 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
cd cmd/mfer && \
|
||||
CGO_ENABLED=0 go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
|
||||
|
||||
# Fail unless /mfer is statically linked: scratch has no C library to run it.
|
||||
RUN ldd /mfer 2>&1 | grep -q 'not a dynamic executable'
|
||||
go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
|
||||
|
||||
FROM scratch
|
||||
# scratch has no CA certificates; fetch needs them to verify HTTPS servers.
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=builder /mfer /mfer
|
||||
ENTRYPOINT ["/mfer"]
|
||||
|
||||
@@ -49,9 +49,7 @@ The `innerMessage` field is compressed with
|
||||
enforce a decompression size limit to prevent decompression bombs. The reference
|
||||
implementation limits decompressed size to 256 MB. It writes zstd frames with a
|
||||
window of at most 8 MiB, the largest window the zstd format recommends decoders
|
||||
support, and refuses frames that ask for a larger one. It also refuses an inner
|
||||
message whose file entries, hashes, timestamps and MIME types, counted at 160,
|
||||
112, 64 and 16 bytes each, add up to more than 8 times its size.
|
||||
support, and refuses frames that ask for a larger one.
|
||||
|
||||
## Inner Message (`MFFile`)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ export DOCKER_BUILDKIT := 1
|
||||
export PROGRESS_NO_TRUNC := 1
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
export PATH := $(PATH):$(GOPATH)/bin
|
||||
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
|
||||
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
||||
ARCH := $(shell uname -m)
|
||||
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
|
||||
@@ -12,7 +13,7 @@ GOLDFLAGS += -X main.Version=$(VERSION)
|
||||
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||
|
||||
.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme generate
|
||||
.PHONY: bootstrap setup docker default run ci test fuzz check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme
|
||||
|
||||
default: fmt test
|
||||
|
||||
@@ -34,6 +35,9 @@ test:
|
||||
fuzz:
|
||||
@script/fuzz
|
||||
|
||||
$(PROTOC_GEN_GO):
|
||||
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
|
||||
|
||||
fixme:
|
||||
@grep -nir fixme . | grep -v Makefile
|
||||
|
||||
@@ -54,14 +58,18 @@ fmt-check-md:
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
generate:
|
||||
@script/generate
|
||||
devprereqs:
|
||||
which golangci-lint || go install -v github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
|
||||
bin/mfer: $(SOURCEFILES)
|
||||
mfer/mf.pb.go: mfer/mf.proto
|
||||
cd mfer && go generate .
|
||||
|
||||
bin/mfer: $(SOURCEFILES) mfer/mf.pb.go
|
||||
protoc --version
|
||||
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
|
||||
|
||||
clean:
|
||||
rm -rfv bin/mfer cmd/mfer/mfer *.dockerimage
|
||||
rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
@@ -36,12 +36,12 @@ Generate a manifest for a directory tree, verify it later, and fetch a published
|
||||
tree by URL:
|
||||
|
||||
```sh
|
||||
# Write index.mf, a manifest of the files under the current directory.
|
||||
# Write .index.mf, a manifest of the files under the current directory.
|
||||
bin/mfer gen .
|
||||
|
||||
# Verify the files on disk against the manifest. Exits nonzero if any file
|
||||
# is missing or corrupted.
|
||||
bin/mfer check index.mf
|
||||
bin/mfer check .index.mf
|
||||
|
||||
# Download and cryptographically verify a tree published over HTTP: mfer
|
||||
# fetches <url>/index.mf, then downloads every file it lists.
|
||||
@@ -65,34 +65,21 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
||||
development workflow, and the Makefile targets are thin shims that call them. We
|
||||
provide:
|
||||
|
||||
- `script/bootstrap` — install all dependencies (Go, Go module download, and
|
||||
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
|
||||
idempotently; golangci-lint is not installed, it runs only in Docker
|
||||
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go module
|
||||
download, and node/yarn plus the prettier version pinned in
|
||||
`package.json`/`yarn.lock`), idempotently
|
||||
- `script/setup` — make a fresh clone ready for development: runs
|
||||
`script/bootstrap`, then `script/install-precommit`
|
||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||
such as `script/docker`
|
||||
- `script/test` — run the test suite (`go test`); one test fails when
|
||||
`mfer/mf.proto` no longer matches the hash `script/generate` recorded
|
||||
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from
|
||||
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in
|
||||
`mfer/mf.proto.sha256`; the only thing that regenerates the committed
|
||||
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file,
|
||||
and refuses to run with any other: `protoc` 33.4 (unpack
|
||||
`protoc-33.4-<platform>.zip` from
|
||||
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
|
||||
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
|
||||
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
|
||||
installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to
|
||||
`PATH`)
|
||||
- `script/test` — run the test suite (`go test`), regenerating the protobuf code
|
||||
first if it is stale
|
||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
||||
as ordinary tests
|
||||
- `script/lint` — run `golangci-lint` in Docker: builds only the `lint` stage of
|
||||
the `Dockerfile` (the Go format check, then the linter), uncached so it runs
|
||||
every time, then removes the image
|
||||
- `script/fmt` — format all code and docs (writes): `gofumpt` and
|
||||
`script/prettier --write`
|
||||
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
|
||||
- `script/fmt` — format all code and docs (writes): `gofumpt`,
|
||||
`golangci-lint run --fix`, and `script/prettier --write`
|
||||
- `script/prettier` — run prettier over the repository's canonical file set
|
||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||
`--check`; the single definition of that file set, so `script/fmt` and
|
||||
@@ -118,10 +105,11 @@ yet. Primary development happens on a privately-run Gitea instance at
|
||||
[tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
||||
|
||||
Changes must always be formatted with a standard `go fmt`, syntactically valid,
|
||||
and must pass the linting defined in the repository's `.golangci.yml`, which
|
||||
`make lint` runs in Docker. The `main` branch is protected and all changes must
|
||||
be made via [pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to
|
||||
be merged. Any changes submitted to this project must also be
|
||||
and must pass the linting defined in the repository (presently only the
|
||||
`golangci-lint` defaults), which can be run with a `make lint`. The `main`
|
||||
branch is protected and all changes must be made via
|
||||
[pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
||||
Any changes submitted to this project must also be
|
||||
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
||||
|
||||
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
|
||||
|
||||
@@ -12,13 +12,13 @@ require (
|
||||
github.com/pterm/pterm v0.12.35
|
||||
github.com/spf13/afero v1.8.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/urfave/cli/v2 v2.27.7
|
||||
github.com/urfave/cli/v2 v2.23.6
|
||||
google.golang.org/protobuf v1.28.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/atomicgo/cursor v0.0.1 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/fatih/color v1.7.0 // indirect
|
||||
github.com/gookit/color v1.4.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
@@ -34,7 +34,7 @@ require (
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||
|
||||
@@ -61,8 +61,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -231,12 +231,12 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
|
||||
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
|
||||
github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4=
|
||||
github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
|
||||
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
|
||||
+11
-12
@@ -75,22 +75,25 @@ func safeRateUint64(rate float64) uint64 {
|
||||
return uint64(rate)
|
||||
}
|
||||
|
||||
// findManifest returns the path of the manifest with the default name in
|
||||
// dir, or an error if there is none.
|
||||
// findManifest looks for a manifest file in the given directory.
|
||||
// It checks for index.mf and .index.mf, returning the first one found.
|
||||
func findManifest(fs afero.Fs, dir string) (string, error) {
|
||||
path := filepath.Join(dir, defaultManifestName)
|
||||
candidates := []string{"index.mf", ".index.mf"}
|
||||
for _, name := range candidates {
|
||||
path := filepath.Join(dir, name)
|
||||
|
||||
exists, err := afero.Exists(fs, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return "", fmt.Errorf("%w in %s (looked for %s)",
|
||||
errNoManifestFound, dir, defaultManifestName)
|
||||
if exists {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
|
||||
return path, nil
|
||||
return "", fmt.Errorf(
|
||||
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir)
|
||||
}
|
||||
|
||||
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
||||
@@ -295,11 +298,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
||||
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
|
||||
|
||||
// Create checker
|
||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||
ManifestPath: manifestPath,
|
||||
BasePath: basePath,
|
||||
Fs: mfa.Fs,
|
||||
})
|
||||
chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load manifest: %w", err)
|
||||
}
|
||||
|
||||
+12
-213
@@ -8,9 +8,6 @@ import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
@@ -28,11 +25,11 @@ const (
|
||||
testFile1 = "/testdir/file1.txt"
|
||||
testMF = "/testdir/test.mf"
|
||||
testOutput = "/output.mf"
|
||||
testOutputTmp = "/output.mf.tmp"
|
||||
testManifest = "/manifest.mf"
|
||||
testFlagBase = "--base"
|
||||
testFlagNoExtra = "--no-extra-files"
|
||||
testFlagVersion = "--version"
|
||||
testFlagVerbose = "--verbose"
|
||||
)
|
||||
|
||||
var errSimulatedWrite = errors.New("simulated write failure")
|
||||
@@ -45,32 +42,20 @@ var errSimulatedWrite = errors.New("simulated write failure")
|
||||
var runMu sync.Mutex
|
||||
|
||||
// runCLI invokes RunWithOptions while holding runMu so parallel tests
|
||||
// capture their own output, and returns its exit code.
|
||||
// capture their own output. Before releasing the lock it points the
|
||||
// process-global logger at io.Discard: other tests log outside the lock
|
||||
// (manifest loads, scans), and those lines must not land in this run's
|
||||
// buffers once it has returned and its test is reading them.
|
||||
func runCLI(opts *RunOptions) int {
|
||||
exitCode, _ := runCLIWithLevel(opts)
|
||||
|
||||
return exitCode
|
||||
}
|
||||
|
||||
// runCLIWithLevel is runCLI that also returns the log level the run left
|
||||
// set, read while runMu still keeps other runs from changing it. Each run
|
||||
// starts at the default level, as a new process does. Before releasing the
|
||||
// lock it points the process-global logger at io.Discard: other tests log
|
||||
// outside the lock (manifest loads, scans), and those lines must not land in
|
||||
// this run's buffers once it has returned and its test is reading them.
|
||||
func runCLIWithLevel(opts *RunOptions) (int, log.Level) {
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
log.SetLevel(log.InfoLevel)
|
||||
|
||||
exitCode := RunWithOptions(opts)
|
||||
level := log.GetLevel()
|
||||
|
||||
log.SetOutput(io.Discard, io.Discard)
|
||||
log.Init()
|
||||
|
||||
return exitCode, level
|
||||
return exitCode
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -250,90 +235,6 @@ func TestRootVerbosityFlags(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// commandsTakingVerbose returns the command lines -v can follow: the root and
|
||||
// the generate, check, freshen and fetch subcommands.
|
||||
func commandsTakingVerbose() [][]string {
|
||||
return [][]string{
|
||||
{testApp},
|
||||
{testApp, cmdGenerate},
|
||||
{testApp, cmdCheck},
|
||||
{testApp, cmdFreshen},
|
||||
{testApp, cmdFetch},
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerboseCount asserts that one -v or --verbose gives verbose output and
|
||||
// two -v give debug output (issue #125). urfave/cli before v2.25.5 counted a
|
||||
// flag given by its alias twice, so one -v gave debug output.
|
||||
func TestVerboseCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
flags []string
|
||||
want log.Level
|
||||
}{
|
||||
{[]string{"-v"}, log.VerboseLevel},
|
||||
{[]string{testFlagVerbose}, log.VerboseLevel},
|
||||
{[]string{"-v", "-v"}, log.DebugLevel},
|
||||
}
|
||||
|
||||
for _, command := range commandsTakingVerbose() {
|
||||
for _, tc := range cases {
|
||||
args := slices.Concat(command, tc.flags)
|
||||
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, level := runCLIWithLevel(testOpts(args, afero.NewMemMapFs()))
|
||||
assert.Equal(t, tc.want, level)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCombinedShortVerboseRefused asserts that -vv is refused (issue #125):
|
||||
// single-letter flags do not combine, so the -v help text says -v -v.
|
||||
func TestCombinedShortVerboseRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range commandsTakingVerbose() {
|
||||
args := slices.Concat(command, []string{"-vv"})
|
||||
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(args, afero.NewMemMapFs())
|
||||
exitCode, level := runCLIWithLevel(opts)
|
||||
|
||||
assert.Equal(t, 1, exitCode)
|
||||
assert.Contains(t, testStderr(t, opts), "flag provided but not defined: -vv")
|
||||
assert.Equal(t, log.InfoLevel, level)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestShortAndLongVerboseRefused asserts that -v and --verbose given together
|
||||
// are refused (issue #125): urfave/cli v2 refuses a flag given under two of its
|
||||
// names, and one flag with an alias keeps help and parsing simple.
|
||||
func TestShortAndLongVerboseRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range commandsTakingVerbose() {
|
||||
args := slices.Concat(command, []string{"-v", testFlagVerbose})
|
||||
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(args, afero.NewMemMapFs())
|
||||
exitCode, level := runCLIWithLevel(opts)
|
||||
|
||||
assert.Equal(t, 1, exitCode)
|
||||
assert.Contains(t, testStderr(t, opts), "Cannot use two forms of the same flag")
|
||||
assert.Equal(t, log.InfoLevel, level)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelpCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -504,10 +405,7 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
|
||||
assert.True(t, exists)
|
||||
|
||||
// Verify manifest only has 1 file (the non-dotfile)
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: testMF,
|
||||
Fs: fs,
|
||||
})
|
||||
manifest, err := mfer.NewManifestFromFile(fs, testMF)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, manifest.Files(), 1)
|
||||
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
|
||||
@@ -531,10 +429,7 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
||||
require.Equal(t, 0, exitCode)
|
||||
|
||||
// Verify manifest has 2 files (including dotfile)
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: testMF,
|
||||
Fs: fs,
|
||||
})
|
||||
manifest, err := mfer.NewManifestFromFile(fs, testMF)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, manifest.Files(), 2)
|
||||
}
|
||||
@@ -682,7 +577,7 @@ func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
||||
assert.True(t, exists, "output file should exist")
|
||||
|
||||
// Verify temp file does NOT exist
|
||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, tmpExists,
|
||||
"temp file should not exist after successful generation")
|
||||
@@ -714,7 +609,7 @@ func TestGenerateAtomicWriteOverwriteWithForce(t *testing.T) {
|
||||
"manifest should be overwritten")
|
||||
|
||||
// Verify temp file does NOT exist
|
||||
tmpExists, err := afero.Exists(fs, manifestTempPath(testOutput))
|
||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, tmpExists,
|
||||
"temp file should not exist after successful generation")
|
||||
@@ -743,102 +638,6 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
||||
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
||||
}
|
||||
|
||||
// manifestPaths returns the file paths listed by the manifest at path.
|
||||
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
||||
t.Helper()
|
||||
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: path,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
paths := make([]string, 0, len(manifest.Files()))
|
||||
for _, f := range manifest.Files() {
|
||||
paths = append(paths, f.GetPath())
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// TestGenerateLeavesOutputOutOfListing overwrites an output file inside the
|
||||
// scanned tree with --force: the old file is not listed, even when the tree
|
||||
// is named through a symlink, while a file named index.mf in a subdirectory
|
||||
// still is. The output file is recognized by file identity, which needs
|
||||
// the real filesystem.
|
||||
func TestGenerateLeavesOutputOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Paths are relative to a temp dir holding data/tree and link, a
|
||||
// symlink to data.
|
||||
for name, tc := range map[string]struct{ input, output string }{
|
||||
"default name": {"data/tree", "data/tree/index.mf"},
|
||||
"other name in a subdirectory": {"data/tree", "data/tree/sub/listing.mf"},
|
||||
"tree named through a symlink": {"link/tree", "data/tree/index.mf"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
tree := filepath.Join(root, "data", "tree")
|
||||
output := filepath.Join(root, tc.output)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(tree, "sub"), 0o750))
|
||||
require.NoError(t,
|
||||
os.Symlink(filepath.Join(root, "data"), filepath.Join(root, "link")))
|
||||
writeTestFile(t, fs, filepath.Join(tree, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, filepath.Join(tree, "sub", "index.mf"), "an ordinary file")
|
||||
writeTestFile(t, fs, output, "previous manifest")
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", "--force", "-o", output, filepath.Join(root, tc.input),
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{testFileTxt, "sub/index.mf"},
|
||||
manifestPaths(t, fs, output))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateDefaultOutputLeftOutOfListing runs gen with --force and no
|
||||
// other arguments, so it scans the current directory and writes the
|
||||
// relative path index.mf: the index.mf already there is not listed.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestGenerateDefaultOutputLeftOutOfListing(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestFile(t, fs, testFileTxt, "hello")
|
||||
writeTestFile(t, fs, "index.mf", "previous manifest")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "--force"}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, "index.mf"))
|
||||
}
|
||||
|
||||
// TestGenerateLeavesLeftoverTempFileOutOfListing runs gen where an
|
||||
// interrupted run left its temp file beside the output: the leftover is
|
||||
// not listed, and gen does not fail when it overwrites it.
|
||||
func TestGenerateLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
output := filepath.Join(root, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, manifestTempPath(output), "part of a manifest")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", output, root}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.Equal(t, []string{testFileTxt}, manifestPaths(t, fs, output))
|
||||
}
|
||||
|
||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -861,7 +660,7 @@ func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
||||
exists, _ := afero.Exists(fs, testOutput)
|
||||
assert.True(t, exists, "output file should exist")
|
||||
|
||||
tmpExists, _ := afero.Exists(fs, manifestTempPath(testOutput))
|
||||
tmpExists, _ := afero.Exists(fs, testOutputTmp)
|
||||
assert.False(t, tmpExists, "temp file should be cleaned up")
|
||||
|
||||
// Verify manifest is valid (not empty)
|
||||
@@ -927,7 +726,7 @@ func TestGenerateAtomicWriteCleansUpOnError(t *testing.T) {
|
||||
"output file should not exist after failed generation (atomic write)")
|
||||
|
||||
// Temp file should also not exist
|
||||
tmpExists, _ := afero.Exists(baseFs, manifestTempPath(testOutput))
|
||||
tmpExists, _ := afero.Exists(baseFs, testOutputTmp)
|
||||
assert.False(t, tmpExists,
|
||||
"temp file should be cleaned up after failed generation")
|
||||
}
|
||||
|
||||
@@ -61,11 +61,7 @@ func unsignedChecker(t *testing.T) *mfer.Checker {
|
||||
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
|
||||
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
|
||||
|
||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||
ManifestPath: "/d/index.mf",
|
||||
BasePath: "/d",
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := mfer.NewChecker("/d/index.mf", "/d", fs)
|
||||
require.NoError(t, err)
|
||||
require.False(t, chk.IsSigned())
|
||||
|
||||
@@ -78,7 +74,7 @@ func TestNoManifestFoundMessage(t *testing.T) {
|
||||
_, err := findManifest(afero.NewMemMapFs(), "/tmp/x")
|
||||
require.ErrorIs(t, err, errNoManifestFound)
|
||||
assert.EqualError(t, err,
|
||||
"no manifest found in /tmp/x (looked for index.mf)")
|
||||
"no manifest found in /tmp/x (looked for index.mf and .index.mf)")
|
||||
}
|
||||
|
||||
func TestVerifyRequiredSignerMessages(t *testing.T) {
|
||||
@@ -171,11 +167,7 @@ func signedChecker(t *testing.T) *mfer.Checker {
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644))
|
||||
|
||||
chk, err := mfer.NewChecker(&mfer.CheckerOptions{
|
||||
ManifestPath: "/index.mf",
|
||||
BasePath: "/",
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := mfer.NewChecker("/index.mf", "/", fs)
|
||||
require.NoError(t, err)
|
||||
require.True(t, chk.IsSigned())
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ func checkNoSymlinks(p string) error {
|
||||
|
||||
// resolveManifestURL takes a URL and returns the manifest URL.
|
||||
// If the URL already ends with .mf, it's returned as-is.
|
||||
// Otherwise, the default manifest name is appended.
|
||||
// Otherwise, index.mf is appended.
|
||||
func resolveManifestURL(inputURL string) (string, error) {
|
||||
parsed, err := url.Parse(inputURL)
|
||||
if err != nil {
|
||||
@@ -330,7 +330,8 @@ func resolveManifestURL(inputURL string) (string, error) {
|
||||
parsed.Path += "/"
|
||||
}
|
||||
|
||||
parsed.Path += defaultManifestName
|
||||
// Append index.mf
|
||||
parsed.Path += "index.mf"
|
||||
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
+10
-35
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
@@ -57,7 +56,7 @@ type freshenEntry struct {
|
||||
type freshenScanner struct {
|
||||
fs afero.Fs
|
||||
absBase string
|
||||
excluded []fs.FileInfo // files left out of the listing
|
||||
manifestBase string
|
||||
includeDotfiles bool
|
||||
followSymlinks bool
|
||||
showProgress bool
|
||||
@@ -157,6 +156,11 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
||||
}
|
||||
|
||||
// Skip the manifest file itself
|
||||
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle dotfiles
|
||||
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
||||
if info.IsDir() {
|
||||
@@ -181,12 +185,6 @@ func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) erro
|
||||
info = realInfo
|
||||
}
|
||||
|
||||
for _, excluded := range s.excluded {
|
||||
if os.SameFile(info, excluded) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
s.scanCount++
|
||||
|
||||
// Check against existing manifest
|
||||
@@ -309,7 +307,7 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
func writeFreshenedManifest(
|
||||
ctx context.Context, afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
||||
) error {
|
||||
tmpPath := manifestTempPath(manifestPath)
|
||||
tmpPath := manifestPath + ".tmp"
|
||||
|
||||
outFile, err := afs.Create(tmpPath)
|
||||
if err != nil {
|
||||
@@ -366,22 +364,10 @@ func (mfa *CLIApp) freshenScan(
|
||||
startScan := time.Now()
|
||||
showProgress := ctx.Bool("progress")
|
||||
|
||||
// Leave out the manifest and a temp file left by an interrupted run,
|
||||
// as gen does. A path that cannot be stat'd, normally because no file
|
||||
// is there, needs no leaving out.
|
||||
var excluded []fs.FileInfo
|
||||
|
||||
for _, p := range []string{manifestPath, manifestTempPath(manifestPath)} {
|
||||
info, err := mfa.Fs.Stat(p)
|
||||
if err == nil {
|
||||
excluded = append(excluded, info)
|
||||
}
|
||||
}
|
||||
|
||||
scanner := &freshenScanner{
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
excluded: excluded,
|
||||
manifestBase: filepath.Base(manifestPath),
|
||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
||||
showProgress: showProgress,
|
||||
@@ -459,10 +445,7 @@ func (mfa *CLIApp) loadExistingEntries(
|
||||
log.Infof("loading manifest from %s", manifestPath)
|
||||
|
||||
// Load existing manifest
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: manifestPath,
|
||||
Fs: mfa.Fs,
|
||||
})
|
||||
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load manifest: %w", err)
|
||||
}
|
||||
@@ -629,15 +612,7 @@ func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||
return b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
||||
entry.GetHashes()[0].GetMultiHash())
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"manifest entry %s: %w (regenerate the manifest with mfer generate)",
|
||||
entry.GetPath(), err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,13 +4,10 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -32,7 +29,7 @@ func (s stubFileInfo) IsDir() bool { return false }
|
||||
func (s stubFileInfo) Sys() any { return nil }
|
||||
|
||||
// setupFreshenDir populates /testdir with two files, scans it, and
|
||||
// writes the resulting manifest to /testdir/index.mf.
|
||||
// writes the resulting manifest to /testdir/.index.mf.
|
||||
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
||||
t.Helper()
|
||||
|
||||
@@ -51,7 +48,7 @@ func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
||||
|
||||
// Write manifest to filesystem
|
||||
require.NoError(t,
|
||||
afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644))
|
||||
afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644))
|
||||
}
|
||||
|
||||
func TestFreshenUnchanged(t *testing.T) {
|
||||
@@ -61,10 +58,7 @@ func TestFreshenUnchanged(t *testing.T) {
|
||||
setupFreshenDir(t, fs)
|
||||
|
||||
// Parse manifest to verify
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: "/testdir/index.mf",
|
||||
Fs: fs,
|
||||
})
|
||||
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, manifest.Files(), 2)
|
||||
}
|
||||
@@ -76,10 +70,7 @@ func TestFreshenWithChanges(t *testing.T) {
|
||||
setupFreshenDir(t, fs)
|
||||
|
||||
// Verify initial manifest has 2 files
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: "/testdir/index.mf",
|
||||
Fs: fs,
|
||||
})
|
||||
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, manifest.Files(), 2)
|
||||
|
||||
@@ -104,64 +95,6 @@ func TestFreshenWithChanges(t *testing.T) {
|
||||
assert.Equal(t, "modified content2", string(content))
|
||||
}
|
||||
|
||||
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
||||
// subdirectory of the tree it lists: the manifest is not listed, while an
|
||||
// ordinary file of the same name at the top of the tree is. The manifest
|
||||
// is recognized by file identity, which needs the real filesystem.
|
||||
func TestFreshenLeavesManifestOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
manifestPath := filepath.Join(root, "sub", "listing.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(filepath.Join(root, "sub"), 0o750))
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
writeTestFile(t, fs, filepath.Join(root, "listing.mf"), "an ordinary file")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
// A new file gives freshen something to write.
|
||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{testFileTxt, "listing.mf", "added.txt"},
|
||||
manifestPaths(t, fs, manifestPath))
|
||||
}
|
||||
|
||||
// TestFreshenLeavesLeftoverTempFileOutOfListing freshens a manifest where
|
||||
// an interrupted run left its temp file beside it: the leftover is not
|
||||
// listed.
|
||||
func TestFreshenLeavesLeftoverTempFileOutOfListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
manifestPath := filepath.Join(root, "index.mf")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
writeTestFile(t, fs, manifestTempPath(manifestPath), "part of a manifest")
|
||||
// A new file gives freshen something to write.
|
||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, "freshen", "-q", testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
assert.ElementsMatch(t, []string{testFileTxt, "added.txt"},
|
||||
manifestPaths(t, fs, manifestPath))
|
||||
}
|
||||
|
||||
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
||||
// with respect to MFFilePath.Mtime, which is a message pointer with
|
||||
// proto3 field presence and may legitimately be absent.
|
||||
@@ -236,50 +169,21 @@ func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
||||
func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := mfer.NewBuilder()
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: "file1.txt",
|
||||
Size: 8,
|
||||
Mtime: nil,
|
||||
Hashes: []*mfer.MFFileChecksum{
|
||||
{MultiHash: hash},
|
||||
{MultiHash: []byte{0x12, 0x20}},
|
||||
},
|
||||
}
|
||||
|
||||
err = addExistingToBuilder(b, entry)
|
||||
err := addExistingToBuilder(b, entry)
|
||||
require.ErrorIs(t, err, errEntryMissingMtime)
|
||||
assert.Contains(t, err.Error(), "file1.txt")
|
||||
}
|
||||
|
||||
// TestFreshenAddExistingRejectsShortHash pins that an existing manifest
|
||||
// entry whose hash the builder refuses is reported as a problem with the
|
||||
// manifest, with the command that fixes it.
|
||||
func TestFreshenAddExistingRejectsShortHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := mfer.NewBuilder()
|
||||
entry := &mfer.MFFilePath{
|
||||
Path: "old.txt",
|
||||
Size: 8,
|
||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000},
|
||||
Hashes: []*mfer.MFFileChecksum{
|
||||
{MultiHash: sha1Hash},
|
||||
},
|
||||
}
|
||||
|
||||
err = addExistingToBuilder(b, entry)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "manifest entry old.txt")
|
||||
assert.Contains(t, err.Error(), "mfer generate")
|
||||
assert.Zero(t, b.FileCount())
|
||||
}
|
||||
|
||||
// TestEntryMtime pins the presence semantics the callers depend on.
|
||||
func TestEntryMtime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
+1
-5
@@ -89,15 +89,11 @@ func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
||||
|
||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
||||
output := ctx.String("output")
|
||||
opts := &mfer.ScannerOptions{
|
||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
||||
Fs: mfa.Fs,
|
||||
// Neither a manifest being replaced nor a temp file left by an
|
||||
// interrupted run belongs in the new manifest.
|
||||
ExcludePaths: []string{output, manifestTempPath(output)},
|
||||
}
|
||||
|
||||
// Set seed for deterministic UUID if provided
|
||||
@@ -221,7 +217,7 @@ func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||
}
|
||||
|
||||
// Create temp file for atomic write
|
||||
tmpPath := manifestTempPath(outputPath)
|
||||
tmpPath := outputPath + ".tmp"
|
||||
|
||||
outFile, err := mfa.Fs.Create(tmpPath)
|
||||
if err != nil {
|
||||
|
||||
@@ -65,7 +65,7 @@ func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
|
||||
}
|
||||
|
||||
// resolveManifestArg resolves the manifest path from CLI arguments.
|
||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf.
|
||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
|
||||
// If no argument is given, the current directory is searched.
|
||||
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
||||
if ctx.Args().Len() > 0 {
|
||||
|
||||
+3
-17
@@ -17,7 +17,6 @@ import (
|
||||
const (
|
||||
cmdGenerate = "generate"
|
||||
cmdCheck = "check"
|
||||
cmdFreshen = "freshen"
|
||||
cmdExport = "export"
|
||||
cmdFetch = "fetch"
|
||||
cmdVersion = "version"
|
||||
@@ -25,19 +24,8 @@ const (
|
||||
flagProgress = "progress"
|
||||
|
||||
manifestArgsUsage = "[manifest file]"
|
||||
|
||||
// defaultManifestName is the filename gen writes by default, the one
|
||||
// looked for when a command is given a directory, and the one fetch
|
||||
// appends to a directory URL.
|
||||
defaultManifestName = "index.mf"
|
||||
)
|
||||
|
||||
// manifestTempPath returns the temp file gen and freshen write a manifest
|
||||
// to before renaming it to out.
|
||||
func manifestTempPath(out string) string {
|
||||
return out + ".tmp"
|
||||
}
|
||||
|
||||
// errUnknownCommand indicates an unrecognized command argument.
|
||||
var errUnknownCommand = errors.New("unknown command")
|
||||
|
||||
@@ -130,7 +118,7 @@ func commonFlags() []cli.Flag {
|
||||
&cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
Aliases: []string{"v"},
|
||||
Usage: "Increase verbosity (-v for verbose, -v -v for debug)",
|
||||
Usage: "Increase verbosity (-v for verbose, -vv for debug)",
|
||||
Count: new(int),
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
@@ -146,7 +134,6 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
Name: cmdGenerate,
|
||||
Aliases: []string{"gen"},
|
||||
Usage: "Generate manifest file",
|
||||
ArgsUsage: "[path ...]",
|
||||
Action: func(c *cli.Context) error {
|
||||
mfa.setVerbosity(c)
|
||||
mfa.printBanner()
|
||||
@@ -167,7 +154,7 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "output",
|
||||
Value: defaultManifestName,
|
||||
Value: "./.index.mf",
|
||||
Aliases: []string{"o"},
|
||||
Usage: "Specify output filename",
|
||||
},
|
||||
@@ -240,7 +227,7 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdFreshen,
|
||||
Name: "freshen",
|
||||
Usage: "Update manifest with changed, new, and removed files",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
Action: func(c *cli.Context) error {
|
||||
@@ -339,7 +326,6 @@ func (mfa *CLIApp) fetchCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdFetch,
|
||||
Usage: "fetch manifest and referenced files",
|
||||
ArgsUsage: "URL",
|
||||
Action: func(c *cli.Context) error {
|
||||
mfa.setVerbosity(c)
|
||||
mfa.printBanner()
|
||||
|
||||
+4
-17
@@ -35,8 +35,7 @@ var (
|
||||
errPathDotDot = errors.New("contains '..' segment")
|
||||
errSizeMismatch = errors.New("size mismatch")
|
||||
errNegativeSize = errors.New("size cannot be negative")
|
||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||
errHashTooShort = errors.New("hash digest is too short")
|
||||
errEmptyHash = errors.New("hash cannot be nil or empty")
|
||||
)
|
||||
|
||||
// ValidatePath checks that a file path conforms to manifest path invariants:
|
||||
@@ -229,8 +228,7 @@ func (b *Builder) FileCount() int {
|
||||
|
||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
||||
// Returns an error if path is invalid, size is negative, or hash is not a
|
||||
// multihash with a digest of at least 32 bytes, as long as SHA-256's.
|
||||
// Returns an error if path is empty, size is negative, or hash is nil/empty.
|
||||
func (b *Builder) AddFileWithHash(
|
||||
path RelFilePath,
|
||||
size FileSize,
|
||||
@@ -246,19 +244,8 @@ func (b *Builder) AddFileWithHash(
|
||||
return errNegativeSize
|
||||
}
|
||||
|
||||
decoded, err := multihash.Decode(hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", errHashNotMultihash, err)
|
||||
}
|
||||
|
||||
// The reader's limit on decoding cost (maxDecodedGrowth) assumes every
|
||||
// hash is at least as long as a SHA-256 multihash, so a manifest of
|
||||
// shorter ones could fail to load.
|
||||
if len(decoded.Digest) < sha256.Size {
|
||||
return fmt.Errorf(
|
||||
"%w: %d bytes, at least %d needed",
|
||||
errHashTooShort, len(decoded.Digest), sha256.Size,
|
||||
)
|
||||
if len(hash) == 0 {
|
||||
return errEmptyHash
|
||||
}
|
||||
|
||||
entry := &MFFilePath{
|
||||
|
||||
+27
-75
@@ -4,15 +4,11 @@ package mfer
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -46,10 +42,9 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
hash := make([]byte, 34) // SHA256 multihash is 34 bytes
|
||||
|
||||
err = b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, b.FileCount())
|
||||
}
|
||||
@@ -57,14 +52,12 @@ func TestBuilderAddFileWithHash(t *testing.T) {
|
||||
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sha256Hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("empty path", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), sha256Hash)
|
||||
hash := make([]byte, 34)
|
||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "path")
|
||||
})
|
||||
@@ -73,58 +66,41 @@ func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), sha256Hash)
|
||||
hash := make([]byte, 34)
|
||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "size")
|
||||
})
|
||||
|
||||
t.Run("nil hash", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "hash")
|
||||
})
|
||||
|
||||
t.Run("empty hash", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "hash")
|
||||
})
|
||||
|
||||
t.Run("valid inputs", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), sha256Hash)
|
||||
hash := make([]byte, 34)
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, b.FileCount())
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuilderAddFileWithHashRejectsBadHashes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sha1Hash, err := multihash.Encode(make([]byte, 20), multihash.SHA1)
|
||||
require.NoError(t, err)
|
||||
|
||||
truncatedHash, err := multihash.Encode(make([]byte, sha256.Size-1), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
hash Multihash
|
||||
want error
|
||||
}{
|
||||
{"nil hash", nil, errHashNotMultihash},
|
||||
{"empty hash", []byte{}, errHashNotMultihash},
|
||||
{"one-byte hash", []byte{0x12}, errHashNotMultihash},
|
||||
// A SHA-256 code and 32-byte length, then only two bytes of digest.
|
||||
{"malformed multihash", []byte{0x12, 0x20, 0x01, 0x02}, errHashNotMultihash},
|
||||
// A valid multihash, but its 20-byte SHA-1 digest is too short.
|
||||
{"SHA-1 multihash", sha1Hash, errHashTooShort},
|
||||
// A valid multihash whose 31-byte digest is one byte short.
|
||||
{"31-byte digest", truncatedHash, errHashTooShort},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBuilder()
|
||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), tt.hash)
|
||||
require.ErrorIs(t, err, tt.want)
|
||||
assert.Equal(t, 0, b.FileCount())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuilderBuild(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -380,8 +356,7 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
|
||||
func TestBuilderBuildRoundTripLargeManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||
|
||||
b := NewBuilder()
|
||||
|
||||
@@ -422,29 +397,6 @@ func TestNewManifestFromReaderTruncated(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestNewManifestFromFileRequiresPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := NewManifestFromFile(nil)
|
||||
require.ErrorIs(t, err, errManifestPathEmpty)
|
||||
|
||||
_, err = NewManifestFromFile(&ManifestFromFileOptions{Fs: afero.NewMemMapFs()})
|
||||
require.ErrorIs(t, err, errManifestPathEmpty)
|
||||
}
|
||||
|
||||
func TestNewManifestFromFileNilFsUsesOsFs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "index.mf")
|
||||
createTestManifest(t, afero.NewOsFs(), path, map[string][]byte{
|
||||
testFileName: []byte("hello"),
|
||||
})
|
||||
|
||||
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, m.Files(), 1)
|
||||
}
|
||||
|
||||
func TestManifestString(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+8
-34
@@ -14,11 +14,7 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
var (
|
||||
errNoSigningPubKey = errors.New("manifest has no signing public key")
|
||||
errManifestPathEmpty = errors.New("manifest path cannot be empty")
|
||||
errBasePathEmpty = errors.New("base path cannot be empty")
|
||||
)
|
||||
var errNoSigningPubKey = errors.New("manifest has no signing public key")
|
||||
|
||||
// Result represents the outcome of checking a single file.
|
||||
type Result struct {
|
||||
@@ -86,42 +82,20 @@ type Checker struct {
|
||||
signingPubKey []byte
|
||||
}
|
||||
|
||||
// CheckerOptions configures a Checker.
|
||||
type CheckerOptions struct {
|
||||
// ManifestPath is the manifest file to check against (required).
|
||||
ManifestPath string
|
||||
// BasePath is the directory relative to which manifest paths are
|
||||
// resolved (required).
|
||||
BasePath string
|
||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
||||
Fs afero.Fs
|
||||
}
|
||||
|
||||
// NewChecker creates a new Checker with the given options. It returns an
|
||||
// error if opts is nil or either path is empty.
|
||||
func NewChecker(opts *CheckerOptions) (*Checker, error) {
|
||||
if opts == nil || opts.ManifestPath == "" {
|
||||
return nil, errManifestPathEmpty
|
||||
}
|
||||
|
||||
if opts.BasePath == "" {
|
||||
return nil, errBasePathEmpty
|
||||
}
|
||||
|
||||
fs := opts.Fs
|
||||
// NewChecker creates a new Checker for the given manifest, base path, and filesystem.
|
||||
// The basePath is the directory relative to which manifest paths are resolved.
|
||||
// If fs is nil, the real filesystem (OsFs) is used.
|
||||
func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, error) {
|
||||
if fs == nil {
|
||||
fs = afero.NewOsFs()
|
||||
}
|
||||
|
||||
m, err := NewManifestFromFile(&ManifestFromFileOptions{
|
||||
Path: opts.ManifestPath,
|
||||
Fs: fs,
|
||||
})
|
||||
m, err := NewManifestFromFile(fs, manifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
abs, err := filepath.Abs(opts.BasePath)
|
||||
abs, err := filepath.Abs(basePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -134,7 +108,7 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
|
||||
}
|
||||
|
||||
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
|
||||
absManifest, err := filepath.Abs(opts.ManifestPath)
|
||||
absManifest, err := filepath.Abs(manifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+47
-193
@@ -5,8 +5,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -19,10 +17,6 @@ const (
|
||||
testFile1 = "file1.txt"
|
||||
testFile2 = "file2.txt"
|
||||
testExistsFile = "exists.txt"
|
||||
testManifestPath = "/manifest.mf"
|
||||
testDataDir = "/data"
|
||||
// testDataManifestPath is a manifest kept inside the checked tree.
|
||||
testDataManifestPath = testDataDir + "/index.mf"
|
||||
)
|
||||
|
||||
func TestStatusString(t *testing.T) {
|
||||
@@ -72,13 +66,15 @@ func createTestManifest(
|
||||
}
|
||||
|
||||
// createFilesOnDisk creates the given files on the filesystem under
|
||||
// testDataDir.
|
||||
// /data.
|
||||
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
|
||||
t.Helper()
|
||||
|
||||
basePath := "/data"
|
||||
|
||||
for path, content := range files {
|
||||
fullPath := testDataDir + "/" + path
|
||||
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
|
||||
fullPath := basePath + "/" + path
|
||||
require.NoError(t, fs.MkdirAll(basePath, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
||||
}
|
||||
}
|
||||
@@ -94,13 +90,9 @@ func TestNewChecker(t *testing.T) {
|
||||
testFile1: []byte("hello"),
|
||||
testFile2: []byte("world"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: "/",
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/", fs)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, chk)
|
||||
assert.Equal(t, FileCount(2), chk.FileCount())
|
||||
@@ -110,11 +102,7 @@ func TestNewChecker(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
_, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: "/nonexistent.mf",
|
||||
BasePath: "/",
|
||||
Fs: fs,
|
||||
})
|
||||
_, err := NewChecker("/nonexistent.mf", "/", fs)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
@@ -123,73 +111,11 @@ func TestNewChecker(t *testing.T) {
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
|
||||
_, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: "/bad.mf",
|
||||
BasePath: "/",
|
||||
Fs: fs,
|
||||
})
|
||||
_, err := NewChecker("/bad.mf", "/", fs)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewCheckerRequiredPaths(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
opts *CheckerOptions
|
||||
want string
|
||||
is error
|
||||
}{
|
||||
{
|
||||
name: "nil options",
|
||||
opts: nil,
|
||||
want: "manifest path cannot be empty",
|
||||
is: errManifestPathEmpty,
|
||||
},
|
||||
{
|
||||
name: "empty manifest path",
|
||||
opts: &CheckerOptions{BasePath: testDataDir},
|
||||
want: "manifest path cannot be empty",
|
||||
is: errManifestPathEmpty,
|
||||
},
|
||||
{
|
||||
name: "empty base path",
|
||||
opts: &CheckerOptions{ManifestPath: testManifestPath},
|
||||
want: "base path cannot be empty",
|
||||
is: errBasePathEmpty,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
chk, err := NewChecker(tc.opts)
|
||||
require.ErrorIs(t, err, tc.is)
|
||||
require.EqualError(t, err, tc.want)
|
||||
assert.Nil(t, chk)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
manifestPath := filepath.Join(dir, "index.mf")
|
||||
content := []byte("hello")
|
||||
createTestManifest(t, afero.NewOsFs(), manifestPath, map[string][]byte{
|
||||
testFile1: content,
|
||||
})
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, testFile1), content, 0o600))
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{ManifestPath: manifestPath, BasePath: dir})
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 1)
|
||||
require.NoError(t, chk.Check(context.Background(), results, nil))
|
||||
assert.Equal(t, StatusOK, (<-results).Status)
|
||||
}
|
||||
|
||||
func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -199,13 +125,9 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
||||
"medium.txt": []byte("hello world"),
|
||||
"large.txt": bytes.Repeat([]byte("x"), 1000),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: "/",
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, FileCount(3), chk.FileCount())
|
||||
@@ -220,14 +142,10 @@ func TestCheckAllFilesOK(t *testing.T) {
|
||||
testFile1: []byte("content one"),
|
||||
testFile2: []byte("content two"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -254,17 +172,13 @@ func TestCheckMissingFile(t *testing.T) {
|
||||
testExistsFile: []byte("I exist"),
|
||||
"missing.txt": []byte("I don't exist on disk"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
// Only create one file
|
||||
createFilesOnDisk(t, fs, map[string][]byte{
|
||||
testExistsFile: []byte("I exist"),
|
||||
})
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -297,17 +211,13 @@ func TestCheckSizeMismatch(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
testFileName: []byte("original content"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
// Create file with different size
|
||||
createFilesOnDisk(t, fs, map[string][]byte{
|
||||
testFileName: []byte("short"),
|
||||
})
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -327,7 +237,7 @@ func TestCheckHashMismatch(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
testFileName: originalContent,
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
// Create file with same size but different content
|
||||
differentContent := []byte("different contnt") // same length (16 bytes) but different
|
||||
require.Len(t, differentContent, len(originalContent), "test requires same length")
|
||||
@@ -335,11 +245,7 @@ func TestCheckHashMismatch(t *testing.T) {
|
||||
testFileName: differentContent,
|
||||
})
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -359,14 +265,10 @@ func TestCheckWithProgress(t *testing.T) {
|
||||
testFile1: bytes.Repeat([]byte("a"), 100),
|
||||
testFile2: bytes.Repeat([]byte("b"), 200),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -403,14 +305,10 @@ func TestCheckContextCancellation(t *testing.T) {
|
||||
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
|
||||
}
|
||||
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -429,7 +327,7 @@ func TestFindExtraFiles(t *testing.T) {
|
||||
manifestFiles := map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, manifestFiles)
|
||||
createTestManifest(t, fs, "/manifest.mf", manifestFiles)
|
||||
|
||||
// Disk has file1 and file2
|
||||
createFilesOnDisk(t, fs, map[string][]byte{
|
||||
@@ -437,11 +335,7 @@ func TestFindExtraFiles(t *testing.T) {
|
||||
testFile2: []byte("extra file"),
|
||||
})
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -466,7 +360,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
||||
manifestFiles := map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
}
|
||||
createTestManifest(t, fs, testDataManifestPath, manifestFiles)
|
||||
createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
|
||||
createFilesOnDisk(t, fs, map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
})
|
||||
@@ -474,14 +368,10 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
|
||||
// Create a real extra file
|
||||
require.NoError(t, fs.MkdirAll(testDataDir, 0o755))
|
||||
require.NoError(t, fs.MkdirAll("/data", 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testDataManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/data/.index.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -493,7 +383,7 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
||||
extras = append(extras, r)
|
||||
}
|
||||
|
||||
// Should only report extra.txt, not .hidden, .config/settings, or index.mf
|
||||
// Should only report extra.txt, not .hidden, .config/settings, or .index.mf
|
||||
for _, e := range extras {
|
||||
t.Logf("extra: %s", e.Path)
|
||||
}
|
||||
@@ -510,14 +400,10 @@ func TestFindExtraFilesContextCancellation(t *testing.T) {
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
files := map[string][]byte{testFileName: []byte("data")}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -533,14 +419,10 @@ func TestCheckNilChannels(t *testing.T) {
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
files := map[string][]byte{testFileName: []byte("data")}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should not panic with nil channels
|
||||
@@ -553,14 +435,10 @@ func TestFindExtraFilesNilChannel(t *testing.T) {
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
files := map[string][]byte{testFileName: []byte("data")}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should not panic with nil channel
|
||||
@@ -577,7 +455,7 @@ func TestCheckSubdirectories(t *testing.T) {
|
||||
"dir1/dir2/file2.txt": []byte("content2"),
|
||||
"dir1/dir2/dir3/deep.txt": []byte("deep content"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
|
||||
// Create files with full directory structure
|
||||
for path, content := range files {
|
||||
@@ -587,11 +465,7 @@ func TestCheckSubdirectories(t *testing.T) {
|
||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
||||
}
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -619,17 +493,13 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
|
||||
testExistsFile: []byte("here"),
|
||||
"missing.txt": []byte("not on disk"),
|
||||
}
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
// Only create one file on disk
|
||||
createFilesOnDisk(t, fs, map[string][]byte{
|
||||
testExistsFile: []byte("here"),
|
||||
})
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -658,7 +528,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
testFile1: []byte("in manifest"),
|
||||
}
|
||||
createTestManifest(t, fs, testDataManifestPath, files)
|
||||
createTestManifest(t, fs, "/data/.index.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
// Add dotfiles and manifest file on disk
|
||||
@@ -667,11 +537,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
||||
require.NoError(t,
|
||||
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testDataManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/data/.index.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -697,14 +563,10 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
|
||||
files := map[string][]byte{
|
||||
testFile1: []byte("content"),
|
||||
}
|
||||
createTestManifest(t, fs, testDataManifestPath, files)
|
||||
createTestManifest(t, fs, "/data/index.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testDataManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/data/index.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 10)
|
||||
@@ -725,13 +587,9 @@ func TestCheckEmptyManifest(t *testing.T) {
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
// Create manifest with no files
|
||||
createTestManifest(t, fs, testManifestPath, map[string][]byte{})
|
||||
createTestManifest(t, fs, "/manifest.mf", map[string][]byte{})
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, FileCount(0), chk.FileCount())
|
||||
@@ -763,14 +621,10 @@ func TestCheckProgressRateLimited(t *testing.T) {
|
||||
files[name] = []byte("content")
|
||||
}
|
||||
|
||||
createTestManifest(t, fs, testManifestPath, files)
|
||||
createTestManifest(t, fs, "/manifest.mf", files)
|
||||
createFilesOnDisk(t, fs, files)
|
||||
|
||||
chk, err := NewChecker(&CheckerOptions{
|
||||
ManifestPath: testManifestPath,
|
||||
BasePath: testDataDir,
|
||||
Fs: fs,
|
||||
})
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
results := make(chan Result, 200)
|
||||
|
||||
+1
-5
@@ -35,10 +35,6 @@ const (
|
||||
decodedTimestampSize = 64
|
||||
decodedMIMETypeSize = 16
|
||||
|
||||
// Each file entry mfer writes holds a path of at least one byte, a
|
||||
// multihash at least as long as SHA-256's 34 bytes (AddFileWithHash
|
||||
// refuses shorter ones) and a modification time: at least 47 bytes,
|
||||
// counted at 336. So its manifests add up to at most about 7.15 times
|
||||
// their size, and this limit is about 12% above that.
|
||||
// The densest manifests mfer writes add up to about 7 times their size.
|
||||
maxDecodedGrowth = 8
|
||||
)
|
||||
|
||||
+6
-21
@@ -267,8 +267,7 @@ func (m *manifest) deserializeInner() error {
|
||||
// Deserialize inner message
|
||||
m.pbInner = new(MFFile)
|
||||
|
||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbInner)
|
||||
err = proto.Unmarshal(dat, m.pbInner)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
||||
}
|
||||
@@ -337,8 +336,7 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
||||
// deserialize outer:
|
||||
m.pbOuter = new(MFFileOuter)
|
||||
|
||||
// Unknown fields would cost memory; mfer never writes a loaded manifest out.
|
||||
err = proto.UnmarshalOptions{DiscardUnknown: true}.Unmarshal(dat, m.pbOuter)
|
||||
err = proto.Unmarshal(dat, m.pbOuter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -352,29 +350,16 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ManifestFromFileOptions configures NewManifestFromFile.
|
||||
type ManifestFromFileOptions struct {
|
||||
// Path is the manifest file to read (required).
|
||||
Path string
|
||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
||||
Fs afero.Fs
|
||||
}
|
||||
|
||||
// NewManifestFromFile reads a manifest from a file. It returns an error if
|
||||
// opts is nil or its path is empty.
|
||||
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
||||
// If fs is nil, the real filesystem (OsFs) is used.
|
||||
//
|
||||
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
||||
func NewManifestFromFile(opts *ManifestFromFileOptions) (*manifest, error) {
|
||||
if opts == nil || opts.Path == "" {
|
||||
return nil, errManifestPathEmpty
|
||||
}
|
||||
|
||||
fs := opts.Fs
|
||||
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
||||
if fs == nil {
|
||||
fs = afero.NewOsFs()
|
||||
}
|
||||
|
||||
f, err := fs.Open(opts.Path)
|
||||
f, err := fs.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -57,12 +57,11 @@ func FuzzNewManifestFromReader(f *testing.F) {
|
||||
// copying, so reaching those sizes allocates up to about six times
|
||||
// them in total. Decoding the decompressed data takes up to
|
||||
// maxDecodedGrowth times its size for file entries, hashes,
|
||||
// timestamps and MIME types, and drops fields it does not know.
|
||||
// The strings and bytes it copies out of it, such as many one-byte
|
||||
// values in one hash, take up to about five times more under the
|
||||
// race detector, which pads every small copy to 16 bytes, and about
|
||||
// half that without it. Twenty times the input and the
|
||||
// decompressed data leaves room for all of that.
|
||||
// timestamps and MIME types, and up to about five times more for
|
||||
// the bytes it copies out of it, such as fields it does not know,
|
||||
// which it keeps in buffers that also grow by copying. Twenty
|
||||
// times the input and the decompressed data leaves room for all of
|
||||
// that.
|
||||
//
|
||||
// The decoder also sets aside a new buffer of one to two times the
|
||||
// window for each frame that asks for a larger window than the
|
||||
@@ -77,8 +76,9 @@ func FuzzNewManifestFromReader(f *testing.F) {
|
||||
// fails if the decoder accepts windows of twice zstdWindowSize; the
|
||||
// seed whose two frames together exceed MaxDecompressedSize fails
|
||||
// if the decoder decodes them in full instead of stopping at the
|
||||
// declared size; the seeds of empty file entries and of a file
|
||||
// entry of empty hashes fail if the parser decodes them.
|
||||
// declared size; the seeds of empty file entries, of a file entry
|
||||
// of empty hashes, and of file entries of only an empty MIME type
|
||||
// and empty times fail if the parser decodes them.
|
||||
limit := 20*(uint64(len(data))+decompressed) + 24*zstdWindowSize
|
||||
|
||||
allocated := after.TotalAlloc - before.TotalAlloc
|
||||
|
||||
@@ -7,13 +7,11 @@ import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
@@ -118,33 +116,13 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Entries of a path, an empty hash, an empty MIME type and empty modification
|
||||
// and change times are counted at 416 bytes each (160 + 112 + 16 + 64 + 64)
|
||||
// and take 16 bytes plus the path to encode. A 35-character path makes that
|
||||
// 51 bytes, about 8.2 times: refused, and leaving any one of the five
|
||||
// uncounted, even the MIME type, brings it under 8. A 37-character path makes
|
||||
// it 53 bytes, about 7.8 times: loaded.
|
||||
// Entries of a one-character path and empty modification and change times
|
||||
// pass every other check, but would take about 23 times their size to decode.
|
||||
func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
pathLen int
|
||||
refused bool
|
||||
}{
|
||||
{35, true},
|
||||
{37, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(strconv.Itoa(tt.pathLen), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||
entry = protowire.AppendString(entry, strings.Repeat("a", tt.pathLen))
|
||||
entry = protowire.AppendTag(entry, 3, protowire.BytesType) // MFFilePath.hashes
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendTag(entry, 301, protowire.BytesType) // MFFilePath.mimeType
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendString(entry, "a")
|
||||
entry = protowire.AppendTag(entry, 302, protowire.BytesType) // MFFilePath.mtime
|
||||
entry = protowire.AppendBytes(entry, nil)
|
||||
entry = protowire.AppendTag(entry, 303, protowire.BytesType) // MFFilePath.ctime
|
||||
@@ -160,43 +138,7 @@ func TestDeserializeRefusesEntriesThatDecodeTooLarge(t *testing.T) {
|
||||
}
|
||||
|
||||
_, err := NewManifestFromReader(bytes.NewReader(wrapInner(t, id, inner)))
|
||||
if tt.refused {
|
||||
require.ErrorIs(t, err, errDecodedTooLarge)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Fields the decoder does not know are dropped, in the outer message, the inner
|
||||
// message and a file entry, so that they take no memory once loaded.
|
||||
func TestDeserializeDropsUnknownFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
unknown := protowire.AppendTag(nil, 99, protowire.BytesType) // in no message
|
||||
unknown = protowire.AppendBytes(unknown, []byte("not known"))
|
||||
|
||||
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
|
||||
entry = protowire.AppendString(entry, "a")
|
||||
entry = append(entry, unknown...)
|
||||
|
||||
id := uuid.New()
|
||||
inner := protowire.AppendTag(nil, 101, protowire.BytesType) // MFFile.files
|
||||
inner = protowire.AppendBytes(inner, entry)
|
||||
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
|
||||
inner = protowire.AppendBytes(inner, id[:])
|
||||
inner = append(inner, unknown...)
|
||||
|
||||
data := wrapInner(t, id, inner)
|
||||
data = append(data, unknown...) // the outer message ends the file
|
||||
|
||||
m, err := NewManifestFromReader(bytes.NewReader(data))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, m.Files(), 1)
|
||||
assert.Empty(t, m.pbOuter.ProtoReflect().GetUnknown())
|
||||
assert.Empty(t, m.pbInner.ProtoReflect().GetUnknown())
|
||||
assert.Empty(t, m.Files()[0].ProtoReflect().GetUnknown())
|
||||
}
|
||||
|
||||
// Many empty files with names of at most three characters and modification
|
||||
@@ -206,8 +148,7 @@ func TestDeserializeDropsUnknownFields(t *testing.T) {
|
||||
func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||
|
||||
b := NewBuilder()
|
||||
b.SetIncludeTimestamps(true)
|
||||
@@ -229,8 +170,7 @@ func TestDeserializeLoadsDensestManifest(t *testing.T) {
|
||||
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hash, err := multihash.Encode(make([]byte, sha256.Size), multihash.SHA2_256)
|
||||
require.NoError(t, err)
|
||||
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
|
||||
|
||||
b := NewBuilder()
|
||||
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
|
||||
|
||||
+18
-34
@@ -10,7 +10,6 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -426,31 +425,18 @@ func TestGPGTimeoutKillsGPG(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "gpg sign failed: gpg timed out")
|
||||
}
|
||||
|
||||
// TestGPGCancelWhenChildHoldsOutput uses a fake gpg that runs sleep as a
|
||||
// TestGPGTimeoutWhenChildHoldsOutput uses a fake gpg that runs sleep as a
|
||||
// child instead of exec-ing it, the way a wrapper script around the real
|
||||
// gpg might. Killing the fake gpg leaves sleep holding its stdout and
|
||||
// stderr open; the call must still return once ctx ends instead of waiting
|
||||
// for sleep to exit. The fake gpg writes the process ID of sleep to a named
|
||||
// pipe; the test ends ctx only after reading it, so sleep is running by
|
||||
// then, and kills sleep before returning.
|
||||
func TestGPGCancelWhenChildHoldsOutput(t *testing.T) {
|
||||
pidPipe := filepath.Join(t.TempDir(), "sleep.pid")
|
||||
require.NoError(t, syscall.Mkfifo(pidPipe, 0o600))
|
||||
// sleep outlasts the 10 s wait below, so a call that waits for it fails.
|
||||
// stderr open; the call must still return shortly after the deadline
|
||||
// instead of waiting for sleep to exit. The fake gpg writes the process ID
|
||||
// of sleep to a file so that the test can kill it before returning.
|
||||
func TestGPGTimeoutWhenChildHoldsOutput(t *testing.T) {
|
||||
pidFile := filepath.Join(t.TempDir(), "sleep.pid")
|
||||
t.Setenv("PATH", fakeGPGPath(t,
|
||||
"#!/bin/sh\nsleep 60 &\necho $! >'"+pidPipe+"'\nwait\n"))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
signErr := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
_, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
||||
signErr <- err
|
||||
}()
|
||||
|
||||
pid, err := os.ReadFile(pidPipe) //nolint:gosec // G304: path inside t.TempDir()
|
||||
"#!/bin/sh\nsleep 3 &\necho $! >'"+pidFile+"'\nwait\n"))
|
||||
t.Cleanup(func() {
|
||||
pid, err := os.ReadFile(pidFile) //nolint:gosec // G304: path inside t.TempDir()
|
||||
require.NoError(t, err)
|
||||
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(pid)))
|
||||
@@ -458,19 +444,17 @@ func TestGPGCancelWhenChildHoldsOutput(t *testing.T) {
|
||||
|
||||
sleep, err := os.FindProcess(n)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, sleep.Kill()) })
|
||||
require.NoError(t, sleep.Kill())
|
||||
})
|
||||
|
||||
cancel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
// The call should return about gpgWaitDelay (one second) after the
|
||||
// cancel. 10 s is far above that and well under the 30 s test timeout,
|
||||
// which would abort the whole package before the cleanup kills sleep.
|
||||
select {
|
||||
case err := <-signErr:
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the call waited for the child holding gpg's output to exit")
|
||||
}
|
||||
start := time.Now()
|
||||
_, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 3*time.Second,
|
||||
"the call waited for the child holding gpg's output to exit")
|
||||
}
|
||||
|
||||
// TestBuildPassesContextToSigning checks that a caller can cancel the gpg
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
103901c42b94396aa7ae128fd503ef693a4b7a03b2169481f25fda3d2c254e00 mf.proto
|
||||
@@ -1,29 +0,0 @@
|
||||
package mfer_test
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mf.pb.go is generated from mf.proto and committed. `make generate`
|
||||
// records the hash of the mf.proto it generated from in mf.proto.sha256.
|
||||
func TestGeneratedCodeMatchesProto(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
proto, err := os.ReadFile("mf.proto")
|
||||
require.NoError(t, err)
|
||||
|
||||
recorded, err := os.ReadFile("mf.proto.sha256")
|
||||
require.NoError(t, err)
|
||||
|
||||
recordedHash, _, _ := strings.Cut(string(recorded), " ")
|
||||
sum := sha256.Sum256(proto)
|
||||
require.Equal(t, recordedHash, hex.EncodeToString(sum[:]),
|
||||
"mfer/mf.proto has changed since mfer/mf.pb.go was generated "+
|
||||
"from it: run `make generate` and commit the result")
|
||||
}
|
||||
+1
-22
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -58,8 +57,6 @@ type ScannerOptions struct {
|
||||
SigningOptions *SigningOptions
|
||||
// Seed, if set, derives a deterministic UUID from this seed.
|
||||
Seed string
|
||||
// ExcludePaths lists files to leave out of the listing, matched with os.SameFile.
|
||||
ExcludePaths []string
|
||||
}
|
||||
|
||||
// FileEntry represents a file that has been enumerated.
|
||||
@@ -78,7 +75,6 @@ type Scanner struct {
|
||||
totalBytes FileSize // cached sum of all file sizes
|
||||
options *ScannerOptions
|
||||
fs afero.Fs
|
||||
excluded []fs.FileInfo // the files named in ExcludePaths that exist
|
||||
}
|
||||
|
||||
// NewScanner creates a new Scanner with default options.
|
||||
@@ -97,22 +93,11 @@ func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
||||
fs = afero.NewOsFs()
|
||||
}
|
||||
|
||||
s := &Scanner{
|
||||
return &Scanner{
|
||||
files: make([]*FileEntry, 0),
|
||||
options: opts,
|
||||
fs: fs,
|
||||
}
|
||||
|
||||
// A path that cannot be stat'd, normally because no file is there,
|
||||
// needs no leaving out.
|
||||
for _, p := range opts.ExcludePaths {
|
||||
info, err := s.fs.Stat(p)
|
||||
if err == nil {
|
||||
s.excluded = append(s.excluded, info)
|
||||
}
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
||||
@@ -450,12 +435,6 @@ func (s *Scanner) enumerateFileWithInfo(
|
||||
info = realInfo
|
||||
}
|
||||
|
||||
for _, excluded := range s.excluded {
|
||||
if os.SameFile(info, excluded) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
entry := &FileEntry{
|
||||
Path: RelFilePath(cleanPath),
|
||||
AbsPath: AbsFilePath(absPath),
|
||||
|
||||
+29
-2
@@ -304,19 +304,46 @@ func TestScannerEnumerateFS(t *testing.T) {
|
||||
func TestSendEnumerateStatusNonBlocking(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Nobody receives, so a blocking send would hang the test into its timeout.
|
||||
// Channel with no buffer - send should not block
|
||||
ch := make(chan EnumerateStatus)
|
||||
|
||||
// This should not block
|
||||
done := make(chan bool)
|
||||
|
||||
go func() {
|
||||
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
|
||||
|
||||
done <- true
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Success - did not block
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("sendEnumerateStatus blocked on full channel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendScanStatusNonBlocking(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Nobody receives, so a blocking send would hang the test into its timeout.
|
||||
// Channel with no buffer - send should not block
|
||||
ch := make(chan ScanStatus)
|
||||
|
||||
done := make(chan bool)
|
||||
|
||||
go func() {
|
||||
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
|
||||
|
||||
done <- true
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Success - did not block
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("sendScanStatus blocked on full channel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendStatusNilChannel(t *testing.T) {
|
||||
|
||||
File diff suppressed because one or more lines are too long
+2
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+6
-1
@@ -140,7 +140,12 @@ main() {
|
||||
|
||||
# ---- Go repos ----
|
||||
if missing go; then pkg_install go golang go go; fi
|
||||
# No golangci-lint: script/lint runs it in Docker only.
|
||||
# golangci-lint: packaged in nix, brew, and apk. On apt there is no
|
||||
# package: download a specific release archive from GitHub and
|
||||
# verify its hash (verify_sha256), never curl | sh.
|
||||
if missing golangci-lint; then
|
||||
pkg_install golangci-lint golangci-lint golangci-lint golangci-lint
|
||||
fi
|
||||
go mod download
|
||||
|
||||
# ---- Python repos ----
|
||||
|
||||
+12
@@ -5,9 +5,21 @@ set -eu
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||
# (mirrors the old Makefile prerequisite; the generated file is
|
||||
# committed, so this is normally a no-op).
|
||||
ensure_pb() {
|
||||
if [ ! -f mfer/mf.pb.go ] ||
|
||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||
(cd mfer && go generate .)
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
ensure_pb
|
||||
gofumpt -l -w mfer internal cmd
|
||||
golangci-lint run --fix
|
||||
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
||||
"$SCRIPT_DIR/prettier" --write
|
||||
}
|
||||
|
||||
@@ -6,8 +6,19 @@ set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||
# (mirrors the old Makefile prerequisite; the generated file is
|
||||
# committed, so this is normally a no-op).
|
||||
ensure_pb() {
|
||||
if [ ! -f mfer/mf.pb.go ] ||
|
||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||
(cd mfer && go generate .)
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
ensure_pb
|
||||
if [ -n "$(gofmt -l .)" ]; then
|
||||
echo "gofmt: files need formatting:" >&2
|
||||
gofmt -l . >&2
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/generate: regenerate mfer/mf.pb.go from mfer/mf.proto, and record
|
||||
# the hash of that mf.proto in mfer/mf.proto.sha256. Nothing else
|
||||
# regenerates mf.pb.go: it is committed, so building and checking need no
|
||||
# protoc. A test fails while mf.proto no longer matches the recorded hash.
|
||||
#
|
||||
# Needs exactly the protoc and protoc-gen-go versions named in the header of
|
||||
# the committed mf.pb.go (README.md says how to install them). Another
|
||||
# version writes a different mf.pb.go, so the script refuses to run.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
|
||||
PROTOC_VERSION="33.4"
|
||||
PROTOC_GEN_GO_VERSION="v1.36.11"
|
||||
|
||||
# require_version <command> <its exact --version output>
|
||||
require_version() {
|
||||
actual="$("$1" --version 2>/dev/null || true)"
|
||||
if [ "$actual" != "$2" ]; then
|
||||
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
|
||||
echo " README.md says how to install it." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
|
||||
# where there is no sha256sum.
|
||||
sha256() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1"
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
shasum -a 256 "$1"
|
||||
else
|
||||
echo "generate: needs sha256sum or shasum on PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT/mfer"
|
||||
require_version protoc "libprotoc $PROTOC_VERSION"
|
||||
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
|
||||
# Hashed before regenerating, so a missing hash tool stops the script
|
||||
# before it changes anything. Regenerating leaves mf.proto as it is.
|
||||
proto_hash="$(sha256 mf.proto)"
|
||||
go generate .
|
||||
echo "$proto_hash" >mf.proto.sha256
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+8
-11
@@ -1,20 +1,17 @@
|
||||
#!/bin/sh
|
||||
# script/lint: run golangci-lint, in Docker only. Builds the lint stage of
|
||||
# the Dockerfile, whose build runs the linter, so a successful build is a
|
||||
# clean lint. --no-cache because a cached build runs no linter. The image
|
||||
# is removed afterwards, whatever the outcome.
|
||||
# script/lint: run the linter.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
# Tagged per run, so concurrent runs never remove each other's image.
|
||||
image="$("$SCRIPT_DIR/projectname")-lint:$$"
|
||||
# A failed build leaves no image, so there is nothing to remove then.
|
||||
trap 'docker image rm "$image" >/dev/null 2>&1 || true' EXIT INT TERM
|
||||
docker build --no-cache --target lint -t "$image" .
|
||||
golangci-lint run
|
||||
if [ -n "$(gofmt -l .)" ]; then
|
||||
echo "gofmt: files need formatting:" >&2
|
||||
gofmt -l . >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+11
@@ -4,8 +4,19 @@ set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
||||
# (mirrors the old Makefile prerequisite; the generated file is
|
||||
# committed, so this is normally a no-op).
|
||||
ensure_pb() {
|
||||
if [ ! -f mfer/mf.pb.go ] ||
|
||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
||||
(cd mfer && go generate .)
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
ensure_pb
|
||||
go test -timeout 30s -race -cover ./... ||
|
||||
{
|
||||
echo "--- Rerunning with -v for details ---"
|
||||
|
||||
Reference in New Issue
Block a user