Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
114ba0a29f | ||
|
|
64eb5cbd40 | ||
|
|
45eac1f6f8 | ||
|
|
b91e92b070 | ||
|
|
a2732cf8da | ||
|
|
a789eb3083 |
@@ -16,6 +16,7 @@ linters:
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
settings:
|
||||
|
||||
+7
-2
@@ -14,7 +14,9 @@ RUN touch mfer/mf.pb.go
|
||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
||||
# markdown half runs in the mdfmt stage below.
|
||||
RUN make fmt-check-go
|
||||
RUN make lint
|
||||
# The linter directly, not `make lint`: script/lint builds this stage, and
|
||||
# there is no docker inside this build.
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
||||
@@ -67,7 +69,10 @@ RUN version="${VERSION:-$(git describe --tags --always)}"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
cd cmd/mfer && \
|
||||
go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
|
||||
CGO_ENABLED=0 go build -tags urfave_cli_no_docs -ldflags "-X main.Gitrev=$version" -o /mfer .
|
||||
|
||||
# Fail unless /mfer is statically linked: scratch has no C library to run it.
|
||||
RUN ldd /mfer 2>&1 | grep -q 'not a dynamic executable'
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /mfer /mfer
|
||||
|
||||
@@ -58,9 +58,6 @@ fmt-check-md:
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
devprereqs:
|
||||
which golangci-lint || go install -v github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
|
||||
mfer/mf.pb.go: mfer/mf.proto
|
||||
cd mfer && go generate .
|
||||
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# mfer
|
||||
|
||||
[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library and
|
||||
thin wrapper command-line utility written in [Go](https://golang.org) and first
|
||||
published in 2022 under the [WTFPL](https://wtfpl.net) (public domain) license.
|
||||
It specifies and generates `.mf` manifest files over a directory tree of files
|
||||
to encapsulate metadata about them (such as cryptographic checksums or
|
||||
signatures over same) to aid in archiving, downloading, and streaming, or
|
||||
mirroring. The manifest files' data is serialized with Google's
|
||||
[mfer](https://git.eeqj.de/sneak/mfer) is a [WTFPL](https://wtfpl.net)-licensed
|
||||
(public domain) [Go](https://golang.org) library and command-line tool by
|
||||
[@sneak](https://sneak.berlin) that specifies and generates `.mf` manifest files
|
||||
over a directory tree to encapsulate metadata about the files — such as
|
||||
cryptographic checksums and signatures over same — to aid in archiving,
|
||||
downloading, streaming, and mirroring. It was first published in 2022. The
|
||||
manifest files' data is serialized with Google's
|
||||
[protobuf serialization format](https://developers.google.com/protocol-buffers).
|
||||
The structure of these files can be found
|
||||
[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
||||
@@ -21,6 +21,36 @@ This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
||||
as a de-facto standard and be incorporated into other software. A compatible
|
||||
javascript library is planned.
|
||||
|
||||
# Getting Started
|
||||
|
||||
`mfer` builds from source with a Go 1.23+ toolchain. The generated protobuf code
|
||||
is committed, so no `protoc` toolchain is required:
|
||||
|
||||
```sh
|
||||
git clone https://git.eeqj.de/sneak/mfer.git
|
||||
cd mfer
|
||||
go build -o bin/mfer ./cmd/mfer
|
||||
```
|
||||
|
||||
Generate a manifest for a directory tree, verify it later, and fetch a published
|
||||
tree by URL:
|
||||
|
||||
```sh
|
||||
# Write .index.mf, a manifest of the files under the current directory.
|
||||
bin/mfer gen .
|
||||
|
||||
# Verify the files on disk against the manifest. Exits nonzero if any file
|
||||
# is missing or corrupted.
|
||||
bin/mfer check .index.mf
|
||||
|
||||
# Download and cryptographically verify a tree published over HTTP: mfer
|
||||
# fetches <url>/index.mf, then downloads every file it lists.
|
||||
bin/mfer fetch https://example.com/tree/
|
||||
```
|
||||
|
||||
Run `bin/mfer help` for the full command list, or `bin/mfer <command> --help`
|
||||
for a single command's options.
|
||||
|
||||
# Build Status
|
||||
|
||||
CI runs `script/cibuild`, which builds the Docker image with `--no-cache`, so
|
||||
@@ -35,9 +65,9 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
||||
development workflow, and the Makefile targets are thin shims that call them. We
|
||||
provide:
|
||||
|
||||
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go module
|
||||
download, and node/yarn plus the prettier version pinned in
|
||||
`package.json`/`yarn.lock`), idempotently
|
||||
- `script/bootstrap` — install all dependencies (Go, Go module download, and
|
||||
node/yarn plus the prettier version pinned in `package.json`/`yarn.lock`),
|
||||
idempotently; golangci-lint is not installed, it runs only in Docker
|
||||
- `script/setup` — make a fresh clone ready for development: runs
|
||||
`script/bootstrap`, then `script/install-precommit`
|
||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
||||
@@ -47,9 +77,11 @@ provide:
|
||||
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand
|
||||
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
|
||||
as ordinary tests
|
||||
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
|
||||
- `script/fmt` — format all code and docs (writes): `gofumpt`,
|
||||
`golangci-lint run --fix`, and `script/prettier --write`
|
||||
- `script/lint` — run `golangci-lint` in Docker: builds only the `lint` stage of
|
||||
the `Dockerfile` (the Go format check, then the linter), uncached so it runs
|
||||
every time, then removes the image
|
||||
- `script/fmt` — format all code and docs (writes): `gofumpt` and
|
||||
`script/prettier --write`
|
||||
- `script/prettier` — run prettier over the repository's canonical file set
|
||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
||||
`--check`; the single definition of that file set, so `script/fmt` and
|
||||
@@ -75,17 +107,18 @@ yet. Primary development happens on a privately-run Gitea instance at
|
||||
[tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
||||
|
||||
Changes must always be formatted with a standard `go fmt`, syntactically valid,
|
||||
and must pass the linting defined in the repository (presently only the
|
||||
`golangci-lint` defaults), which can be run with a `make lint`. The `main`
|
||||
branch is protected and all changes must be made via
|
||||
[pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
||||
Any changes submitted to this project must also be
|
||||
and must pass the linting defined in the repository's `.golangci.yml`, which
|
||||
`make lint` runs in Docker. The `main` branch is protected and all changes must
|
||||
be made via [pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to
|
||||
be merged. Any changes submitted to this project must also be
|
||||
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
||||
|
||||
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
|
||||
tooling requirements, and workflow conventions.
|
||||
|
||||
# Problem Statement
|
||||
# Rationale
|
||||
|
||||
## The problem
|
||||
|
||||
Given a plain URL, there is no standard way to safely and programmatically
|
||||
download everything "under" that URL path. `wget -r` can traverse directory
|
||||
@@ -109,7 +142,7 @@ Real issues I face:
|
||||
- when I download a large file via HTTP, I have no way of knowing if the file
|
||||
content is what it's supposed to be
|
||||
|
||||
# Proposed Solution
|
||||
## The solution
|
||||
|
||||
A standard, a manifest file format, and a tool for generating same.
|
||||
|
||||
@@ -141,6 +174,28 @@ The manifest file would do several important things:
|
||||
- maybe a bittorrent chunklist for torrent client compatibility? perhaps a
|
||||
top-level infohash for the whole manifest?
|
||||
|
||||
# Design
|
||||
|
||||
The repository is split into a reusable library and a thin command-line wrapper
|
||||
around it.
|
||||
|
||||
- `mfer/` is the reusable library and the heart of the project: it defines the
|
||||
manifest format and implements building, scanning, checking, serialization,
|
||||
and signing. The protobuf schema is `mfer/mf.proto`, and the generated code it
|
||||
produces (`mfer/mf.pb.go`) is committed alongside it so the library builds
|
||||
with `go get` and needs no `protoc` toolchain.
|
||||
- `internal/cli/` holds the command implementations — `generate` (alias `gen`),
|
||||
`check`, `freshen`, `export`, `list` (alias `ls`), `fetch`, and `version` —
|
||||
that wire the library to the command-line interface.
|
||||
- `internal/log/` provides the logging used by the commands and the library.
|
||||
- `internal/bork/` provides the error the library returns when a manifest's
|
||||
decompressed contents are not the size the manifest records.
|
||||
- `cmd/mfer/` is the entrypoint: its `main` package runs `internal/cli` and
|
||||
exits with the status it returns.
|
||||
|
||||
Everything under `internal/` is private to this repository; only the `mfer/`
|
||||
package is intended for import by other software.
|
||||
|
||||
# Design Goals
|
||||
|
||||
- Replace SHASUMS/SHASUMS.asc files
|
||||
@@ -274,9 +329,9 @@ Open work, open design questions included, is tracked in this repo's issues:
|
||||
- Issues:
|
||||
[https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
||||
|
||||
# Authors
|
||||
# Author
|
||||
|
||||
- [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
|
||||
- [@sneak](https://sneak.berlin)
|
||||
|
||||
# License
|
||||
|
||||
|
||||
+125
-3
@@ -5,6 +5,7 @@ import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sync"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
urfcli "github.com/urfave/cli/v2"
|
||||
"sneak.berlin/go/mfer/internal/log"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
@@ -27,6 +29,7 @@ const (
|
||||
testManifest = "/manifest.mf"
|
||||
testFlagBase = "--base"
|
||||
testFlagNoExtra = "--no-extra-files"
|
||||
testFlagVersion = "--version"
|
||||
)
|
||||
|
||||
var errSimulatedWrite = errors.New("simulated write failure")
|
||||
@@ -39,12 +42,20 @@ var errSimulatedWrite = errors.New("simulated write failure")
|
||||
var runMu sync.Mutex
|
||||
|
||||
// runCLI invokes RunWithOptions while holding runMu so parallel tests
|
||||
// capture their own output.
|
||||
// capture their own output. Before releasing the lock it points the
|
||||
// process-global logger at io.Discard: other tests log outside the lock
|
||||
// (manifest loads, scans), and those lines must not land in this run's
|
||||
// buffers once it has returned and its test is reading them.
|
||||
func runCLI(opts *RunOptions) int {
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
return RunWithOptions(opts)
|
||||
exitCode := RunWithOptions(opts)
|
||||
|
||||
log.SetOutput(io.Discard, io.Discard)
|
||||
log.Init()
|
||||
|
||||
return exitCode
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -102,7 +113,7 @@ func TestVersionCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
opts := testOpts([]string{testApp, "version"}, fs)
|
||||
opts := testOpts([]string{testApp, cmdVersion}, fs)
|
||||
|
||||
exitCode := runCLI(opts)
|
||||
|
||||
@@ -113,6 +124,117 @@ func TestVersionCommand(t *testing.T) {
|
||||
assert.Contains(t, stdout, "abc123")
|
||||
}
|
||||
|
||||
// TestVFlagCollision covers the -v/--verbose vs --version flag interaction
|
||||
// (issue #64). Verbose owns -v; version answers to --version and -V. None of
|
||||
// these invocations may produce a parser error, and the two ways of asking
|
||||
// for the version must print the same thing.
|
||||
func TestVFlagCollision(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Invocations that must print the version and exit 0.
|
||||
versionCases := map[string][]string{
|
||||
"long version flag": {testApp, testFlagVersion},
|
||||
"short version flag": {testApp, "-V"},
|
||||
"verbose then version": {testApp, "-v", testFlagVersion},
|
||||
"long verbose and version": {testApp, "--verbose", testFlagVersion},
|
||||
}
|
||||
|
||||
for name, args := range versionCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(args, afero.NewMemMapFs())
|
||||
exitCode := runCLI(opts)
|
||||
|
||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
||||
assert.Contains(t, testStdout(t, opts), mfer.Version)
|
||||
assert.NotContains(t, testStderr(t, opts), "two forms of the same flag")
|
||||
})
|
||||
}
|
||||
|
||||
// Invocations that must enable verbose and exit 0 without a parser error.
|
||||
verboseCases := map[string][]string{
|
||||
"short verbose flag": {testApp, "-v"},
|
||||
"long verbose flag": {testApp, "--verbose"},
|
||||
}
|
||||
|
||||
for name, args := range verboseCases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(args, afero.NewMemMapFs())
|
||||
exitCode := runCLI(opts)
|
||||
|
||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
||||
assert.Contains(t, testStdout(t, opts), cmdGenerate,
|
||||
"root should show help listing subcommands")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionFlagAndCommandMatch asserts that "mfer --version" and
|
||||
// "mfer version" produce identical output (issue #64).
|
||||
func TestVersionFlagAndCommandMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
flagOpts := testOpts([]string{testApp, testFlagVersion}, afero.NewMemMapFs())
|
||||
require.Equal(t, 0, runCLI(flagOpts))
|
||||
|
||||
cmdOpts := testOpts([]string{testApp, cmdVersion}, afero.NewMemMapFs())
|
||||
require.Equal(t, 0, runCLI(cmdOpts))
|
||||
|
||||
assert.Equal(t, testStdout(t, flagOpts), testStdout(t, cmdOpts))
|
||||
}
|
||||
|
||||
// TestRootVerbosityFlags asserts that -q and -v given before any subcommand
|
||||
// name take effect: in the root itself, and in the fetch and export
|
||||
// subcommands (issue #64). It does not run in parallel: other tests log
|
||||
// outside runMu (manifest loads, scans), and a line logged while one of these
|
||||
// runs is in progress lands in its output.
|
||||
//
|
||||
//nolint:paralleltest // see above
|
||||
func TestRootVerbosityFlags(t *testing.T) {
|
||||
t.Run("quiet with no subcommand hides the banner", func(t *testing.T) {
|
||||
loud := testOpts([]string{testApp}, afero.NewMemMapFs())
|
||||
require.Equal(t, 0, runCLI(loud))
|
||||
assert.Contains(t, testStdout(t, loud), banner)
|
||||
|
||||
quiet := testOpts([]string{testApp, "-q"}, afero.NewMemMapFs())
|
||||
require.Equal(t, 0, runCLI(quiet))
|
||||
assert.Contains(t, testStdout(t, quiet), cmdGenerate)
|
||||
assert.NotContains(t, testStdout(t, quiet), banner)
|
||||
})
|
||||
|
||||
t.Run("quiet before fetch hides the banner", func(t *testing.T) {
|
||||
opts := testOpts([]string{testApp, "-q", cmdFetch}, afero.NewMemMapFs())
|
||||
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), errURLRequired.Error())
|
||||
assert.NotContains(t, testStdout(t, opts), banner)
|
||||
})
|
||||
|
||||
t.Run("verbose twice before fetch enables debug logging", func(t *testing.T) {
|
||||
opts := testOpts([]string{testApp, "-v", "-v", cmdFetch}, afero.NewMemMapFs())
|
||||
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), "fetchManifestOperation()")
|
||||
})
|
||||
|
||||
t.Run("quiet before export hides the manifest summary", func(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
manifest := buildTestManifest(t, map[string][]byte{"a.txt": []byte("a")})
|
||||
require.NoError(t, afero.WriteFile(fs, testManifest, manifest, 0o644))
|
||||
|
||||
loud := testOpts([]string{testApp, cmdExport, testManifest}, fs)
|
||||
require.Equal(t, 0, runCLI(loud))
|
||||
assert.Contains(t, testStderr(t, loud), "loaded manifest")
|
||||
|
||||
quiet := testOpts([]string{testApp, "-q", cmdExport, testManifest}, fs)
|
||||
require.Equal(t, 0, runCLI(quiet))
|
||||
assert.NotContains(t, testStderr(t, quiet), "loaded manifest")
|
||||
})
|
||||
}
|
||||
|
||||
func TestHelpCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+45
-5
@@ -19,6 +19,7 @@ const (
|
||||
cmdCheck = "check"
|
||||
cmdExport = "export"
|
||||
cmdFetch = "fetch"
|
||||
cmdVersion = "version"
|
||||
|
||||
flagProgress = "progress"
|
||||
|
||||
@@ -68,6 +69,12 @@ func (mfa *CLIApp) VersionString() string {
|
||||
return mfer.Version
|
||||
}
|
||||
|
||||
// printVersion writes the version line shared by the --version flag and the
|
||||
// version subcommand, so both produce identical output.
|
||||
func (mfa *CLIApp) printVersion() {
|
||||
_, _ = fmt.Fprintf(mfa.Stdout, "%s version %s\n", mfa.appname, mfa.VersionString())
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) printBanner() {
|
||||
if log.GetLevel() <= log.InfoLevel {
|
||||
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
||||
@@ -78,20 +85,34 @@ func (mfa *CLIApp) printBanner() {
|
||||
}
|
||||
}
|
||||
|
||||
// setVerbosity sets the log level from -v and -q, given before the subcommand
|
||||
// name (the root's copies), after it (the subcommand's own copies), or both.
|
||||
// urfave/cli reads a flag from the nearest command that defines it, so each
|
||||
// command in the lineage is asked. The highest -v count wins rather than the
|
||||
// sum, because a subcommand without its own copies reads the root's.
|
||||
func (mfa *CLIApp) setVerbosity(c *cli.Context) {
|
||||
_, present := os.LookupEnv("MFER_DEBUG")
|
||||
|
||||
verbosity := 0
|
||||
quiet := false
|
||||
|
||||
for _, ctx := range c.Lineage() {
|
||||
verbosity = max(verbosity, ctx.Count("verbose"))
|
||||
quiet = quiet || ctx.Bool("quiet")
|
||||
}
|
||||
|
||||
switch {
|
||||
case present:
|
||||
log.EnableDebugLogging()
|
||||
case c.Bool("quiet"):
|
||||
case quiet:
|
||||
log.SetLevel(log.ErrorLevel)
|
||||
default:
|
||||
log.SetLevelFromVerbosity(c.Count("verbose"))
|
||||
log.SetLevelFromVerbosity(verbosity)
|
||||
}
|
||||
}
|
||||
|
||||
// commonFlags returns the flags shared by most commands (-v, -q)
|
||||
// commonFlags returns the -v and -q flags taken by the root and by the
|
||||
// generate, check, freshen and fetch subcommands.
|
||||
func commonFlags() []cli.Flag {
|
||||
return []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
@@ -259,6 +280,8 @@ func (mfa *CLIApp) exportCommand() *cli.Command {
|
||||
Usage: "Export manifest contents as JSON",
|
||||
ArgsUsage: "[manifest file or URL]",
|
||||
Action: func(c *cli.Context) error {
|
||||
mfa.setVerbosity(c)
|
||||
|
||||
return mfa.exportManifestOperation(c)
|
||||
},
|
||||
}
|
||||
@@ -266,10 +289,10 @@ func (mfa *CLIApp) exportCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) versionCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "version",
|
||||
Name: cmdVersion,
|
||||
Usage: "Show version",
|
||||
Action: func(_ *cli.Context) error {
|
||||
_, _ = fmt.Fprintln(mfa.Stdout, mfa.VersionString())
|
||||
mfa.printVersion()
|
||||
|
||||
return nil
|
||||
},
|
||||
@@ -325,6 +348,21 @@ func (mfa *CLIApp) run(args []string) {
|
||||
log.SetOutput(mfa.Stdout, mfa.Stderr)
|
||||
log.Init()
|
||||
|
||||
// -v means verbose, not version. urfave/cli's built-in version flag
|
||||
// claims -v by default, which made "mfer -v --version" fail to parse and
|
||||
// gave -v a different meaning at the root than on the generate, check,
|
||||
// freshen and fetch subcommands, where it means verbose. Verbose is the
|
||||
// more common meaning of -v in tools that offer both, so -v means verbose
|
||||
// at the root too and the version flag takes the capital -V.
|
||||
// VersionFlag and VersionPrinter are urfave/cli package globals; run() is
|
||||
// serialized in tests, so assigning them here is safe.
|
||||
cli.VersionFlag = &cli.BoolFlag{
|
||||
Name: cmdVersion,
|
||||
Aliases: []string{"V"},
|
||||
Usage: "print the version",
|
||||
}
|
||||
cli.VersionPrinter = func(_ *cli.Context) { mfa.printVersion() }
|
||||
|
||||
mfa.app = &cli.App{
|
||||
Name: mfa.appname,
|
||||
Usage: "Manifest generator",
|
||||
@@ -332,11 +370,13 @@ func (mfa *CLIApp) run(args []string) {
|
||||
EnableBashCompletion: true,
|
||||
Writer: mfa.Stdout,
|
||||
ErrWriter: mfa.Stderr,
|
||||
Flags: commonFlags(),
|
||||
Action: func(c *cli.Context) error {
|
||||
if c.Args().Len() > 0 {
|
||||
return fmt.Errorf("%w %q", errUnknownCommand, c.Args().First())
|
||||
}
|
||||
|
||||
mfa.setVerbosity(c)
|
||||
mfa.printBanner()
|
||||
|
||||
return cli.ShowAppHelp(c)
|
||||
|
||||
+40
-24
@@ -10,6 +10,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -425,36 +426,51 @@ func TestGPGTimeoutKillsGPG(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "gpg sign failed: gpg timed out")
|
||||
}
|
||||
|
||||
// TestGPGTimeoutWhenChildHoldsOutput uses a fake gpg that runs sleep as a
|
||||
// TestGPGCancelWhenChildHoldsOutput uses a fake gpg that runs sleep as a
|
||||
// child instead of exec-ing it, the way a wrapper script around the real
|
||||
// gpg might. Killing the fake gpg leaves sleep holding its stdout and
|
||||
// stderr open; the call must still return shortly after the deadline
|
||||
// instead of waiting for sleep to exit. The fake gpg writes the process ID
|
||||
// of sleep to a file so that the test can kill it before returning.
|
||||
func TestGPGTimeoutWhenChildHoldsOutput(t *testing.T) {
|
||||
pidFile := filepath.Join(t.TempDir(), "sleep.pid")
|
||||
// stderr open; the call must still return once ctx ends instead of waiting
|
||||
// for sleep to exit. The fake gpg writes the process ID of sleep to a named
|
||||
// pipe; the test ends ctx only after reading it, so sleep is running by
|
||||
// then, and kills sleep before returning.
|
||||
func TestGPGCancelWhenChildHoldsOutput(t *testing.T) {
|
||||
pidPipe := filepath.Join(t.TempDir(), "sleep.pid")
|
||||
require.NoError(t, syscall.Mkfifo(pidPipe, 0o600))
|
||||
// sleep outlasts the 10 s wait below, so a call that waits for it fails.
|
||||
t.Setenv("PATH", fakeGPGPath(t,
|
||||
"#!/bin/sh\nsleep 3 &\necho $! >'"+pidFile+"'\nwait\n"))
|
||||
t.Cleanup(func() {
|
||||
pid, err := os.ReadFile(pidFile) //nolint:gosec // G304: path inside t.TempDir()
|
||||
require.NoError(t, err)
|
||||
"#!/bin/sh\nsleep 60 &\necho $! >'"+pidPipe+"'\nwait\n"))
|
||||
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(pid)))
|
||||
require.NoError(t, err)
|
||||
|
||||
sleep, err := os.FindProcess(n)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sleep.Kill())
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
_, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 3*time.Second,
|
||||
"the call waited for the child holding gpg's output to exit")
|
||||
signErr := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
_, err := gpgSign(ctx, []byte("data"), GPGKeyID("any"))
|
||||
signErr <- err
|
||||
}()
|
||||
|
||||
pid, err := os.ReadFile(pidPipe) //nolint:gosec // G304: path inside t.TempDir()
|
||||
require.NoError(t, err)
|
||||
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(pid)))
|
||||
require.NoError(t, err)
|
||||
|
||||
sleep, err := os.FindProcess(n)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, sleep.Kill()) })
|
||||
|
||||
cancel()
|
||||
|
||||
// The call should return about gpgWaitDelay (one second) after the
|
||||
// cancel. 10 s is far above that and well under the 30 s test timeout,
|
||||
// which would abort the whole package before the cleanup kills sleep.
|
||||
select {
|
||||
case err := <-signErr:
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the call waited for the child holding gpg's output to exit")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPassesContextToSigning checks that a caller can cancel the gpg
|
||||
|
||||
+4
-31
@@ -304,46 +304,19 @@ func TestScannerEnumerateFS(t *testing.T) {
|
||||
func TestSendEnumerateStatusNonBlocking(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Channel with no buffer - send should not block
|
||||
// Nobody receives, so a blocking send would hang the test into its timeout.
|
||||
ch := make(chan EnumerateStatus)
|
||||
|
||||
// This should not block
|
||||
done := make(chan bool)
|
||||
|
||||
go func() {
|
||||
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
|
||||
|
||||
done <- true
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Success - did not block
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("sendEnumerateStatus blocked on full channel")
|
||||
}
|
||||
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
|
||||
}
|
||||
|
||||
func TestSendScanStatusNonBlocking(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Channel with no buffer - send should not block
|
||||
// Nobody receives, so a blocking send would hang the test into its timeout.
|
||||
ch := make(chan ScanStatus)
|
||||
|
||||
done := make(chan bool)
|
||||
|
||||
go func() {
|
||||
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
|
||||
|
||||
done <- true
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Success - did not block
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Fatal("sendScanStatus blocked on full channel")
|
||||
}
|
||||
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
|
||||
}
|
||||
|
||||
func TestSendStatusNilChannel(t *testing.T) {
|
||||
|
||||
+1
-6
@@ -140,12 +140,7 @@ main() {
|
||||
|
||||
# ---- Go repos ----
|
||||
if missing go; then pkg_install go golang go go; fi
|
||||
# golangci-lint: packaged in nix, brew, and apk. On apt there is no
|
||||
# package: download a specific release archive from GitHub and
|
||||
# verify its hash (verify_sha256), never curl | sh.
|
||||
if missing golangci-lint; then
|
||||
pkg_install golangci-lint golangci-lint golangci-lint golangci-lint
|
||||
fi
|
||||
# No golangci-lint: script/lint runs it in Docker only.
|
||||
go mod download
|
||||
|
||||
# ---- Python repos ----
|
||||
|
||||
@@ -19,7 +19,6 @@ main() {
|
||||
cd "$ROOT"
|
||||
ensure_pb
|
||||
gofumpt -l -w mfer internal cmd
|
||||
golangci-lint run --fix
|
||||
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
||||
"$SCRIPT_DIR/prettier" --write
|
||||
}
|
||||
|
||||
+11
-8
@@ -1,17 +1,20 @@
|
||||
#!/bin/sh
|
||||
# script/lint: run the linter.
|
||||
# script/lint: run golangci-lint, in Docker only. Builds the lint stage of
|
||||
# the Dockerfile, whose build runs the linter, so a successful build is a
|
||||
# clean lint. --no-cache because a cached build runs no linter. The image
|
||||
# is removed afterwards, whatever the outcome.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
golangci-lint run
|
||||
if [ -n "$(gofmt -l .)" ]; then
|
||||
echo "gofmt: files need formatting:" >&2
|
||||
gofmt -l . >&2
|
||||
exit 1
|
||||
fi
|
||||
# Tagged per run, so concurrent runs never remove each other's image.
|
||||
image="$("$SCRIPT_DIR/projectname")-lint:$$"
|
||||
# A failed build leaves no image, so there is nothing to remove then.
|
||||
trap 'docker image rm "$image" >/dev/null 2>&1 || true' EXIT INT TERM
|
||||
docker build --no-cache --target lint -t "$image" .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user