Compare commits
13
Commits
9a1641890d
..
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a3f532c8d | ||
|
|
0baf83deb6 | ||
|
|
589f1a9580 | ||
|
|
7542a68863 | ||
|
|
6540e5f285 | ||
|
|
2ac238d767 | ||
|
|
2cfea66d27 | ||
|
|
8ec3fd9a67 | ||
|
|
0057f6463b | ||
|
|
7abe9a0ab1 | ||
|
|
9199444d11 | ||
|
|
60dbcc03bc | ||
|
|
a79dd463f1 |
@@ -17,6 +17,15 @@ The current version is pre-1.0 and while the repo was published in 2022, there
|
||||
has not yet been any versioned release. [SemVer](https://semver.org) will be
|
||||
used for releases.
|
||||
|
||||
A version is a git tag, and no file in the tree records a version number.
|
||||
`mfer version` prints the revision the build stamped into the binary, which for
|
||||
`make build` and the Docker image is the output of `git describe` on the
|
||||
checkout they build. A binary built without that stamp, as
|
||||
`go install sneak.berlin/go/mfer/cmd/mfer@<tag>` builds it, prints the module
|
||||
version Go records in it instead. A program that imports the `mfer` package
|
||||
finds the version it was built against in its own build information
|
||||
(`runtime/debug.ReadBuildInfo`).
|
||||
|
||||
This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
||||
2022 and is currently a one-person effort, though the goal is for this to emerge
|
||||
as a de-facto standard and be incorporated into other software. A compatible
|
||||
@@ -289,9 +298,12 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
else the directory holding the manifest, or the current directory for a
|
||||
manifest given by URL. So `mfer check /media/drive` checks a drive against
|
||||
the `index.mf` at its root, from any directory
|
||||
- warns about each file under the base directory that the manifest does not
|
||||
list, hidden files included; with `--no-extra-files` each one is a failure
|
||||
instead
|
||||
- warns about each regular file under the base directory that the manifest
|
||||
does not list, hidden files included, and each symlink there to a regular
|
||||
file, which gen lists only with `--follow-symlinks`; with
|
||||
`--no-extra-files` each one is a failure instead. A named pipe, socket or
|
||||
device, a symlink to a directory and a broken symlink are never reported,
|
||||
since gen cannot list them
|
||||
- `mfer freshen` / `mfer freshen .`
|
||||
- rewrites `index.mf` to list the files now under the directory holding it,
|
||||
or under the one `--base` names, hashing only the files that are new or
|
||||
@@ -300,6 +312,9 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
||||
- leaves out hidden files unless given `--include-dotfiles`, and symlinks
|
||||
unless given `--follow-symlinks`, which lists each symlink to a regular
|
||||
file under its own name with the contents of the file it points to
|
||||
- refuses a signed manifest, leaving it as it was and naming the fingerprint
|
||||
of the key that signed it, unless `--sign-key` or `MFER_SIGN_KEY` names a
|
||||
key to sign the new one with
|
||||
- `mfer gen --sign-key key.asc` / `mfer freshen --sign-key key.asc`
|
||||
- signs the manifest with the OpenPGP secret key in `key.asc`, armored or
|
||||
binary, as `gpg --export-secret-keys` writes it; `MFER_SIGN_KEY` names the
|
||||
|
||||
+4
-6
@@ -10,14 +10,12 @@ import (
|
||||
// Appname is the name of this program.
|
||||
const Appname = "mfer"
|
||||
|
||||
// Version and Gitrev are injected at build time via -ldflags.
|
||||
// Gitrev is the revision `mfer version` prints, injected at build time via
|
||||
// -ldflags.
|
||||
//
|
||||
//nolint:gochecknoglobals // set via ldflags at build time
|
||||
var (
|
||||
Version string
|
||||
Gitrev string
|
||||
)
|
||||
var Gitrev string
|
||||
|
||||
func main() {
|
||||
os.Exit(cli.Run(Appname, Version, Gitrev))
|
||||
os.Exit(cli.Run(Appname, Gitrev))
|
||||
}
|
||||
|
||||
@@ -15,11 +15,10 @@ func TestAppname(t *testing.T) {
|
||||
assert.Equal(t, "mfer", Appname)
|
||||
}
|
||||
|
||||
// TestVersionDefaults documents that Version and Gitrev are empty unless
|
||||
// injected at build time via -ldflags.
|
||||
func TestVersionDefaults(t *testing.T) {
|
||||
// TestGitrevDefault documents that Gitrev is empty unless injected at build
|
||||
// time via -ldflags.
|
||||
func TestGitrevDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Empty(t, Version)
|
||||
assert.Empty(t, Gitrev)
|
||||
}
|
||||
|
||||
+13
-3
@@ -112,6 +112,7 @@ All `path` values must satisfy these invariants:
|
||||
- **Forward slashes**: use `/` as the path separator (never `\`)
|
||||
- **Relative only**: no leading `/`
|
||||
- **No parent traversal**: no `..` path segments
|
||||
- **No current-directory segments**: no `.` path segments
|
||||
- **No empty segments**: no `//` sequences
|
||||
- **No trailing slash**: paths refer to files, not directories
|
||||
- **Listed once**: each path appears at most once in a manifest, compared byte
|
||||
@@ -183,9 +184,18 @@ also refuses a manifest whose `signingPubKey` holds a DSA key or subkey, or any
|
||||
secret key or subkey, since checking the self-signatures of a DSA key or the
|
||||
numbers of a secret key can take hours when those numbers are very large.
|
||||
|
||||
A signing key that has expired does not make the signature bad, because a
|
||||
manifest outlives its key. A signature past its own expiry time is not good,
|
||||
whether or not the key has expired too.
|
||||
A signature is good only if the primary key, and the subkey that made the
|
||||
signature, were valid at the signature's creation time under the key validity
|
||||
rules of RFC 9580. The key is judged by its current self-signatures and binding
|
||||
signatures: for a version 4 key the newest self-signature on its primary user
|
||||
ID, for a version 6 key its newest direct-key signature, and for a subkey its
|
||||
newest binding signature with the signature embedded in it. So a signature made
|
||||
before the newest of those is not good, even if an older one made the key valid
|
||||
at that time. The reference implementation checks this by verifying the
|
||||
signature a second time with its clock set to the signature's creation time. A
|
||||
key that has expired since does not make the signature bad, because a manifest
|
||||
outlives its key. A signature past its own expiry time is not good, whether or
|
||||
not the key has expired too.
|
||||
|
||||
## Deterministic Serialization
|
||||
|
||||
|
||||
@@ -329,10 +329,10 @@ func (mfa *CLIApp) checkManifestOperation(
|
||||
return fmt.Errorf("load manifest: %w", err)
|
||||
}
|
||||
|
||||
// Check signature requirement
|
||||
requiredSigner := cmd.String(flagRequireSignature)
|
||||
if requiredSigner != "" {
|
||||
err = verifyRequiredSigner(chk, requiredSigner)
|
||||
// Check signature requirement. An empty fingerprint is checked too,
|
||||
// and refused as one of the wrong length.
|
||||
if cmd.IsSet(flagRequireSignature) {
|
||||
err = verifyRequiredSigner(chk, cmd.String(flagRequireSignature))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -26,8 +26,7 @@ func noColorEnvSet() bool {
|
||||
// Use DefaultRunOptions for standard CLI execution, or construct manually for testing.
|
||||
type RunOptions struct {
|
||||
Appname string // Application name displayed in help and version output
|
||||
Version string // Version string (typically set at build time)
|
||||
Gitrev string // Git revision hash (typically set at build time)
|
||||
Gitrev string // Revision stamped in at build time, or empty
|
||||
Args []string // Command-line arguments (typically os.Args)
|
||||
Stdin io.Reader // Standard input stream
|
||||
Stdout io.Writer // Standard output stream
|
||||
@@ -36,10 +35,9 @@ type RunOptions struct {
|
||||
}
|
||||
|
||||
// DefaultRunOptions returns RunOptions configured for normal CLI execution.
|
||||
func DefaultRunOptions(appname, version, gitrev string) *RunOptions {
|
||||
func DefaultRunOptions(appname, gitrev string) *RunOptions {
|
||||
return &RunOptions{
|
||||
Appname: appname,
|
||||
Version: version,
|
||||
Gitrev: gitrev,
|
||||
Args: os.Args,
|
||||
Stdin: os.Stdin,
|
||||
@@ -50,15 +48,14 @@ func DefaultRunOptions(appname, version, gitrev string) *RunOptions {
|
||||
}
|
||||
|
||||
// Run creates and runs the CLI application with default options.
|
||||
func Run(appname, version, gitrev string) int {
|
||||
return RunWithOptions(DefaultRunOptions(appname, version, gitrev))
|
||||
func Run(appname, gitrev string) int {
|
||||
return RunWithOptions(DefaultRunOptions(appname, gitrev))
|
||||
}
|
||||
|
||||
// RunWithOptions creates and runs the CLI application with the given options.
|
||||
func RunWithOptions(opts *RunOptions) int {
|
||||
m := &CLIApp{
|
||||
appname: opts.Appname,
|
||||
version: opts.Version,
|
||||
gitrev: opts.Gitrev,
|
||||
exitCode: 0,
|
||||
maxManifestSize: mfer.MaxManifestSize,
|
||||
|
||||
+190
-28
@@ -11,6 +11,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
|
||||
const (
|
||||
testApp = "mfer"
|
||||
testGitrev = "abc123"
|
||||
testDir = "/testdir"
|
||||
testFile1 = "/testdir/file1.txt"
|
||||
testMF = "/testdir/test.mf"
|
||||
@@ -36,6 +38,8 @@ const (
|
||||
testFlagNoExtra = "--no-extra-files"
|
||||
testFlagVersion = "--version"
|
||||
testFlagVerbose = "--verbose"
|
||||
testFlagHelp = "--help"
|
||||
testCmdHelp = "help"
|
||||
)
|
||||
|
||||
var errSimulatedWrite = errors.New("simulated write failure")
|
||||
@@ -58,9 +62,10 @@ func runCLI(opts *RunOptions) int {
|
||||
// runCLIWithLevel is runCLI that also returns the log level the run left
|
||||
// set, read while runMu still keeps other runs from changing it. Each run
|
||||
// starts at the default level, as a new process does. Before releasing the
|
||||
// lock it points the process-global logger at io.Discard: other tests log
|
||||
// outside the lock (manifest loads, scans), and those lines must not land in
|
||||
// this run's buffers once it has returned and its test is reading them.
|
||||
// lock it points the process-global logger at io.Discard: test helpers also
|
||||
// log under runMu outside any run (manifest loads, scans), and those lines
|
||||
// must not land in this run's buffers once it has returned and its test is
|
||||
// reading them.
|
||||
func runCLIWithLevel(opts *RunOptions) (int, log.Level) {
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
@@ -93,8 +98,7 @@ func TestBuild(t *testing.T) {
|
||||
func testOpts(args []string, fs afero.Fs) *RunOptions {
|
||||
return &RunOptions{
|
||||
Appname: testApp,
|
||||
Version: "1.0.0",
|
||||
Gitrev: "abc123",
|
||||
Gitrev: testGitrev,
|
||||
Args: args,
|
||||
Stdin: &bytes.Buffer{},
|
||||
Stdout: &bytes.Buffer{},
|
||||
@@ -136,10 +140,35 @@ func TestVersionCommand(t *testing.T) {
|
||||
exitCode := runCLI(opts)
|
||||
|
||||
assert.Equal(t, 0, exitCode)
|
||||
assert.Equal(t, "mfer version "+testGitrev+"\n", testStdout(t, opts))
|
||||
}
|
||||
|
||||
stdout := testStdout(t, opts)
|
||||
assert.Contains(t, stdout, mfer.Version)
|
||||
assert.Contains(t, stdout, "abc123")
|
||||
// TestVersionFromBuildInfo covers a binary whose build stamped no Gitrev,
|
||||
// as `go install` builds it: mfer version prints the main module's version
|
||||
// from the build information instead.
|
||||
func TestVersionFromBuildInfo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
require.True(t, ok)
|
||||
require.NotEmpty(t, info.Main.Version)
|
||||
|
||||
opts := testOpts([]string{testApp, cmdVersion}, afero.NewMemMapFs())
|
||||
opts.Gitrev = ""
|
||||
|
||||
require.Equal(t, 0, runCLI(opts))
|
||||
assert.Equal(t, "mfer version "+info.Main.Version+"\n", testStdout(t, opts))
|
||||
}
|
||||
|
||||
// TestModuleVersionEmpty covers build information that holds no module
|
||||
// version, as in a build with GO111MODULE=off, and a binary with no build
|
||||
// information at all. Both report "unknown": an empty version would print
|
||||
// nothing after "mfer version" and drop the --version flag.
|
||||
func TestModuleVersionEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, "unknown", moduleVersion(&debug.BuildInfo{}, true))
|
||||
assert.Equal(t, "unknown", moduleVersion(nil, false))
|
||||
}
|
||||
|
||||
// TestVFlagCollision covers the -v/--verbose vs --version flag interaction
|
||||
@@ -165,7 +194,7 @@ func TestVFlagCollision(t *testing.T) {
|
||||
exitCode := runCLI(opts)
|
||||
|
||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
||||
assert.Contains(t, testStdout(t, opts), mfer.Version)
|
||||
assert.Contains(t, testStdout(t, opts), testGitrev)
|
||||
assert.NotContains(t, testStderr(t, opts), "two forms of the same flag")
|
||||
})
|
||||
}
|
||||
@@ -206,13 +235,13 @@ func TestVersionFlagAndCommandMatch(t *testing.T) {
|
||||
|
||||
// TestRootVerbosityFlags asserts that -q and -v given before any subcommand
|
||||
// name take effect: in the root itself, and in the fetch and export
|
||||
// subcommands (issue #64). It does not run in parallel: other tests log
|
||||
// outside runMu (manifest loads, scans), and a line logged while one of these
|
||||
// runs is in progress lands in its output.
|
||||
//
|
||||
//nolint:paralleltest // see above
|
||||
// subcommands (issue #64).
|
||||
func TestRootVerbosityFlags(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("quiet with no subcommand hides the banner", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
loud := testOpts([]string{testApp}, afero.NewMemMapFs())
|
||||
require.Equal(t, 0, runCLI(loud))
|
||||
assert.Contains(t, testStdout(t, loud), banner)
|
||||
@@ -224,6 +253,8 @@ func TestRootVerbosityFlags(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("quiet before fetch hides the banner", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts([]string{testApp, "-q", cmdFetch}, afero.NewMemMapFs())
|
||||
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
@@ -232,6 +263,8 @@ func TestRootVerbosityFlags(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("verbose twice before fetch enables debug logging", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts([]string{testApp, "-v", "-v", cmdFetch}, afero.NewMemMapFs())
|
||||
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
@@ -239,6 +272,8 @@ func TestRootVerbosityFlags(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("quiet before export hides the manifest summary", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
manifest := buildTestManifest(t, map[string][]byte{"a.txt": []byte("a")})
|
||||
require.NoError(t, afero.WriteFile(fs, testManifest, manifest, 0o644))
|
||||
@@ -321,7 +356,7 @@ func TestHelpCommand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
opts := testOpts([]string{testApp, "--help"}, fs)
|
||||
opts := testOpts([]string{testApp, testFlagHelp}, fs)
|
||||
|
||||
exitCode := runCLI(opts)
|
||||
|
||||
@@ -739,6 +774,9 @@ func TestBannerOutput(t *testing.T) {
|
||||
stdout := testStdout(t, opts)
|
||||
assert.Contains(t, stdout, "___")
|
||||
assert.Contains(t, stdout, "\\")
|
||||
|
||||
// The banner shows the version mfer version prints, and nothing after it.
|
||||
assert.Contains(t, stdout, "mfer by @sneak: "+testGitrev+"\n")
|
||||
}
|
||||
|
||||
func TestUnknownCommand(t *testing.T) {
|
||||
@@ -771,13 +809,9 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
|
||||
assert.True(t, exists)
|
||||
|
||||
// Verify manifest only has 1 file (the non-dotfile)
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: testMF,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, manifest.Files(), 1)
|
||||
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
|
||||
paths := manifestPaths(t, fs, testMF)
|
||||
require.Len(t, paths, 1)
|
||||
assert.Equal(t, "file1.txt", paths[0])
|
||||
}
|
||||
|
||||
func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
||||
@@ -798,12 +832,7 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
||||
require.Equal(t, 0, exitCode)
|
||||
|
||||
// Verify manifest has 2 files (including dotfile)
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: testMF,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, manifest.Files(), 2)
|
||||
assert.Len(t, manifestPaths(t, fs, testMF), 2)
|
||||
}
|
||||
|
||||
func TestMultipleInputPaths(t *testing.T) {
|
||||
@@ -960,6 +989,53 @@ func TestCheckWarnsAboutUnlistedFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckReportsUnlistedSymlinkToFileOnly makes a tree holding a symlink
|
||||
// to a directory, one to a file and a broken one. gen lists the symlink to
|
||||
// the file only with --follow-symlinks, and never lists the other two, so
|
||||
// check --no-extra-files passes on the manifest gen --follow-symlinks writes
|
||||
// and fails only on the symlink to the file on the one plain gen writes.
|
||||
func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
genFlags []string
|
||||
exitCode int
|
||||
extras []string // what check reports as not in the manifest
|
||||
}{
|
||||
"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
|
||||
"plain gen": {nil, 1, []string{"filelink"}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
fs := afero.NewOsFs()
|
||||
writeTestTree(t, fs, dir)
|
||||
require.NoError(t, os.Symlink(testSubdir, filepath.Join(dir, "dirlink")))
|
||||
require.NoError(t, os.Symlink(testFileTxt, filepath.Join(dir, "filelink")))
|
||||
require.NoError(t,
|
||||
os.Symlink("nowhere", filepath.Join(dir, "brokenlink")))
|
||||
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdGenerate, "-q"}, tc.genFlags, []string{dir},
|
||||
), fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
opts = testOpts([]string{testApp, cmdCheck, testFlagNoExtra, dir}, fs)
|
||||
exitCode := runCLI(opts)
|
||||
stderr := testStderr(t, opts)
|
||||
|
||||
assert.Equal(t, tc.exitCode, exitCode, "stderr: %s", stderr)
|
||||
assert.Equal(t, len(tc.extras),
|
||||
strings.Count(stderr, "not in manifest"), stderr)
|
||||
|
||||
for _, extra := range tc.extras {
|
||||
assert.Contains(t, stderr, "EXTRA: "+extra)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckNeverReportsManifest keeps the manifest inside the checked tree,
|
||||
// under several names and path spellings, and names the tree both directly
|
||||
// and through a symlink: the manifest is never reported, even under
|
||||
@@ -1224,6 +1300,11 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
||||
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
||||
t.Helper()
|
||||
|
||||
// Loading a manifest logs to the process-global logger, so serialize
|
||||
// with the CLI runs.
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: path,
|
||||
Fs: fs,
|
||||
@@ -1807,6 +1888,87 @@ func TestGenerateReadsPathAfterDoubleDash(t *testing.T) {
|
||||
assert.Equal(t, testFileTxt, files[0].GetPath())
|
||||
}
|
||||
|
||||
// TestPathNamedHelp runs gen, then check with --require-signature, on a
|
||||
// directory named help and on one named h, the two names of urfave/cli's
|
||||
// help command. gen must write the manifest into the directory, and check
|
||||
// must refuse it as unsigned.
|
||||
//
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestPathNamedHelp(t *testing.T) {
|
||||
for _, name := range []string{testCmdHelp, "h"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
chdirTemp(t)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.Mkdir(name, 0o755))
|
||||
writeTestFile(t, fs, filepath.Join(name, testFileTxt), "content")
|
||||
|
||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", name}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
files := manifestFiles(t, fs, filepath.Join(name, defaultManifestName))
|
||||
require.Len(t, files, 1)
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q", name, "--" + flagRequireSignature, msgFpA,
|
||||
}, fs)
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), "manifest is not signed")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCommandsReadHelpAsArgument runs each command that takes a path or URL
|
||||
// on help and on h where nothing by that name exists. Each must fail on the
|
||||
// missing path or bad URL instead of printing its help.
|
||||
func TestCommandsReadHelpAsArgument(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range []string{
|
||||
cmdGenerate, cmdCheck, cmdFreshen, cmdList, cmdExport, cmdFetch,
|
||||
} {
|
||||
for _, name := range []string{testCmdHelp, "h"} {
|
||||
t.Run(command+" "+name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts([]string{testApp, command, name}, afero.NewMemMapFs())
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.NotContains(t, testStdout(t, opts), "USAGE:")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHelpBesideArgument runs each command that takes a path or URL with
|
||||
// --help or -h before and after a directory, and runs help on that command.
|
||||
// Each must print what the command's own --help prints.
|
||||
func TestHelpBesideArgument(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, command := range []string{
|
||||
cmdGenerate, cmdCheck, cmdFreshen, cmdList, cmdExport, cmdFetch,
|
||||
} {
|
||||
want := testOpts([]string{testApp, command, testFlagHelp}, afero.NewMemMapFs())
|
||||
require.Equal(t, 0, runCLI(want))
|
||||
require.Contains(t, testStdout(t, want), testApp+" "+command)
|
||||
|
||||
for _, args := range [][]string{
|
||||
{command, testDir, testFlagHelp},
|
||||
{command, testFlagHelp, testDir},
|
||||
{command, testDir, "-h"},
|
||||
{testCmdHelp, command},
|
||||
} {
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testOpts(slices.Concat([]string{testApp}, args), afero.NewMemMapFs())
|
||||
assert.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
assert.Equal(t, testStdout(t, want), testStdout(t, opts))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDetectsManifestCorruption(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -54,6 +54,11 @@ func runLocked(fn func() error) error {
|
||||
func unsignedChecker(t *testing.T) *mfer.Checker {
|
||||
t.Helper()
|
||||
|
||||
// The scan and the manifest load log to the process-global logger, so
|
||||
// serialize with the CLI runs.
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll("/d", 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
|
||||
@@ -183,6 +188,11 @@ func signedManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
func signedChecker(t *testing.T, manifest []byte) *mfer.Checker {
|
||||
t.Helper()
|
||||
|
||||
// Loading a manifest logs to the process-global logger, so serialize
|
||||
// with the CLI runs.
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/index.mf", manifest, 0o644))
|
||||
|
||||
@@ -473,6 +483,49 @@ func TestFreshenReadErrorMessage(t *testing.T) {
|
||||
"read "+filepath.Join(root, "sub")+": is a directory")
|
||||
}
|
||||
|
||||
// TestFreshenSizeChangedMessage has freshen hash a file that has grown or
|
||||
// shrunk since the scan recorded its size.
|
||||
func TestFreshenSizeChangedMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
writeTestFile(t, fs, "/d/log.txt", "hello")
|
||||
|
||||
t.Run("grown", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hasher := &freshenHasher{
|
||||
fs: fs,
|
||||
absBase: "/d",
|
||||
builder: mfer.NewBuilder(),
|
||||
}
|
||||
|
||||
err := hasher.processEntry(
|
||||
&freshenEntry{path: "log.txt", size: 3, needsHash: true})
|
||||
require.ErrorIs(t, err, errSizeMismatch)
|
||||
assert.EqualError(t, err, `size mismatch for "log.txt": `+
|
||||
`3 bytes when scanned but 5 bytes when read; `+
|
||||
`the file changed while freshen ran`)
|
||||
})
|
||||
|
||||
t.Run("shrunk", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hasher := &freshenHasher{
|
||||
fs: fs,
|
||||
absBase: "/d",
|
||||
builder: mfer.NewBuilder(),
|
||||
}
|
||||
|
||||
err := hasher.processEntry(
|
||||
&freshenEntry{path: "log.txt", size: 8, needsHash: true})
|
||||
require.ErrorIs(t, err, errSizeMismatch)
|
||||
assert.EqualError(t, err, `size mismatch for "log.txt": `+
|
||||
`8 bytes when scanned but 5 bytes when read; `+
|
||||
`the file changed while freshen ran`)
|
||||
})
|
||||
}
|
||||
|
||||
func TestURLRequiredMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -20,6 +20,11 @@ const testCmdExport = "export"
|
||||
func buildTestManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
// The scan logs to the process-global logger, so serialize with the
|
||||
// CLI runs.
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
sourceFs := afero.NewMemMapFs()
|
||||
for path, content := range files {
|
||||
require.NoError(t, sourceFs.MkdirAll("/", 0o755))
|
||||
|
||||
+43
-20
@@ -86,7 +86,8 @@ var (
|
||||
// symlink that already exists in the target directory.
|
||||
errSymlinkInPath = errors.New("symlink in path not allowed")
|
||||
// errSizeMismatch indicates a downloaded file with an unexpected
|
||||
// size.
|
||||
// size, or a file freshen read at a size other than the one its scan
|
||||
// recorded.
|
||||
errSizeMismatch = errors.New("size mismatch")
|
||||
// errHashMismatch indicates a downloaded file whose hash matches no
|
||||
// manifest hash.
|
||||
@@ -125,8 +126,9 @@ type retryingClient struct {
|
||||
// to a limit that doubles each time, unless the server's Retry-After
|
||||
// header says how long to wait. Any other failure, an error from use
|
||||
// included, is returned at once and unwrapped; a non-OK status is
|
||||
// returned as errHTTPStatus. use must start over each time it is called,
|
||||
// since a retry calls it again with a new response.
|
||||
// returned as errHTTPStatus. If ctx ends before a retry, the error wraps
|
||||
// both ctx's error and the last try's error. use must start over each
|
||||
// time it is called, since a retry calls it again with a new response.
|
||||
func (c retryingClient) get(
|
||||
ctx context.Context, rawURL string, use func(*http.Response) error,
|
||||
) error {
|
||||
@@ -168,15 +170,19 @@ func (c retryingClient) get(
|
||||
}
|
||||
}
|
||||
|
||||
if !retry || attempt == fetchAttempts || ctx.Err() != nil {
|
||||
if !retry || attempt == fetchAttempts {
|
||||
return err
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("%w: %w", ctx.Err(), err)
|
||||
}
|
||||
|
||||
log.Warnf("%s: %s, retrying in %s", rawURL, err, wait.Round(time.Millisecond))
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
return fmt.Errorf("%w: %w", ctx.Err(), err)
|
||||
case <-time.After(wait):
|
||||
}
|
||||
|
||||
@@ -429,6 +435,14 @@ func (mfa *CLIApp) fetchManifestOperation(
|
||||
func (mfa *CLIApp) fetchManifest(
|
||||
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
||||
) ([]byte, []*mfer.MFFilePath, error) {
|
||||
// An empty fingerprint is refused before anything is requested, with
|
||||
// the message check gives for it; any other is checked against the
|
||||
// manifest once it is downloaded.
|
||||
requiredSigner := cmd.String(flagRequireSignature)
|
||||
if cmd.IsSet(flagRequireSignature) && requiredSigner == "" {
|
||||
return nil, nil, fmt.Errorf("%w, got 0", errInvalidFingerprint)
|
||||
}
|
||||
|
||||
log.Infof("fetching manifest from %s", manifestURL)
|
||||
|
||||
// Read the whole manifest before parsing it, so that a connection
|
||||
@@ -460,7 +474,6 @@ func (mfa *CLIApp) fetchManifest(
|
||||
return nil, nil, fmt.Errorf("parse manifest: %w", err)
|
||||
}
|
||||
|
||||
requiredSigner := cmd.String(flagRequireSignature)
|
||||
if requiredSigner != "" {
|
||||
err = verifyFetchedSigner(manifestData, requiredSigner)
|
||||
if err != nil {
|
||||
@@ -679,21 +692,26 @@ func resolveManifestURL(inputURL string) (string, error) {
|
||||
return parsed.JoinPath(defaultManifestName).String(), nil
|
||||
}
|
||||
|
||||
// progressWriter wraps an io.Writer and reports progress to a channel.
|
||||
// progressWriter wraps an io.Writer and reports progress to a channel,
|
||||
// at most once per second and once more when the last byte is written.
|
||||
type progressWriter struct {
|
||||
w io.Writer
|
||||
path string
|
||||
total int64
|
||||
written int64
|
||||
startTime time.Time
|
||||
progress chan<- DownloadProgress
|
||||
w io.Writer
|
||||
path string
|
||||
total int64
|
||||
written int64
|
||||
startTime time.Time
|
||||
lastProgress time.Time
|
||||
progress chan<- DownloadProgress
|
||||
}
|
||||
|
||||
func (pw *progressWriter) Write(p []byte) (int, error) {
|
||||
n, err := pw.w.Write(p)
|
||||
|
||||
pw.written += int64(n)
|
||||
if pw.progress != nil {
|
||||
|
||||
isLast := pw.written == pw.total
|
||||
if pw.progress != nil &&
|
||||
(isLast || time.Since(pw.lastProgress) >= time.Second) {
|
||||
var (
|
||||
bytesPerSec float64
|
||||
eta time.Duration
|
||||
@@ -715,6 +733,8 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
|
||||
BytesPerSec: bytesPerSec,
|
||||
ETA: eta,
|
||||
})
|
||||
|
||||
pw.lastProgress = time.Now()
|
||||
}
|
||||
|
||||
return n, err
|
||||
@@ -899,13 +919,16 @@ func saveResponse(
|
||||
// Set up hash computation
|
||||
h := sha256.New()
|
||||
|
||||
// Create progress-reporting writer that also computes hash
|
||||
// Create progress-reporting writer that also computes hash. The first
|
||||
// update waits a second, as gen's and check's do.
|
||||
start := time.Now()
|
||||
pw := &progressWriter{
|
||||
w: io.MultiWriter(out, h),
|
||||
path: localPath,
|
||||
total: totalBytes,
|
||||
startTime: time.Now(),
|
||||
progress: progress,
|
||||
w: io.MultiWriter(out, h),
|
||||
path: localPath,
|
||||
total: totalBytes,
|
||||
startTime: start,
|
||||
lastProgress: start,
|
||||
progress: progress,
|
||||
}
|
||||
|
||||
// Copy content while hashing and reporting progress. One byte past
|
||||
|
||||
+103
-34
@@ -171,6 +171,11 @@ func TestResolveManifestURL(t *testing.T) {
|
||||
func scanToManifest(t *testing.T, sourceFs afero.Fs) []byte {
|
||||
t.Helper()
|
||||
|
||||
// The scan logs to the process-global logger, so serialize with the
|
||||
// CLI runs.
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: sourceFs})
|
||||
require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
|
||||
|
||||
@@ -314,8 +319,11 @@ func TestFetchFromHTTP(t *testing.T) {
|
||||
// Change to a fresh destination directory for the test
|
||||
destDir := chdirTemp(t)
|
||||
|
||||
// Parse the manifest to get file entries
|
||||
// Parse the manifest to get file entries. Loading a manifest logs to
|
||||
// the process-global logger.
|
||||
runMu.Lock()
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
runMu.Unlock()
|
||||
require.NoError(t, err)
|
||||
|
||||
files := manifest.Files()
|
||||
@@ -365,7 +373,10 @@ func TestFetchHashMismatch(t *testing.T) {
|
||||
// Generate and parse manifest
|
||||
manifestData := scanToManifest(t, sourceFs)
|
||||
|
||||
// Loading a manifest logs to the process-global logger.
|
||||
runMu.Lock()
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
runMu.Unlock()
|
||||
require.NoError(t, err)
|
||||
|
||||
files := manifest.Files()
|
||||
@@ -411,7 +422,10 @@ func TestFetchSizeMismatch(t *testing.T) {
|
||||
// Generate and parse manifest
|
||||
manifestData := scanToManifest(t, sourceFs)
|
||||
|
||||
// Loading a manifest logs to the process-global logger.
|
||||
runMu.Lock()
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
runMu.Unlock()
|
||||
require.NoError(t, err)
|
||||
|
||||
files := manifest.Files()
|
||||
@@ -516,14 +530,17 @@ func TestManifestDownloadStopsAtLimit(t *testing.T) {
|
||||
func TestFetchProgress(t *testing.T) {
|
||||
// Create source filesystem with a larger test file
|
||||
sourceFs := afero.NewMemMapFs()
|
||||
// Create content large enough to trigger multiple progress updates
|
||||
// Content large enough that io.Copy writes it in at least four chunks
|
||||
content := bytes.Repeat([]byte("x"), 100*1024) // 100KB
|
||||
require.NoError(t, afero.WriteFile(sourceFs, "/large.txt", content, 0o644))
|
||||
|
||||
// Generate and parse manifest
|
||||
manifestData := scanToManifest(t, sourceFs)
|
||||
|
||||
// Loading a manifest logs to the process-global logger.
|
||||
runMu.Lock()
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
runMu.Unlock()
|
||||
require.NoError(t, err)
|
||||
|
||||
files := manifest.Files()
|
||||
@@ -559,8 +576,11 @@ func TestFetchProgress(t *testing.T) {
|
||||
}()
|
||||
|
||||
// Download
|
||||
start := time.Now()
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/large.txt", ".", "large.txt", files[0], progress)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
close(progress)
|
||||
<-done
|
||||
|
||||
@@ -569,6 +589,12 @@ func TestFetchProgress(t *testing.T) {
|
||||
// Verify we got progress updates
|
||||
assert.NotEmpty(t, progressUpdates, "should have received progress updates")
|
||||
|
||||
// At most one update per second, plus the one reporting the file
|
||||
// complete: a download that takes under a second sends two at most,
|
||||
// however many chunks it writes.
|
||||
assert.LessOrEqual(t, len(progressUpdates), 2+int(elapsed/time.Second),
|
||||
"one progress update per second at most, plus the last")
|
||||
|
||||
// Verify final progress shows complete
|
||||
if len(progressUpdates) > 0 {
|
||||
last := progressUpdates[len(progressUpdates)-1]
|
||||
@@ -871,23 +897,59 @@ func TestGetHonorsRetryAfter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetStopsWaitingWhenCanceled gives get a server that always answers
|
||||
// 503 and an hour's wait before each retry, then lets the context expire
|
||||
// during that wait. get returning at all shows it stopped waiting.
|
||||
// instant503Transport answers every request with a 503 at once, so the
|
||||
// 503 is in hand before any deadline can pass.
|
||||
type instant503Transport struct{}
|
||||
|
||||
func (instant503Transport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Body: http.NoBody,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TestGetStopsWaitingWhenCanceled answers every request with a 503 and
|
||||
// sets an hour's wait before each retry, then lets the context expire
|
||||
// during that wait. get returning at all shows it stopped waiting; it must
|
||||
// return the deadline error and keep the 503 with it.
|
||||
func TestGetStopsWaitingWhenCanceled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := testClient()
|
||||
client.client.Transport = instant503Transport{}
|
||||
client.firstDelay = time.Hour
|
||||
|
||||
err := getNothing(client, server.URL, 50*time.Millisecond)
|
||||
err := getNothing(client, "http://example.invalid/", 50*time.Millisecond)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
}
|
||||
|
||||
// late503Transport answers every request with a 503, but only once the
|
||||
// request's context has ended. A real server cannot stand in for it: the
|
||||
// client drops a response still on its way when the context ends.
|
||||
type late503Transport struct{}
|
||||
|
||||
func (late503Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
<-req.Context().Done()
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Body: http.NoBody,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TestGetStopsWhenContextEndsDuringRequest lets the context expire while
|
||||
// the first request is still waiting for its 503. get must return the
|
||||
// deadline error and keep the 503 with it.
|
||||
func TestGetStopsWhenContextEndsDuringRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := testClient()
|
||||
client.client.Transport = late503Transport{}
|
||||
|
||||
err := getNothing(client, "http://example.invalid/", 50*time.Millisecond)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
require.ErrorIs(t, err, errHTTPStatus)
|
||||
}
|
||||
|
||||
// TestDownloadFileRetriesToSuccess serves a file that fails twice before
|
||||
@@ -897,9 +959,12 @@ func TestGetStopsWaitingWhenCanceled(t *testing.T) {
|
||||
//nolint:paralleltest // changes the process-global working directory
|
||||
func TestDownloadFileRetriesToSuccess(t *testing.T) {
|
||||
content := []byte("the whole file, every byte of it")
|
||||
manifestData := manifestOf(t, map[string][]byte{testFileTxt: content})
|
||||
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(
|
||||
manifestOf(t, map[string][]byte{testFileTxt: content})))
|
||||
// Loading a manifest logs to the process-global logger.
|
||||
runMu.Lock()
|
||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||
runMu.Unlock()
|
||||
require.NoError(t, err)
|
||||
|
||||
var requests atomic.Int32
|
||||
@@ -922,8 +987,11 @@ func TestDownloadFileRetriesToSuccess(t *testing.T) {
|
||||
|
||||
chdirTemp(t)
|
||||
|
||||
err = downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/"+testFileTxt, ".", testFileTxt, manifest.Files()[0], nil)
|
||||
// downloadFile logs each retry to the process-global logger.
|
||||
err = runLocked(func() error {
|
||||
return downloadFile(context.Background(), testClient(),
|
||||
server.URL+"/"+testFileTxt, ".", testFileTxt, manifest.Files()[0], nil)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(3), requests.Load())
|
||||
|
||||
@@ -1243,10 +1311,10 @@ func TestFetchRequireSignature(t *testing.T) {
|
||||
|
||||
// TestFetchRefusesListedManifestName fetches manifests that list, at the
|
||||
// top of the tree, the name fetch saves the manifest under or that name's
|
||||
// temp file: as a file, as a directory, in capitals, and with a leading
|
||||
// "./". Saving the manifest would replace or remove what is listed there,
|
||||
// so fetch must refuse the manifest before it creates the destination or
|
||||
// requests any file.
|
||||
// temp file: as a file, as a directory and in capitals. Saving the
|
||||
// manifest would replace or remove what is listed there, so fetch must
|
||||
// refuse the manifest before it creates the destination or requests any
|
||||
// file.
|
||||
func TestFetchRefusesListedManifestName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1255,7 +1323,6 @@ func TestFetchRefusesListedManifestName(t *testing.T) {
|
||||
tempPathFor(defaultManifestName),
|
||||
defaultManifestName + "/" + testFileTxt,
|
||||
"INDEX.MF",
|
||||
"./" + defaultManifestName,
|
||||
} {
|
||||
t.Run(listed, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -1331,9 +1398,9 @@ func TestFetchRefusesListedTempName(t *testing.T) {
|
||||
// directory, one of them under a spelling the manifest does not list,
|
||||
// and check reports that file as not in the manifest. So fetch must
|
||||
// refuse each on every filesystem before it creates the destination or
|
||||
// requests any file. A file and a directory with the same name, and a
|
||||
// file listed twice, are refused the same way. A manifest whose names
|
||||
// differ in more than letter case is fetched in full.
|
||||
// requests any file. A file and a directory with the same name are
|
||||
// refused the same way. A manifest whose names differ in more than
|
||||
// letter case is fetched in full.
|
||||
func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1343,7 +1410,6 @@ func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
||||
{"Dir", "dir/x", "dir/x (the file Dir)"},
|
||||
{"Dir/a.txt", "dir/b.txt", "dir/b.txt (the directory Dir)"},
|
||||
{"dir", "dir/x", "dir/x (the file dir)"},
|
||||
{"./x.txt", "x.txt", "x.txt (the file ./x.txt)"},
|
||||
} {
|
||||
t.Run(tc.first+" and "+tc.second, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -1404,14 +1470,18 @@ func TestFetchSetsRecordedMode(t *testing.T) {
|
||||
require.NoError(t, afero.WriteFile(sourceFs, "/"+p, content, modes[p]))
|
||||
}
|
||||
|
||||
scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
|
||||
Fs: sourceFs,
|
||||
IncludePermissions: true,
|
||||
})
|
||||
require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
|
||||
|
||||
var manifest bytes.Buffer
|
||||
require.NoError(t, scanner.ToManifest(context.Background(), &manifest, nil))
|
||||
|
||||
// The scan logs to the process-global logger.
|
||||
require.NoError(t, runLocked(func() error {
|
||||
scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
|
||||
Fs: sourceFs,
|
||||
IncludePermissions: true,
|
||||
})
|
||||
require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
|
||||
|
||||
return scanner.ToManifest(context.Background(), &manifest, nil)
|
||||
}))
|
||||
|
||||
tree := fetchTestHandler(manifest.Bytes(), files)
|
||||
|
||||
@@ -1573,8 +1643,7 @@ func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []
|
||||
}
|
||||
|
||||
// builtManifest returns a manifest of files, built directly rather than
|
||||
// scanned, since a scan lists no hidden files and never a path starting
|
||||
// with "./".
|
||||
// scanned, since a scan lists no hidden files.
|
||||
func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
|
||||
+68
-19
@@ -38,6 +38,10 @@ var (
|
||||
errEntryMissingHash = errors.New("manifest entry has no hash")
|
||||
)
|
||||
|
||||
// errSignKeyNeeded indicates a signed manifest given to freshen with no
|
||||
// key to sign the rewritten one.
|
||||
var errSignKeyNeeded = errors.New("give --sign-key to keep it signed")
|
||||
|
||||
// FreshenStatus contains progress information for the freshen operation.
|
||||
type FreshenStatus struct {
|
||||
Phase string // "scan" or "hash"
|
||||
@@ -311,6 +315,16 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// The scan recorded the size; a file that grew or shrank since would
|
||||
// get an entry whose size and hash describe different contents.
|
||||
if bytesRead != e.size {
|
||||
return fmt.Errorf(
|
||||
"%w for %q: %d bytes when scanned but %d bytes when read; "+
|
||||
"the file changed while freshen ran",
|
||||
errSizeMismatch, e.path, e.size, bytesRead,
|
||||
)
|
||||
}
|
||||
|
||||
h.hashedBytes += bytesRead
|
||||
h.hashedFiles++
|
||||
|
||||
@@ -351,24 +365,39 @@ func writeFreshenedManifest(
|
||||
}
|
||||
|
||||
// newFreshenBuilder constructs the manifest builder configured from CLI
|
||||
// flags.
|
||||
func (mfa *CLIApp) newFreshenBuilder(cmd *cli.Command) (*mfer.Builder, error) {
|
||||
// flags, and returns the fingerprint of the --sign-key key, or "" when
|
||||
// none is given. A --sign-key given with an empty value is refused.
|
||||
func (mfa *CLIApp) newFreshenBuilder(
|
||||
cmd *cli.Command,
|
||||
) (*mfer.Builder, string, error) {
|
||||
builder := mfer.NewBuilder()
|
||||
if cmd.Bool("include-timestamps") {
|
||||
builder.SetIncludeTimestamps(true)
|
||||
}
|
||||
|
||||
var signKeyFingerprint string
|
||||
|
||||
// Set up signing options if sign-key is provided
|
||||
if signKey := cmd.String("sign-key"); signKey != "" {
|
||||
if cmd.IsSet("sign-key") {
|
||||
signKey := cmd.String("sign-key")
|
||||
if signKey == "" {
|
||||
return nil, "", errEmptySignKey
|
||||
}
|
||||
|
||||
signing, err := mfa.signingOptions(signKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
signKeyFingerprint, err = mfer.SecretKeyFingerprint(signing.SecretKey)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("%s: %w", signKey, err)
|
||||
}
|
||||
|
||||
builder.SetSigningOptions(signing)
|
||||
}
|
||||
|
||||
return builder, nil
|
||||
return builder, signKeyFingerprint, nil
|
||||
}
|
||||
|
||||
// freshenScan runs the scan phase against the loaded manifest entries
|
||||
@@ -480,11 +509,19 @@ func runFreshenHash(
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadExistingEntries loads the manifest and indexes its file entries
|
||||
// by path.
|
||||
func (mfa *CLIApp) loadExistingEntries(
|
||||
manifestPath string,
|
||||
) (map[string]*mfer.MFFilePath, error) {
|
||||
// existingManifest is what freshen uses of the manifest it rewrites.
|
||||
type existingManifest struct {
|
||||
entriesByPath map[string]*mfer.MFFilePath
|
||||
signer string // fingerprint of the key it is signed with, "" if unsigned
|
||||
hasCreatedAt bool
|
||||
}
|
||||
|
||||
// loadExistingManifest loads the manifest and indexes its file entries
|
||||
// by path. It refuses a signed manifest when no --sign-key is given, as
|
||||
// the rewritten manifest would be unsigned.
|
||||
func (mfa *CLIApp) loadExistingManifest(
|
||||
cmd *cli.Command, manifestPath string,
|
||||
) (*existingManifest, error) {
|
||||
log.Infof("loading manifest from %s", manifestPath)
|
||||
|
||||
// Load existing manifest
|
||||
@@ -496,6 +533,10 @@ func (mfa *CLIApp) loadExistingEntries(
|
||||
return nil, fmt.Errorf("load manifest: %w", err)
|
||||
}
|
||||
|
||||
if signer := manifest.Signer(); signer != nil && cmd.String("sign-key") == "" {
|
||||
return nil, fmt.Errorf("manifest is signed by %s: %w", signer, errSignKeyNeeded)
|
||||
}
|
||||
|
||||
existingFiles := manifest.Files()
|
||||
log.Infof("manifest contains %d files", len(existingFiles))
|
||||
|
||||
@@ -505,7 +546,11 @@ func (mfa *CLIApp) loadExistingEntries(
|
||||
existingByPath[f.GetPath()] = f
|
||||
}
|
||||
|
||||
return existingByPath, nil
|
||||
return &existingManifest{
|
||||
entriesByPath: existingByPath,
|
||||
signer: string(manifest.Signer()),
|
||||
hasCreatedAt: manifest.CreatedAt() != nil,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (mfa *CLIApp) freshenManifestOperation(
|
||||
@@ -513,20 +558,18 @@ func (mfa *CLIApp) freshenManifestOperation(
|
||||
) error {
|
||||
log.Debug("freshenManifestOperation()")
|
||||
|
||||
showProgress := cmd.Bool("progress")
|
||||
|
||||
// Find manifest file
|
||||
manifestPath, err := mfa.resolveFreshenManifestPath(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder, err := mfa.newFreshenBuilder(cmd)
|
||||
builder, signKeyFingerprint, err := mfa.newFreshenBuilder(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existingByPath, err := mfa.loadExistingEntries(manifestPath)
|
||||
existing, err := mfa.loadExistingManifest(cmd, manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -538,7 +581,7 @@ func (mfa *CLIApp) freshenManifestOperation(
|
||||
|
||||
// Phase 1: Scan filesystem
|
||||
scanner, removed, err := mfa.freshenScan(cmd, manifestPath, absBase,
|
||||
existingByPath)
|
||||
existing.entriesByPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -555,7 +598,7 @@ func (mfa *CLIApp) freshenManifestOperation(
|
||||
hasher := &freshenHasher{
|
||||
fs: mfa.Fs,
|
||||
absBase: absBase,
|
||||
showProgress: showProgress,
|
||||
showProgress: cmd.Bool("progress"),
|
||||
totalHashBytes: totalHashBytes,
|
||||
filesToHash: filesToHash,
|
||||
startHash: time.Now(),
|
||||
@@ -571,8 +614,14 @@ func (mfa *CLIApp) freshenManifestOperation(
|
||||
log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
|
||||
scanner.unchanged, scanner.changed, scanner.added, removed)
|
||||
|
||||
// Skip writing if nothing changed
|
||||
if scanner.changed == 0 && scanner.added == 0 && removed == 0 {
|
||||
// Skip writing only when the manifest would come out the same: nothing
|
||||
// in the tree changed, and the manifest already holds what --sign-key,
|
||||
// --include-timestamps and --include-permissions ask for. The scan
|
||||
// counts a file whose recorded mode is not what --include-permissions
|
||||
// asks for as changed.
|
||||
if scanner.changed == 0 && scanner.added == 0 && removed == 0 &&
|
||||
existing.signer == signKeyFingerprint &&
|
||||
existing.hasCreatedAt == cmd.Bool("include-timestamps") {
|
||||
log.Infof("manifest unchanged, skipping write")
|
||||
|
||||
return nil
|
||||
|
||||
@@ -70,6 +70,11 @@ func runFreshen(t *testing.T, fs afero.Fs, root, manifestPath string) {
|
||||
func manifestFiles(t *testing.T, fs afero.Fs, path string) []*mfer.MFFilePath {
|
||||
t.Helper()
|
||||
|
||||
// Loading a manifest logs to the process-global logger, so serialize
|
||||
// with the CLI runs.
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: path,
|
||||
Fs: fs,
|
||||
@@ -79,6 +84,23 @@ func manifestFiles(t *testing.T, fs afero.Fs, path string) []*mfer.MFFilePath {
|
||||
return manifest.Files()
|
||||
}
|
||||
|
||||
// manifestCreatedAt returns the createdAt timestamp of the manifest at
|
||||
// path, or nil if it has none.
|
||||
func manifestCreatedAt(t *testing.T, fs afero.Fs, path string) *mfer.Timestamp {
|
||||
t.Helper()
|
||||
|
||||
runMu.Lock()
|
||||
defer runMu.Unlock()
|
||||
|
||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||
Path: path,
|
||||
Fs: fs,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return manifest.CreatedAt()
|
||||
}
|
||||
|
||||
// TestFreshenUnchanged freshens a tree that has not changed since gen
|
||||
// made its manifest: the manifest must list the same entries as before.
|
||||
func TestFreshenUnchanged(t *testing.T) {
|
||||
@@ -136,6 +158,41 @@ func TestFreshenRecordsModeOnlyWhenAsked(t *testing.T) {
|
||||
recordedModes())
|
||||
}
|
||||
|
||||
// TestFreshenIncludeTimestampsOnUnchangedTree runs freshen
|
||||
// --include-timestamps on a tree that has not changed since gen made its
|
||||
// manifest, which has no createdAt timestamp. freshen must rewrite the
|
||||
// manifest with one. Run again with the flag it must leave the manifest as
|
||||
// it is; run without it, it must rewrite the manifest without one.
|
||||
func TestFreshenIncludeTimestampsOnUnchangedTree(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
root, manifestPath := setupFreshenDir(t, fs, map[string]string{testFileTxt: "hello"})
|
||||
|
||||
freshenWithTimestamps := func() {
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFreshen, "-q", "--include-timestamps",
|
||||
testFlagBase, root, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
}
|
||||
|
||||
freshenWithTimestamps()
|
||||
assert.NotNil(t, manifestCreatedAt(t, fs, manifestPath))
|
||||
|
||||
before, err := afero.ReadFile(fs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
freshenWithTimestamps()
|
||||
|
||||
after, err := afero.ReadFile(fs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, before, after, "manifest that has a createdAt was rewritten")
|
||||
|
||||
runFreshen(t, fs, root, manifestPath)
|
||||
assert.Nil(t, manifestCreatedAt(t, fs, manifestPath))
|
||||
}
|
||||
|
||||
// assertManifestLists asserts that the manifest at manifestPath lists
|
||||
// exactly the files in want, each with the size and SHA-256 hash of its
|
||||
// content in want and the mtime of the file of that name under root.
|
||||
@@ -243,6 +300,58 @@ func TestFreshenWithChanges(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// growingFileFs appends to one file each time it is opened, as a program
|
||||
// still writing to the file would between freshen's scan and its hash.
|
||||
type growingFileFs struct {
|
||||
afero.Fs
|
||||
|
||||
path string
|
||||
}
|
||||
|
||||
//nolint:ireturn // Open must return afero.File to satisfy afero.Fs.
|
||||
func (f growingFileFs) Open(name string) (afero.File, error) {
|
||||
if name == f.path {
|
||||
file, err := f.OpenFile(name, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = file.WriteString(" world")
|
||||
_ = file.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return f.Fs.Open(name)
|
||||
}
|
||||
|
||||
// TestFreshenFileChangedSinceScan has a new file grow after freshen's scan
|
||||
// recorded its size and before freshen hashes it: freshen must fail and
|
||||
// leave the manifest as it was.
|
||||
func TestFreshenFileChangedSinceScan(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
osFs := afero.NewOsFs()
|
||||
root, manifestPath := setupFreshenDir(t, osFs,
|
||||
map[string]string{testFileTxt: "content"})
|
||||
grown := filepath.Join(root, "log.txt")
|
||||
writeTestFile(t, osFs, grown, "hello")
|
||||
|
||||
before, err := afero.ReadFile(osFs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFreshen, "-q", testFlagBase, root, manifestPath,
|
||||
}, growingFileFs{Fs: osFs, path: grown})
|
||||
assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||
|
||||
after, err := afero.ReadFile(osFs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, before, after, "manifest changed")
|
||||
}
|
||||
|
||||
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
||||
// subdirectory of the tree it lists: the manifest is not listed, while an
|
||||
// ordinary file of the same name at the top of the tree is. The manifest
|
||||
@@ -436,7 +545,11 @@ func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
||||
s := &freshenScanner{
|
||||
existingByPath: map[string]*mfer.MFFilePath{relPath: tc.entry},
|
||||
}
|
||||
|
||||
// recordEntry logs to the process-global logger.
|
||||
runMu.Lock()
|
||||
s.recordEntry(relPath, info)
|
||||
runMu.Unlock()
|
||||
|
||||
require.Len(t, s.entries, 1)
|
||||
assert.Equal(t, tc.needsHash, s.entries[0].needsHash)
|
||||
|
||||
+10
-2
@@ -29,6 +29,8 @@ var (
|
||||
"already exists (use --force to overwrite)")
|
||||
// errEmptyOutput indicates --output given with an empty value.
|
||||
errEmptyOutput = errors.New("--output must not be empty")
|
||||
// errEmptySignKey indicates --sign-key given with an empty value.
|
||||
errEmptySignKey = errors.New("--sign-key must not be empty")
|
||||
)
|
||||
|
||||
// reportEnumProgress renders enumeration progress until the channel
|
||||
@@ -121,7 +123,8 @@ func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
|
||||
}
|
||||
|
||||
// buildScannerOptions constructs scanner options from the CLI flags and
|
||||
// the path the manifest is written to.
|
||||
// the path the manifest is written to. A --sign-key given with an empty
|
||||
// value is refused.
|
||||
func (mfa *CLIApp) buildScannerOptions(
|
||||
cmd *cli.Command, output string,
|
||||
) (*mfer.ScannerOptions, error) {
|
||||
@@ -144,7 +147,12 @@ func (mfa *CLIApp) buildScannerOptions(
|
||||
}
|
||||
|
||||
// Set up signing options if sign-key is provided
|
||||
if signKey := cmd.String("sign-key"); signKey != "" {
|
||||
if cmd.IsSet("sign-key") {
|
||||
signKey := cmd.String("sign-key")
|
||||
if signKey == "" {
|
||||
return nil, errEmptySignKey
|
||||
}
|
||||
|
||||
signing, err := mfa.signingOptions(signKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+67
-33
@@ -6,12 +6,12 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/urfave/cli/v3"
|
||||
"sneak.berlin/go/mfer/internal/log"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
// Command and flag names shared across command definitions and tests.
|
||||
@@ -58,7 +58,6 @@ var errUnexpectedArgument = errors.New("unexpected argument")
|
||||
//nolint:revive // established name used throughout the codebase and tests
|
||||
type CLIApp struct {
|
||||
appname string
|
||||
version string
|
||||
gitrev string
|
||||
startupTime time.Time
|
||||
exitCode int
|
||||
@@ -87,13 +86,26 @@ const banner = `
|
||||
\ \:\ \ \:\ \ \::/ \ \:\
|
||||
\__\/ \__\/ \__\/ \__\/`
|
||||
|
||||
// VersionString returns the version and git revision formatted for display.
|
||||
// VersionString returns the version mfer reports: the revision the build
|
||||
// stamped into main.Gitrev or, when it stamped none, as with `go install`,
|
||||
// the main module's version from the binary's build information.
|
||||
func (mfa *CLIApp) VersionString() string {
|
||||
if mfa.gitrev != "" {
|
||||
return fmt.Sprintf("%s (%s)", mfer.Version, mfa.gitrev)
|
||||
return mfa.gitrev
|
||||
}
|
||||
|
||||
return mfer.Version
|
||||
return moduleVersion(debug.ReadBuildInfo())
|
||||
}
|
||||
|
||||
// moduleVersion returns the main module's version from the binary's build
|
||||
// information, or "unknown" when the binary has none or it holds no version,
|
||||
// as in a build with GO111MODULE=off.
|
||||
func moduleVersion(info *debug.BuildInfo, ok bool) string {
|
||||
if !ok || info.Main.Version == "" {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
return info.Main.Version
|
||||
}
|
||||
|
||||
// printVersion writes the version line shared by the --version flag and the
|
||||
@@ -106,8 +118,7 @@ func (mfa *CLIApp) printBanner() {
|
||||
if log.GetLevel() <= log.InfoLevel {
|
||||
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
||||
_, _ = fmt.Fprintf(mfa.Stdout,
|
||||
" mfer by @sneak: v%s released %s\n",
|
||||
mfer.Version, mfer.ReleaseDate)
|
||||
" mfer by @sneak: %s\n", mfa.VersionString())
|
||||
_, _ = fmt.Fprintln(mfa.Stdout, " https://sneak.berlin/go/mfer")
|
||||
}
|
||||
}
|
||||
@@ -168,6 +179,17 @@ func refuseExtraArgs(_ context.Context, cmd *cli.Command) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// showCommandHelp is the CommandNotFound of the commands that take a path or
|
||||
// URL. Those commands set HideHelpCommand, so that a path or URL named help or
|
||||
// h is an argument rather than urfave/cli's help command; each sets it itself,
|
||||
// because on the root it would also remove `mfer help`. With --help,
|
||||
// urfave/cli looks the command's first argument up as a help topic, so
|
||||
// `check DIR --help` asks for a topic DIR and finds none; this prints the
|
||||
// command's own help instead, as `check --help` does.
|
||||
func showCommandHelp(ctx context.Context, cmd *cli.Command, _ string) {
|
||||
_ = cli.ShowCommandHelp(ctx, cmd.Root(), cmd.Name)
|
||||
}
|
||||
|
||||
// requireSignatureFlag returns the --require-signature flag taken by the
|
||||
// check and fetch subcommands.
|
||||
func requireSignatureFlag() *cli.StringFlag {
|
||||
@@ -191,10 +213,12 @@ func includePermissionsFlag() *cli.BoolFlag {
|
||||
|
||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdGenerate,
|
||||
Aliases: []string{"gen"},
|
||||
Usage: "Generate manifest file",
|
||||
ArgsUsage: "[path ...]",
|
||||
Name: cmdGenerate,
|
||||
Aliases: []string{"gen"},
|
||||
Usage: "Generate manifest file",
|
||||
ArgsUsage: "[path ...]",
|
||||
HideHelpCommand: true,
|
||||
CommandNotFound: showCommandHelp,
|
||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||
mfa.setVerbosity(cmd)
|
||||
mfa.printBanner()
|
||||
@@ -253,10 +277,12 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdCheck,
|
||||
Usage: "Validate files using manifest file",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
ArgValidator: refuseExtraArgs,
|
||||
Name: cmdCheck,
|
||||
Usage: "Validate files using manifest file",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
ArgValidator: refuseExtraArgs,
|
||||
HideHelpCommand: true,
|
||||
CommandNotFound: showCommandHelp,
|
||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||
mfa.setVerbosity(cmd)
|
||||
mfa.printBanner()
|
||||
@@ -287,10 +313,12 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdFreshen,
|
||||
Usage: "Update manifest with changed, new, and removed files",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
ArgValidator: refuseExtraArgs,
|
||||
Name: cmdFreshen,
|
||||
Usage: "Update manifest with changed, new, and removed files",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
ArgValidator: refuseExtraArgs,
|
||||
HideHelpCommand: true,
|
||||
CommandNotFound: showCommandHelp,
|
||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||
mfa.setVerbosity(cmd)
|
||||
mfa.printBanner()
|
||||
@@ -338,10 +366,12 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) exportCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdExport,
|
||||
Usage: "Export manifest contents as JSON",
|
||||
ArgsUsage: "[manifest file or URL]",
|
||||
ArgValidator: refuseExtraArgs,
|
||||
Name: cmdExport,
|
||||
Usage: "Export manifest contents as JSON",
|
||||
ArgsUsage: "[manifest file or URL]",
|
||||
ArgValidator: refuseExtraArgs,
|
||||
HideHelpCommand: true,
|
||||
CommandNotFound: showCommandHelp,
|
||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||
mfa.setVerbosity(cmd)
|
||||
|
||||
@@ -364,12 +394,14 @@ func (mfa *CLIApp) versionCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdList,
|
||||
Aliases: []string{"ls"},
|
||||
Usage: "List files in manifest",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
ArgValidator: refuseExtraArgs,
|
||||
Action: mfa.listManifestOperation,
|
||||
Name: cmdList,
|
||||
Aliases: []string{"ls"},
|
||||
Usage: "List files in manifest",
|
||||
ArgsUsage: manifestArgsUsage,
|
||||
ArgValidator: refuseExtraArgs,
|
||||
HideHelpCommand: true,
|
||||
CommandNotFound: showCommandHelp,
|
||||
Action: mfa.listManifestOperation,
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: "long",
|
||||
@@ -386,10 +418,12 @@ func (mfa *CLIApp) listCommand() *cli.Command {
|
||||
|
||||
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: cmdFetch,
|
||||
Usage: "fetch manifest and referenced files",
|
||||
ArgsUsage: "URL",
|
||||
ArgValidator: refuseExtraArgs,
|
||||
Name: cmdFetch,
|
||||
Usage: "fetch manifest and referenced files",
|
||||
ArgsUsage: "URL",
|
||||
ArgValidator: refuseExtraArgs,
|
||||
HideHelpCommand: true,
|
||||
CommandNotFound: showCommandHelp,
|
||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||
mfa.setVerbosity(cmd)
|
||||
mfa.printBanner()
|
||||
|
||||
@@ -4,8 +4,12 @@ package cli
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -73,6 +77,103 @@ func TestGenAndFreshenSignWithKeyFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFreshenSignedManifestNeedsSignKey runs freshen on a signed manifest
|
||||
// after a file is added. Without --sign-key it must fail, naming the
|
||||
// signer and --sign-key, and leave the manifest as it was; with --sign-key
|
||||
// it must rewrite the manifest, signed.
|
||||
func TestFreshenSignedManifestNeedsSignKey(t *testing.T) {
|
||||
t.Setenv(envSignKeyPassphrase, "")
|
||||
|
||||
secretKey, fingerprint := testSecretKey(t, nil, nil)
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
keyFile := filepath.Join(t.TempDir(), "key.asc")
|
||||
root := t.TempDir()
|
||||
manifestPath := filepath.Join(root, defaultManifestName)
|
||||
|
||||
require.NoError(t, afero.WriteFile(fs, keyFile, secretKey, 0o600))
|
||||
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdGenerate, "-q", testFlagSignKey, keyFile,
|
||||
"-o", manifestPath, root,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
before, err := afero.ReadFile(fs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||
|
||||
opts = testOpts([]string{testApp, cmdFreshen, "-q", manifestPath}, fs)
|
||||
assert.Equal(t, 1, runCLI(opts))
|
||||
assert.Contains(t, testStderr(t, opts), fingerprint)
|
||||
assert.Contains(t, testStderr(t, opts), testFlagSignKey)
|
||||
|
||||
after, err := afero.ReadFile(fs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, before, after)
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdFreshen, "-q", testFlagSignKey, keyFile, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q",
|
||||
"--" + flagRequireSignature, fingerprint, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
assert.Len(t, manifestFiles(t, fs, manifestPath), 2)
|
||||
}
|
||||
|
||||
// TestFreshenSignsUnchangedTree runs freshen with --sign-key on a tree that
|
||||
// has not changed since gen made its unsigned manifest. freshen must
|
||||
// rewrite the manifest signed by that key. Run again with the same key it
|
||||
// must leave the manifest as it is; with another key it must rewrite it
|
||||
// signed by that one.
|
||||
func TestFreshenSignsUnchangedTree(t *testing.T) {
|
||||
t.Setenv(envSignKeyPassphrase, "")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
_, manifestPath := setupFreshenDir(t, fs, map[string]string{testFileTxt: "hello"})
|
||||
|
||||
newKeyFile := func() (string, string) {
|
||||
secretKey, fingerprint := testSecretKey(t, nil, nil)
|
||||
keyFile := filepath.Join(t.TempDir(), "key.asc")
|
||||
require.NoError(t, afero.WriteFile(fs, keyFile, secretKey, 0o600))
|
||||
|
||||
return keyFile, fingerprint
|
||||
}
|
||||
|
||||
freshenAndCheckSignedBy := func(keyFile, fingerprint string) {
|
||||
opts := testOpts([]string{
|
||||
testApp, cmdFreshen, "-q", testFlagSignKey, keyFile, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
|
||||
opts = testOpts([]string{
|
||||
testApp, cmdCheck, "-q",
|
||||
"--" + flagRequireSignature, fingerprint, manifestPath,
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||
}
|
||||
|
||||
keyFile, fingerprint := newKeyFile()
|
||||
freshenAndCheckSignedBy(keyFile, fingerprint)
|
||||
|
||||
before, err := afero.ReadFile(fs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
freshenAndCheckSignedBy(keyFile, fingerprint)
|
||||
|
||||
after, err := afero.ReadFile(fs, manifestPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, before, after, "manifest already signed by the key was rewritten")
|
||||
|
||||
freshenAndCheckSignedBy(newKeyFile())
|
||||
}
|
||||
|
||||
// TestSignWithProtectedKeyNeedsPassphrase runs gen with a protected key,
|
||||
// with MFER_SIGN_KEY_PASSPHRASE empty and no terminal to ask on. gen must
|
||||
// fail, naming the variable, and write no manifest.
|
||||
@@ -207,3 +308,105 @@ func TestGenAsksForPassphraseOnTerminal(t *testing.T) {
|
||||
}, fs)
|
||||
require.Equal(t, 0, runCLI(check), testStderr(t, check))
|
||||
}
|
||||
|
||||
// assertEmptySignKeyRefused runs gen and freshen, each with flags, on a
|
||||
// directory holding a file and an unsigned manifest. Each must fail,
|
||||
// naming --sign-key, and leave the manifest as it was.
|
||||
func assertEmptySignKeyRefused(t *testing.T, flags ...string) {
|
||||
t.Helper()
|
||||
|
||||
manifest := manifestOf(t, map[string][]byte{testFileTxt: []byte("listed")})
|
||||
|
||||
for command, rest := range map[string][]string{
|
||||
cmdGenerate: {"-q", "-f", "-o", testMF, testDir},
|
||||
cmdFreshen: {"-q", testMF},
|
||||
} {
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, testFile1, "hello")
|
||||
require.NoError(t, afero.WriteFile(fs, testMF, manifest, 0o644))
|
||||
|
||||
opts := testOpts(slices.Concat([]string{testApp, command}, flags, rest), fs)
|
||||
assert.Equal(t, 1, runCLI(opts), command)
|
||||
assert.Contains(t, testStderr(t, opts), errEmptySignKey.Error(), command)
|
||||
|
||||
content, err := afero.ReadFile(fs, testMF)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, manifest, content, command)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenAndFreshenRefuseEmptySignKey runs gen and freshen with --sign-key
|
||||
// given an empty value, as an unset shell variable gives it.
|
||||
func TestGenAndFreshenRefuseEmptySignKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assertEmptySignKeyRefused(t, testFlagSignKey, "")
|
||||
}
|
||||
|
||||
// TestGenAndFreshenRefuseEmptySignKeyFromEnvironment runs gen and freshen
|
||||
// with MFER_SIGN_KEY set to an empty string.
|
||||
func TestGenAndFreshenRefuseEmptySignKeyFromEnvironment(t *testing.T) {
|
||||
t.Setenv("MFER_SIGN_KEY", "")
|
||||
|
||||
assertEmptySignKeyRefused(t)
|
||||
}
|
||||
|
||||
// assertEmptyFingerprintRefused runs check on an unsigned manifest, and
|
||||
// fetch of that manifest from a server, each with flags. Each must refuse
|
||||
// the empty fingerprint as malformed, and fetch must request nothing.
|
||||
func assertEmptyFingerprintRefused(t *testing.T, flags ...string) {
|
||||
t.Helper()
|
||||
|
||||
const want = "invalid fingerprint: must be exactly 40 hex characters, got 0"
|
||||
|
||||
files := map[string][]byte{testFileTxt: []byte("listed")}
|
||||
manifest := manifestOf(t, files)
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||
writeTestFile(t, fs, filepath.Join(testDir, testFileTxt), "listed")
|
||||
require.NoError(t, afero.WriteFile(fs, testMF, manifest, 0o644))
|
||||
|
||||
opts := testOpts(slices.Concat(
|
||||
[]string{testApp, cmdCheck, "-q"}, flags, []string{testMF},
|
||||
), fs)
|
||||
assert.Equal(t, 1, runCLI(opts), cmdCheck)
|
||||
assert.Contains(t, testStderr(t, opts), want, cmdCheck)
|
||||
|
||||
var requests atomic.Int32
|
||||
|
||||
tree := fetchTestHandler(manifest, files)
|
||||
|
||||
server := httptest.NewServer(
|
||||
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
tree.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
opts = testOpts(slices.Concat(
|
||||
[]string{testApp, cmdFetch, "-q", "--" + flagDest, t.TempDir()},
|
||||
flags, []string{server.URL},
|
||||
), afero.NewOsFs())
|
||||
assert.Equal(t, 1, runCLI(opts), cmdFetch)
|
||||
assert.Contains(t, testStderr(t, opts), want, cmdFetch)
|
||||
assert.Zero(t, requests.Load(), "fetch made a request")
|
||||
}
|
||||
|
||||
// TestCheckAndFetchRefuseEmptyRequireSignature runs check and fetch with
|
||||
// --require-signature given an empty value, as an unset shell variable
|
||||
// gives it.
|
||||
func TestCheckAndFetchRefuseEmptyRequireSignature(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assertEmptyFingerprintRefused(t, "--"+flagRequireSignature, "")
|
||||
}
|
||||
|
||||
// TestCheckAndFetchRefuseEmptyRequireSignatureFromEnvironment runs check
|
||||
// and fetch with MFER_REQUIRE_SIGNATURE set to an empty string.
|
||||
func TestCheckAndFetchRefuseEmptyRequireSignatureFromEnvironment(t *testing.T) {
|
||||
t.Setenv("MFER_REQUIRE_SIGNATURE", "")
|
||||
|
||||
assertEmptyFingerprintRefused(t)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ var (
|
||||
errPathAbsolute = errors.New("is absolute; must be relative")
|
||||
errPathEmptySegment = errors.New("contains empty segment")
|
||||
errPathDotDot = errors.New("contains '..' segment")
|
||||
errPathDot = errors.New("contains '.' segment")
|
||||
errSizeMismatch = errors.New("size mismatch")
|
||||
errNegativeSize = errors.New("size cannot be negative")
|
||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||
@@ -46,6 +47,7 @@ var (
|
||||
// - Must use forward slashes only (no backslashes)
|
||||
// - Must be relative (no leading /)
|
||||
// - Must not contain ".." segments
|
||||
// - Must not contain "." segments
|
||||
// - Must not contain empty segments (no "//")
|
||||
// - Must not be empty
|
||||
func ValidatePath(p string) error {
|
||||
@@ -73,6 +75,10 @@ func ValidatePath(p string) error {
|
||||
if seg == ".." {
|
||||
return fmt.Errorf("path %q %w", p, errPathDotDot)
|
||||
}
|
||||
|
||||
if seg == "." {
|
||||
return fmt.Errorf("path %q %w", p, errPathDot)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -286,6 +286,9 @@ func TestValidatePath(t *testing.T) {
|
||||
{"has/../traversal", "dot-dot segment"},
|
||||
{"has//double", "empty segment"},
|
||||
{"..", "just dot-dot"},
|
||||
{"./a.txt", "leading dot segment"},
|
||||
{"has/./dot", "interior dot segment"},
|
||||
{".", "just dot"},
|
||||
{string([]byte{0xff, 0xfe}), "invalid UTF-8"},
|
||||
}
|
||||
for _, tt := range invalid {
|
||||
|
||||
+13
-10
@@ -264,11 +264,12 @@ func (c *Checker) Check(
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindExtraFiles walks the filesystem and reports files not in the manifest,
|
||||
// hidden files and directories included. The manifest file itself is not
|
||||
// reported. Anything the search cannot read, such as a directory that cannot
|
||||
// be listed, is reported with StatusError and the search goes on. Results are
|
||||
// sent to the results channel. The channel is closed when done.
|
||||
// FindExtraFiles walks the filesystem and reports the regular files, and
|
||||
// symlinks to them, that are not in the manifest, hidden files and
|
||||
// directories included. The manifest file itself is not reported. Anything
|
||||
// the search cannot read, such as a directory that cannot be listed, is
|
||||
// reported with StatusError and the search goes on. Results are sent to the
|
||||
// results channel. The channel is closed when done.
|
||||
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
|
||||
if results != nil {
|
||||
defer close(results)
|
||||
@@ -318,11 +319,6 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// gen never lists a named pipe, a socket or a device.
|
||||
if info.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// A symlink is compared by what it points to, so a manifest reached
|
||||
// through one is not reported either.
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
@@ -332,6 +328,13 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
||||
}
|
||||
}
|
||||
|
||||
// gen lists regular files and symlinks to them only, so a named
|
||||
// pipe, a socket, a device, a symlink to a directory and a broken
|
||||
// symlink are never reported.
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip the manifest file itself, however its path is spelled
|
||||
if os.SameFile(info, c.manifestInfo) {
|
||||
return nil
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
package mfer
|
||||
|
||||
const (
|
||||
// Version is the current mfer release version.
|
||||
Version = "0.1.0"
|
||||
|
||||
// ReleaseDate is the date on which Version was released.
|
||||
ReleaseDate = "2025-12-17"
|
||||
|
||||
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
|
||||
// data (256 MiB). This prevents decompression bombs from consuming excessive
|
||||
// memory.
|
||||
|
||||
@@ -80,6 +80,8 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
||||
}{
|
||||
{"parent traversal", "../escape"},
|
||||
{"interior traversal", "a/../../escape"},
|
||||
{"leading dot segment", "./a.txt"},
|
||||
{"interior dot segment", "dir/./a.txt"},
|
||||
{"absolute path", "/etc/passwd"},
|
||||
{"backslash path", `a\b`},
|
||||
{"double slash", "a//b"},
|
||||
|
||||
@@ -62,6 +62,12 @@ func TestValidatePathMessagesVerbatim(t *testing.T) {
|
||||
want: `path "a/../b" contains '..' segment`,
|
||||
is: errPathDotDot,
|
||||
},
|
||||
{
|
||||
name: "dot segment",
|
||||
path: "a/./b",
|
||||
want: `path "a/./b" contains '.' segment`,
|
||||
is: errPathDot,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -47,6 +47,23 @@ func (m *manifest) Files() []*MFFilePath {
|
||||
return m.pbInner.GetFiles()
|
||||
}
|
||||
|
||||
// Signer returns the fingerprint of the key that made a loaded manifest's
|
||||
// signature, which loading the manifest checked, or nil if the manifest is
|
||||
// not signed.
|
||||
func (m *manifest) Signer() []byte {
|
||||
if len(m.pbOuter.GetSignature()) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return m.pbOuter.GetSigner()
|
||||
}
|
||||
|
||||
// CreatedAt returns the createdAt timestamp of a loaded manifest, or nil
|
||||
// if the manifest has none.
|
||||
func (m *manifest) CreatedAt() *Timestamp {
|
||||
return m.pbInner.GetCreatedAt()
|
||||
}
|
||||
|
||||
// signatureString generates the canonical string used for signing/verification.
|
||||
// Format: MAGIC-UUID-MULTIHASH where UUID and multihash are hex-encoded.
|
||||
// Requires pbOuter to be set with Uuid and Sha256 fields.
|
||||
|
||||
+2
-2
@@ -333,8 +333,8 @@ type MFFilePath struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// required attributes:
|
||||
// Path invariants: must be valid UTF-8, use forward slashes only,
|
||||
// be relative (no leading /), contain no ".." segments, and no
|
||||
// empty segments (no "//").
|
||||
// be relative (no leading /), contain no ".." or "." segments, and
|
||||
// no empty segments (no "//").
|
||||
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
|
||||
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||
// gotta have at least one:
|
||||
|
||||
+2
-2
@@ -49,8 +49,8 @@ message MFFileOuter {
|
||||
message MFFilePath {
|
||||
// required attributes:
|
||||
// Path invariants: must be valid UTF-8, use forward slashes only,
|
||||
// be relative (no leading /), contain no ".." segments, and no
|
||||
// empty segments (no "//").
|
||||
// be relative (no leading /), contain no ".." or "." segments, and
|
||||
// no empty segments (no "//").
|
||||
string path = 1;
|
||||
int64 size = 2;
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
4f8ea5276a8e66590e71ed154a9176b0f68ffa9ab69d8de5aff0e0d9b731164d mf.proto
|
||||
1668b2f6b8639a9213873474e58dbacb3a4ef77cc27c42253cad8dd076e5f206 mf.proto
|
||||
|
||||
+39
-3
@@ -57,7 +57,9 @@ var (
|
||||
"signature must hold exactly one signature")
|
||||
errNotOneArmoredBlock = errors.New(
|
||||
"must be exactly one armored block and nothing else")
|
||||
errMalformedArmor = errors.New("armor is malformed")
|
||||
errMalformedArmor = errors.New("armor is malformed")
|
||||
errKeyNotValidWhenSigned = errors.New(
|
||||
"signature was made when its key was not valid")
|
||||
)
|
||||
|
||||
// SigningOptions holds the key a manifest is signed with.
|
||||
@@ -81,6 +83,19 @@ func SecretKeyIsProtected(secretKey []byte) (bool, error) {
|
||||
return isProtected(key), nil
|
||||
}
|
||||
|
||||
// SecretKeyFingerprint returns the fingerprint of the OpenPGP secret key
|
||||
// secretKey, armored or binary, as a manifest signed with it records its
|
||||
// signer. It fails unless secretKey holds one version 4 primary key with
|
||||
// its secret key.
|
||||
func SecretKeyFingerprint(secretKey []byte) (string, error) {
|
||||
key, err := readSecretKey(secretKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fingerprint(key), nil
|
||||
}
|
||||
|
||||
// CheckSigningKey fails unless opts can sign now: opts.SecretKey must hold
|
||||
// one version 4 primary key with a secret key that may sign and has not
|
||||
// expired or been revoked, and opts.Passphrase must unlock it when it is
|
||||
@@ -245,8 +260,9 @@ func fingerprint(key *openpgp.Entity) string {
|
||||
|
||||
// verifySignature checks that signature is one good, unexpired OpenPGP
|
||||
// signature over data, made by the one primary key in pubKey or one of its
|
||||
// subkeys, and returns that primary key's fingerprint. signature and pubKey
|
||||
// may each be armored or binary.
|
||||
// subkeys while that key was valid, and returns that primary key's
|
||||
// fingerprint. The key may have expired since. signature and pubKey may
|
||||
// each be armored or binary.
|
||||
func verifySignature(data, signature, pubKey []byte) (string, error) {
|
||||
key, err := readOneKey(pubKey, "embedded public key block", true)
|
||||
if err != nil {
|
||||
@@ -281,6 +297,26 @@ func verifySignature(data, signature, pubKey []byte) (string, error) {
|
||||
return "", fmt.Errorf("verify signature: %w", err)
|
||||
}
|
||||
|
||||
// The check above judges the key as of now. Checking the signature a
|
||||
// second time with the library's clock set to its creation time
|
||||
// requires the primary key, and the subkey that made the signature, to
|
||||
// have been valid then under the key validity rules of RFC 9580. The
|
||||
// key is judged by its current self-signatures and binding signatures:
|
||||
// for a version 4 key the newest self-signature on its primary user
|
||||
// ID, for a version 6 key its newest direct-key signature, and for a
|
||||
// subkey its newest binding signature with the signature embedded in
|
||||
// it. So a signature made before the newest of those is refused.
|
||||
signedAt := &packet.Config{
|
||||
Time: func() time.Time { return sig.CreationTime },
|
||||
}
|
||||
|
||||
_, _, err = openpgp.VerifyDetachedSignature(openpgp.EntityList{key},
|
||||
bytes.NewReader(data), bytes.NewReader(sigData), signedAt)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("verify signature: %w: %w",
|
||||
errKeyNotValidWhenSigned, err)
|
||||
}
|
||||
|
||||
return fingerprint(key), nil
|
||||
}
|
||||
|
||||
|
||||
+160
-2
@@ -4,6 +4,7 @@ package mfer
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/dsa" //nolint:staticcheck // SA1019: tests need a DSA key to refuse
|
||||
"io"
|
||||
"math/big"
|
||||
@@ -176,6 +177,25 @@ func TestBuilderWithSigning(t *testing.T) {
|
||||
"-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
||||
}
|
||||
|
||||
// TestManifestSigner checks that Signer gives the signing key's
|
||||
// fingerprint for a signed manifest, and nil once the signature is taken
|
||||
// out, although the signer field is still there.
|
||||
func TestManifestSigner(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
key := newTestKey(t, nil)
|
||||
data := signedTestManifest(t, &SigningOptions{SecretKey: armoredSecretKeys(t, key)})
|
||||
|
||||
signed, err := NewManifestFromReader(bytes.NewReader(data))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fingerprint(key), string(signed.Signer()))
|
||||
|
||||
unsigned, err := NewManifestFromReader(bytes.NewReader(
|
||||
rewriteOuter(t, data, func(outer *MFFileOuter) { outer.Signature = nil })))
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, unsigned.Signer())
|
||||
}
|
||||
|
||||
func TestScannerWithSigning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -261,6 +281,21 @@ func TestSecretKeyIsProtectedWithoutPassphrase(t *testing.T) {
|
||||
assert.False(t, protected)
|
||||
}
|
||||
|
||||
// TestSecretKeyFingerprint checks that SecretKeyFingerprint gives the
|
||||
// signer that a manifest signed with the key records.
|
||||
func TestSecretKeyFingerprint(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
opts := testSigningOptions(t)
|
||||
|
||||
got, err := SecretKeyFingerprint(opts.SecretKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
m, err := NewManifestFromReader(bytes.NewReader(signedTestManifest(t, opts)))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(m.Signer()), got)
|
||||
}
|
||||
|
||||
// TestSigningKeyFileWithTwoKeys signs with a key file that holds two keys,
|
||||
// as gpg writes it for a user ID that two keys have. It names no one key
|
||||
// to sign with, so signing must fail.
|
||||
@@ -419,8 +454,9 @@ func TestVerifySignature(t *testing.T) {
|
||||
|
||||
// TestVerifySignatureKeyExpiredSince verifies two signatures made in 2020
|
||||
// by a key that expired a day after it was made. A signature with no
|
||||
// expiry time is still good. One that expired an hour after it was made is
|
||||
// refused, although the key has expired too.
|
||||
// expiry time is still good, as is one made then by a signing subkey that
|
||||
// expired a day after it was made. One that expired an hour after it was
|
||||
// made is refused, although the key has expired too.
|
||||
func TestVerifySignatureKeyExpiredSince(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -448,10 +484,132 @@ func TestVerifySignatureKeyExpiredSince(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fingerprint(key), signer)
|
||||
|
||||
withSubkey := newTestKey(t, &packet.Config{Time: config.Time})
|
||||
require.NoError(t, withSubkey.AddSigningSubkey(&packet.Config{
|
||||
Algorithm: packet.PubKeyAlgoEdDSA, Time: config.Time,
|
||||
KeyLifetimeSecs: config.KeyLifetimeSecs,
|
||||
}))
|
||||
|
||||
var subkeySig bytes.Buffer
|
||||
|
||||
require.NoError(t, openpgp.ArmoredDetachSign(&subkeySig, withSubkey,
|
||||
bytes.NewReader(data), config))
|
||||
|
||||
subkeyPubKey, err := armoredPublicKey(withSubkey)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = verifySignature(data, subkeySig.Bytes(), subkeyPubKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = verifySignature(data, expiringSig.Bytes(), pubKey)
|
||||
require.ErrorIs(t, err, pgperrors.ErrSignatureExpired)
|
||||
}
|
||||
|
||||
// signDated returns a detached signature over data by signer, dated when.
|
||||
// openpgp.DetachSign signs only with a key that is valid at the time its
|
||||
// config gives, so this makes the signature packet without that check.
|
||||
func signDated(t *testing.T, signer *packet.PrivateKey, data []byte,
|
||||
when time.Time,
|
||||
) []byte {
|
||||
t.Helper()
|
||||
|
||||
sig := &packet.Signature{
|
||||
SigType: packet.SigTypeBinary,
|
||||
PubKeyAlgo: signer.PubKeyAlgo,
|
||||
Hash: crypto.SHA256,
|
||||
CreationTime: when,
|
||||
IssuerKeyId: &signer.KeyId,
|
||||
}
|
||||
|
||||
h, err := sig.PrepareSign(nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = h.Write(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, sig.Sign(h, signer, nil))
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
require.NoError(t, sig.Serialize(&buf))
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TestVerifySignatureRefusesKeyNotValidWhenSigned verifies signatures
|
||||
// dated when the key or subkey that made them was not valid: after the key
|
||||
// expired; after the key's self-signature, or the signing subkey's binding
|
||||
// signature, expired, the key or subkey having expired since as well; and
|
||||
// before the key was made, also by a key made in 2100. Each must be
|
||||
// refused.
|
||||
func TestVerifySignatureRefusesKeyNotValidWhenSigned(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
made := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
twoDaysLater := made.Add(48 * time.Hour)
|
||||
day := uint32((24 * time.Hour).Seconds())
|
||||
clock := func(when time.Time) func() time.Time {
|
||||
return func() time.Time { return when }
|
||||
}
|
||||
|
||||
expiredAfterADay := newTestKey(t, &packet.Config{
|
||||
Time: clock(made), KeyLifetimeSecs: day,
|
||||
})
|
||||
selfSignatureExpiredAfterADay := newTestKey(t, &packet.Config{
|
||||
Time: clock(made), KeyLifetimeSecs: 30 * day, SigLifetimeSecs: day,
|
||||
})
|
||||
|
||||
withSubkey := newTestKey(t, &packet.Config{Time: clock(made)})
|
||||
require.NoError(t, withSubkey.AddSigningSubkey(&packet.Config{
|
||||
Algorithm: packet.PubKeyAlgoEdDSA, Time: clock(made),
|
||||
KeyLifetimeSecs: 30 * day, SigLifetimeSecs: day,
|
||||
}))
|
||||
subkey := withSubkey.Subkeys[len(withSubkey.Subkeys)-1].PrivateKey
|
||||
|
||||
madeADayLater := newTestKey(t, &packet.Config{
|
||||
Time: clock(made.Add(24 * time.Hour)),
|
||||
})
|
||||
madeIn2100 := newTestKey(t, &packet.Config{
|
||||
Time: clock(time.Date(2100, 1, 1, 0, 0, 0, 0, time.UTC)),
|
||||
})
|
||||
|
||||
for name, test := range map[string]struct {
|
||||
key *openpgp.Entity
|
||||
signer *packet.PrivateKey
|
||||
when time.Time
|
||||
}{
|
||||
"after the key expired": {
|
||||
expiredAfterADay, expiredAfterADay.PrivateKey, twoDaysLater,
|
||||
},
|
||||
"after the self-signature expired": {
|
||||
selfSignatureExpiredAfterADay,
|
||||
selfSignatureExpiredAfterADay.PrivateKey, twoDaysLater,
|
||||
},
|
||||
"after the binding signature expired": {
|
||||
withSubkey, subkey, twoDaysLater,
|
||||
},
|
||||
"before the key was made": {
|
||||
madeADayLater, madeADayLater.PrivateKey, made,
|
||||
},
|
||||
"before a key made in 2100": {
|
||||
madeIn2100, madeIn2100.PrivateKey, made,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := []byte("signed in 2020")
|
||||
|
||||
pubKey, err := armoredPublicKey(test.key)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = verifySignature(data,
|
||||
signDated(t, test.signer, data, test.when), pubKey)
|
||||
require.ErrorIs(t, err, errKeyNotValidWhenSigned)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestSignatureVerification(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user